Compare commits

..

5 Commits

Author SHA1 Message Date
Frostebite
05a3792dc2 ci: run github check integration optionally 2025-08-03 20:03:22 +01:00
Frostebite
53e2b89833 test: add github check integration 2025-08-03 20:03:16 +01:00
Frostebite
d6dc078d5a test: always mock github check requests 2025-08-03 00:50:29 +01:00
Frostebite
b924651eb9 chore: format cloud runner github checks test 2025-08-02 20:15:17 +01:00
Frostebite
44586b3388 fix: mock github checks in tests 2025-08-02 19:38:47 +01:00
186 changed files with 41527 additions and 54544 deletions

View File

@@ -14,8 +14,7 @@
"env": {
"node": true,
"es6": true,
"jest/globals": true,
"es2020": true
"jest/globals": true
},
"rules": {
// Error out for code formatting errors
@@ -78,13 +77,5 @@
"unicorn/prefer-spread": "off",
// Temp disable to prevent mixing changes with other PRs
"i18n-text/no-en": "off"
},
"overrides": [
{
"files": ["jest.setup.js"],
"rules": {
"import/no-commonjs": "off"
}
}
]
}
}

View File

@@ -24,13 +24,6 @@ jobs:
targetPlatform:
- StandaloneOSX # Build a MacOS executable
- iOS # Build an iOS executable
include:
# Additionally test enableGpu build for a standalone windows target
- unityVersion: 6000.0.36f1
targetPlatform: StandaloneOSX
- unityVersion: 6000.0.36f1
targetPlatform: StandaloneOSX
buildProfile: 'Assets/Settings/Build Profiles/Sample macOS Build Profile.asset'
steps:
###########################
@@ -72,7 +65,6 @@ jobs:
projectPath: ${{ matrix.projectPath }}
unityVersion: ${{ matrix.unityVersion }}
targetPlatform: ${{ matrix.targetPlatform }}
buildProfile: ${{ matrix.buildProfile }}
customParameters: -profile SomeProfile -someBoolean -someValue exampleValue
# We use dirty build because we are replacing the default project settings file above
allowDirtyBuild: true
@@ -82,6 +74,6 @@ jobs:
###########################
- uses: actions/upload-artifact@v4
with:
name: Build ${{ matrix.targetPlatform }} on MacOS (${{ matrix.unityVersion }})${{ matrix.buildProfile && ' With Build Profile' || '' }}
name: Build ${{ matrix.targetPlatform }} on MacOS (${{ matrix.unityVersion }})
path: build
retention-days: 14

View File

@@ -34,12 +34,7 @@ jobs:
unityVersion: 2023.2.2f1
targetPlatform: StandaloneWindows64
enableGpu: true
- unityVersion: 6000.0.36f1
targetPlatform: StandaloneWindows64
- unityVersion: 6000.0.36f1
targetPlatform: StandaloneWindows64
buildProfile: 'Assets/Settings/Build Profiles/Sample Windows Build Profile.asset'
steps:
###########################
# Checkout #
@@ -84,7 +79,6 @@ jobs:
projectPath: ${{ matrix.projectPath }}
unityVersion: ${{ matrix.unityVersion }}
targetPlatform: ${{ matrix.targetPlatform }}
buildProfile: ${{ matrix.buildProfile }}
enableGpu: ${{ matrix.enableGpu }}
customParameters: -profile SomeProfile -someBoolean -someValue exampleValue
allowDirtyBuild: true
@@ -146,6 +140,6 @@ jobs:
###########################
- uses: actions/upload-artifact@v4
with:
name: Build ${{ matrix.targetPlatform }} on Windows (${{ matrix.unityVersion }})${{ matrix.enableGpu && ' With GPU' || '' }}${{ matrix.buildProfile && ' With Build Profile' || '' }}
name: Build ${{ matrix.targetPlatform }} on Windows (${{ matrix.unityVersion }})${{ matrix.enableGpu && ' With GPU' || '' }}
path: build
retention-days: 14

37
.github/workflows/cleanup.yml vendored Normal file
View File

@@ -0,0 +1,37 @@
name: Cleanup (cron)
on:
schedule:
- cron: '30 10 * * SUN' # every sunday at 10:30
jobs:
deleteArtifacts:
runs-on: ubuntu-latest
steps:
- name: Delete old artifacts
uses: kolpav/purge-artifacts-action@v1
with:
token: ${{ secrets.GITHUB_TOKEN }}
expire-in: 21 days
cleanupCloudRunner:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
if: github.event.event_type != 'pull_request_target'
with:
lfs: true
- uses: actions/setup-node@v4
with:
node-version: '18'
- run: yarn
- run: yarn run cli --help
env:
AWS_REGION: eu-west-2
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: eu-west-2
- run: yarn run cli -m list-resources
env:
AWS_REGION: eu-west-2
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: eu-west-2

View File

@@ -18,16 +18,15 @@ env:
GKE_CLUSTER: 'game-ci-github-pipelines'
GCP_LOGGING: true
GCP_PROJECT: unitykubernetesbuilder
GCP_LOG_FILE: ${{ github.workspace }}/orchestrator-logs.txt
# Commented out: Using LocalStack tests instead of real AWS
# AWS_REGION: eu-west-2
# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
# AWS_DEFAULT_REGION: eu-west-2
# AWS_STACK_NAME: game-ci-github-pipelines
ORCHESTRATOR_BRANCH: ${{ github.ref }}
ORCHESTRATOR_DEBUG: true
ORCHESTRATOR_DEBUG_TREE: true
GCP_LOG_FILE: ${{ github.workspace }}/cloud-runner-logs.txt
AWS_REGION: eu-west-2
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: eu-west-2
AWS_STACK_NAME: game-ci-github-pipelines
CLOUD_RUNNER_BRANCH: ${{ github.ref }}
CLOUD_RUNNER_DEBUG: true
CLOUD_RUNNER_DEBUG_TREE: true
DEBUG: true
UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
PROJECT_PATH: test-project
@@ -47,14 +46,13 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TARGET_PLATFORM: StandaloneWindows64
orchestratorTests: true
cloudRunnerTests: true
versioning: None
ORCHESTRATOR_CLUSTER: local-docker
# Commented out: Using LocalStack tests instead of real AWS
# AWS_STACK_NAME: game-ci-github-pipelines
CLOUD_RUNNER_CLUSTER: local-docker
AWS_STACK_NAME: game-ci-github-pipelines
CHECKS_UPDATE: ${{ github.event.inputs.checksObject }}
run: |
git clone -b main https://github.com/game-ci/unity-builder
git clone -b cloud-runner-develop https://github.com/game-ci/unity-builder
cd unity-builder
yarn
ls

View File

@@ -0,0 +1,231 @@
name: Cloud Runner CI Pipeline
on:
push: { branches: [cloud-runner-develop, cloud-runner-preview, main] }
workflow_dispatch:
inputs:
runGithubIntegrationTests:
description: 'Run GitHub Checks integration tests'
required: false
default: 'false'
permissions:
checks: write
contents: read
actions: write
env:
GKE_ZONE: 'us-central1'
GKE_REGION: 'us-central1'
GKE_PROJECT: 'unitykubernetesbuilder'
GKE_CLUSTER: 'game-ci-github-pipelines'
GCP_LOGGING: true
GCP_PROJECT: unitykubernetesbuilder
GCP_LOG_FILE: ${{ github.workspace }}/cloud-runner-logs.txt
AWS_REGION: eu-west-2
AWS_DEFAULT_REGION: eu-west-2
AWS_STACK_NAME: game-ci-team-pipelines
CLOUD_RUNNER_BRANCH: ${{ github.ref }}
DEBUG: true
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
UNITY_VERSION: 2019.3.15f1
USE_IL2CPP: false
USE_GKE_GCLOUD_AUTH_PLUGIN: true
jobs:
tests:
name: Tests
if: github.event.event_type != 'pull_request_target'
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
test:
- 'cloud-runner-end2end-locking'
- 'cloud-runner-end2end-caching'
- 'cloud-runner-end2end-retaining'
- 'cloud-runner-caching'
- 'cloud-runner-environment'
- 'cloud-runner-image'
- 'cloud-runner-hooks'
- 'cloud-runner-local-persistence'
- 'cloud-runner-locking-core'
- 'cloud-runner-locking-get-locked'
steps:
- name: Checkout (default)
uses: actions/checkout@v4
with:
lfs: false
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: eu-west-2
- run: yarn
- run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
timeout-minutes: 60
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: local-docker
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
k8sTests:
name: K8s Tests
if: github.event.event_type != 'pull_request_target'
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
test:
# - 'cloud-runner-async-workflow'
- 'cloud-runner-end2end-locking'
- 'cloud-runner-end2end-caching'
- 'cloud-runner-end2end-retaining'
- 'cloud-runner-kubernetes'
- 'cloud-runner-environment'
- 'cloud-runner-github-checks'
steps:
- name: Checkout (default)
uses: actions/checkout@v2
with:
lfs: false
- run: yarn
- name: actions-k3s
uses: debianmaster/actions-k3s@v1.0.5
with:
version: 'latest'
- run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
timeout-minutes: 60
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: k8s
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
awsTests:
name: AWS Tests
if: github.event.event_type != 'pull_request_target'
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
test:
- 'cloud-runner-end2end-locking'
- 'cloud-runner-end2end-caching'
- 'cloud-runner-end2end-retaining'
- 'cloud-runner-environment'
- 'cloud-runner-s3-steps'
steps:
- name: Checkout (default)
uses: actions/checkout@v2
with:
lfs: false
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: eu-west-2
- run: yarn
- run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
timeout-minutes: 60
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
buildTargetTests:
name: Local Build Target Tests
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
providerStrategy:
#- aws
- local-docker
#- k8s
targetPlatform:
- StandaloneOSX # Build a macOS standalone (Intel 64-bit).
- StandaloneWindows64 # Build a Windows 64-bit standalone.
- StandaloneLinux64 # Build a Linux 64-bit standalone.
- WebGL # WebGL.
- iOS # Build an iOS player.
# - Android # Build an Android .apk.
steps:
- name: Checkout (default)
uses: actions/checkout@v4
with:
lfs: false
- run: yarn
- uses: ./
id: unity-build
timeout-minutes: 30
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
cloudRunnerTests: true
versioning: None
targetPlatform: ${{ matrix.targetPlatform }}
providerStrategy: ${{ matrix.providerStrategy }}
- run: |
cp ./cloud-runner-cache/cache/${{ steps.unity-build.outputs.CACHE_KEY }}/build/${{ steps.unity-build.outputs.BUILD_ARTIFACT }} ${{ steps.unity-build.outputs.BUILD_ARTIFACT }}
- uses: actions/upload-artifact@v4
with:
name: ${{ matrix.providerStrategy }} Build (${{ matrix.targetPlatform }})
path: ${{ steps.unity-build.outputs.BUILD_ARTIFACT }}
retention-days: 14
githubChecksIntegration:
name: GitHub Checks Integration
runs-on: ubuntu-latest
if: github.event_name == 'workflow_dispatch' && github.event.inputs.runGithubIntegrationTests == 'true'
env:
RUN_GITHUB_INTEGRATION_TESTS: true
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'yarn'
- run: yarn install --frozen-lockfile
- run: yarn test cloud-runner-github-checks.integration --detectOpenHandles --forceExit --runInBand
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

View File

@@ -4,11 +4,6 @@ on:
push: { branches: [main] }
pull_request: {}
permissions:
contents: read
checks: write
statuses: write
env:
CODECOV_TOKEN: '2f2eb890-30e2-4724-83eb-7633832cf0de'
@@ -27,12 +22,7 @@ jobs:
node-version: '18'
- run: yarn
- run: yarn lint
- run: yarn test:ci --coverage
- run: yarn test --coverage
- run: bash <(curl -s https://codecov.io/bash)
- run: yarn build || { echo "build command should always succeed" ; exit 61; }
# - run: yarn build --quiet && git diff --quiet dist || { echo "dist should be auto generated" ; git diff dist ; exit 62; }
orchestrator:
name: Orchestrator Integrity
uses: ./.github/workflows/orchestrator-integrity.yml
secrets: inherit
# - run: yarn build --quiet && git diff --quiet dist || { echo "dist should be auto generated" ; git diff dist ; exit 62; }

File diff suppressed because it is too large Load Diff

2
.gitignore vendored
View File

@@ -5,5 +5,3 @@ lib/
.vsconfig
yarn-error.log
.orig
$LOG_FILE
temp/

View File

@@ -104,11 +104,11 @@ inputs:
gitPrivateToken:
required: false
default: ''
description: '[Orchestrator] Github private token to pull from github'
description: '[CloudRunner] Github private token to pull from github'
githubOwner:
required: false
default: ''
description: '[Orchestrator] GitHub owner name or organization/team name'
description: '[CloudRunner] GitHub owner name or organization/team name'
runAsHostUser:
required: false
default: 'false'
@@ -149,101 +149,97 @@ inputs:
allowDirtyBuild:
required: false
default: ''
description: '[Orchestrator] Allows the branch of the build to be dirty, and still generate the build.'
description: '[CloudRunner] Allows the branch of the build to be dirty, and still generate the build.'
postBuildSteps:
required: false
default: ''
description:
'[Orchestrator] run a post build job in yaml format with the keys image, secrets (name, value object array),
'[CloudRunner] run a post build job in yaml format with the keys image, secrets (name, value object array),
command string'
preBuildSteps:
required: false
default: ''
description:
'[Orchestrator] Run a pre build job after the repository setup but before the build job (in yaml format with the
'[CloudRunner] Run a pre build job after the repository setup but before the build job (in yaml format with the
keys image, secrets (name, value object array), command line string)'
containerHookFiles:
required: false
default: ''
description:
'[Orchestrator] Specify the names (by file name) of custom steps to run before or after orchestrator jobs, must
'[CloudRunner] Specify the names (by file name) of custom steps to run before or after cloud runner jobs, must
match a yaml step file inside your repo in the folder .game-ci/steps/'
customHookFiles:
required: false
default: ''
description:
'[Orchestrator] Specify the names (by file name) of custom hooks to run before or after orchestrator jobs, must
'[CloudRunner] Specify the names (by file name) of custom hooks to run before or after cloud runner jobs, must
match a yaml step file inside your repo in the folder .game-ci/hooks/'
customCommandHooks:
required: false
default: ''
description: '[Orchestrator] Specify custom commands and trigger hooks (injects commands into jobs)'
description: '[CloudRunner] Specify custom commands and trigger hooks (injects commands into jobs)'
customJob:
required: false
default: ''
description:
'[Orchestrator] Run a custom job instead of the standard build automation for orchestrator (in yaml format with
the keys image, secrets (name, value object array), command line string)'
'[CloudRunner] Run a custom job instead of the standard build automation for cloud runner (in yaml format with the
keys image, secrets (name, value object array), command line string)'
awsStackName:
default: 'game-ci'
required: false
description: '[Orchestrator] The Cloud Formation stack name that must be setup before using this option.'
description: '[CloudRunner] The Cloud Formation stack name that must be setup before using this option.'
providerStrategy:
default: 'local'
required: false
description:
'[Orchestrator] Either local, k8s or aws can be used to run builds on a remote cluster. Additional parameters must
'[CloudRunner] Either local, k8s or aws can be used to run builds on a remote cluster. Additional parameters must
be configured.'
resourceTracking:
default: 'false'
required: false
description: '[Orchestrator] Enable resource tracking logs for disk usage and allocation summaries.'
containerCpu:
default: ''
required: false
description: '[Orchestrator] Amount of CPU time to assign the remote build container'
description: '[CloudRunner] Amount of CPU time to assign the remote build container'
containerMemory:
default: ''
required: false
description: '[Orchestrator] Amount of memory to assign the remote build container'
description: '[CloudRunner] Amount of memory to assign the remote build container'
readInputFromOverrideList:
default: ''
required: false
description: '[Orchestrator] Comma separated list of input value names to read from "input override command"'
description: '[CloudRunner] Comma separated list of input value names to read from "input override command"'
readInputOverrideCommand:
default: ''
required: false
description:
'[Orchestrator] Extend game ci by specifying a command to execute to pull input from external source e.g cloud
'[CloudRunner] Extend game ci by specifying a command to execute to pull input from external source e.g cloud
provider secret managers'
kubeConfig:
default: ''
required: false
description:
'[Orchestrator] Supply a base64 encoded kubernetes config to run builds on kubernetes and stream logs until
'[CloudRunner] Supply a base64 encoded kubernetes config to run builds on kubernetes and stream logs until
completion.'
kubeVolume:
default: ''
required: false
description: '[Orchestrator] Supply a Persistent Volume Claim name to use for the Unity build.'
description: '[CloudRunner] Supply a Persistent Volume Claim name to use for the Unity build.'
kubeStorageClass:
default: ''
required: false
description:
'[Orchestrator] Kubernetes storage class to use for orchestrator jobs, leave empty to install rook cluster.'
'[CloudRunner] Kubernetes storage class to use for cloud runner jobs, leave empty to install rook cluster.'
kubeVolumeSize:
default: '5Gi'
required: false
description: '[Orchestrator] Amount of disc space to assign the Kubernetes Persistent Volume'
description: '[CloudRunner] Amount of disc space to assign the Kubernetes Persistent Volume'
cacheKey:
default: ''
required: false
description: '[Orchestrator] Cache key to indicate bucket for cache'
description: '[CloudRunner] Cache key to indicate bucket for cache'
watchToEnd:
default: 'true'
required: false
description:
'[Orchestrator] Whether or not to watch the build to the end. Can be used for especially long running jobs e.g
'[CloudRunner] Whether or not to watch the build to the end. Can be used for especially long running jobs e.g
imports or self-hosted ephemeral runners.'
cacheUnityInstallationOnMac:
default: 'false'
@@ -269,139 +265,6 @@ inputs:
default: 'false'
required: false
description: 'Skip the activation/deactivation of Unity. This assumes Unity is already activated.'
cloneDepth:
default: '50'
required: false
description: '[Orchestrator] Specifies the depth of the git clone for the repository. Use 0 for full clone.'
orchestratorRepoName:
default: 'game-ci/unity-builder'
required: false
description:
'[Orchestrator] Specifies the repo for the unity builder. Useful if you forked the repo for testing, features, or
fixes.'
submoduleProfilePath:
required: false
default: ''
description:
'Path to a YAML submodule profile file (relative to repo root). Defines which submodules to initialize (branch:
main) or skip (branch: empty). See docs for format.'
submoduleVariantPath:
required: false
default: ''
description:
'Path to a YAML variant overlay file that modifies the base submodule profile. Used for server or debug build
variants.'
submoduleToken:
required: false
default: ''
description:
'Git token for authenticating submodule clones. Falls back to gitPrivateToken or GITHUB_TOKEN if empty.'
localCacheEnabled:
required: false
default: 'false'
description:
'Enable filesystem-based caching for local builds. Caches the Unity Library folder and optionally LFS objects
between builds without requiring actions/cache.'
localCacheRoot:
required: false
default: ''
description:
'Root directory for local build cache. Defaults to $RUNNER_TEMP/game-ci-cache or .game-ci/cache if RUNNER_TEMP is
not set.'
localCacheLibrary:
required: false
default: 'true'
description: 'Cache the Unity Library folder for local builds. Only effective when localCacheEnabled is true.'
localCacheLfs:
required: false
default: 'false'
description: 'Cache Git LFS objects for local builds. Only effective when localCacheEnabled is true.'
childWorkspacesEnabled:
required: false
default: 'false'
description:
'Enable child workspace isolation for multi-product builds. Uses atomic filesystem moves for O(1) workspace
restore instead of tar/download/extract. Ideal for 50GB+ workspaces on self-hosted runners.'
childWorkspaceName:
required: false
default: ''
description:
'Name for this child workspace (e.g., product name like "TurnOfWar"). Used as the cache key for workspace
isolation. Required when childWorkspacesEnabled is true.'
childWorkspaceCacheRoot:
required: false
default: ''
description:
'Parent directory for cached child workspaces. Should be on the same NTFS volume as the build directory for O(1)
atomic restore via filesystem rename. Defaults to $RUNNER_TEMP/game-ci-workspaces.'
childWorkspacePreserveGit:
required: false
default: 'true'
description:
'Preserve .git directory in cached child workspace. Enables delta operations on restore but increases cache size.
Set to false to save disk space at the cost of full re-clone on restore.'
childWorkspaceSeparateLibrary:
required: false
default: 'true'
description:
'Cache Unity Library folder separately from the child workspace. Allows independent Library restore even when
workspace cache is invalidated. Recommended for large projects.'
lfsTransferAgent:
required: false
default: ''
description:
'Custom Git LFS transfer agent. Set to "elastic-git-storage" for built-in support (auto-installs from GitHub
releases). Append @version for a specific release (e.g. "elastic-git-storage@v1.0.0"). Or provide a path to any
custom transfer agent executable. When set, the agent is registered via git config before LFS operations.'
lfsTransferAgentArgs:
required: false
default: ''
description: 'Additional arguments to pass to the custom LFS transfer agent.'
lfsStoragePaths:
required: false
default: ''
description:
'Semicolon-separated list of storage paths for the custom LFS transfer agent. Interpretation depends on the agent
(e.g. local paths, WebDAV URLs, rclone remotes).'
gitHooksEnabled:
required: false
default: 'false'
description:
'Install and run git hooks (lefthook, husky, or native) during builds. When false (default), hooks are disabled
for build performance.'
gitHooksSkipList:
required: false
default: ''
description:
'Comma-separated list of hook names to skip even when gitHooksEnabled is true. Example: pre-push,post-merge'
gitHooksRunBeforeBuild:
required: false
default: ''
description:
'Comma-separated list of lefthook hook groups to run before the Unity build. Allows CI to trigger checks that
normally only run on git events. Example: pre-commit,pre-push. Requires lefthook. Works with Unity Git Hooks
(com.frostebite.unitygithooks) when installed as a UPM package — the init script runs automatically.'
providerExecutable:
required: false
default: ''
description:
'Path to an external CLI executable that implements the provider protocol. Enables providers written in any
language (Go, Python, Rust, shell). Uses JSON-over-stdin/stdout communication.'
middleware:
required: false
default: ''
description:
'[Orchestrator] Inline YAML middleware pipeline definitions. Middleware wraps around build phases with
before/after semantics, supports trigger conditions (phase, provider, platform, environment expressions), and
resolves to either command hooks (inline in build container) or container hooks (separate Docker containers).
Middleware executes in priority order — lower priority values run first for before phases and last for after
phases (wrapping pattern). See documentation for full YAML format.'
middlewareFiles:
required: false
default: ''
description:
'[Orchestrator] Comma-separated list of middleware file names (without extension) to load from the
game-ci/middleware/ directory in your repository. Each file should contain YAML middleware definitions.'
outputs:
volume:

View File

@@ -56,19 +56,14 @@ namespace UnityBuilderAction
// of either `UnityEditor.BuildPlayerOptions` or `UnityEditor.BuildPlayerWithProfileOptions`
dynamic buildPlayerOptions;
if (options.TryGetValue("activeBuildProfile", out var buildProfilePath)) {
if (string.IsNullOrEmpty(buildProfilePath)) {
throw new Exception("`-activeBuildProfile` is set but with an empty value; this shouldn't happen");
}
if (options["customBuildProfile"] != "") {
#if UNITY_6000_0_OR_NEWER
// Load build profile from Assets folder
var buildProfile = AssetDatabase.LoadAssetAtPath<BuildProfile>(buildProfilePath)
?? throw new Exception("Build profile file not found at path: " + buildProfilePath);
BuildProfile buildProfile = AssetDatabase.LoadAssetAtPath<BuildProfile>(options["customBuildProfile"]);
// no need to set active profile, as already set by `-activeBuildProfile` CLI argument
// BuildProfile.SetActiveBuildProfile(buildProfile);
Debug.Log($"build profile: {buildProfile.name}");
// Set it as active
BuildProfile.SetActiveBuildProfile(buildProfile);
// Define BuildPlayerWithProfileOptions
buildPlayerOptions = new BuildPlayerWithProfileOptions {
@@ -76,16 +71,12 @@ namespace UnityBuilderAction
locationPathName = options["customBuildPath"],
options = buildOptions,
};
#else // UNITY_6000_0_OR_NEWER
#else
throw new Exception("Build profiles are not supported by this version of Unity (" + Application.unityVersion +")");
#endif // UNITY_6000_0_OR_NEWER
#endif
} else {
#if BUILD_PROFILE_LOADED
throw new Exception("Build profile's define symbol present; shouldn't happen");
#endif // BUILD_PROFILE_LOADED
// Gather values from project
var scenes = EditorBuildSettings.scenes.Where(scene => scene.enabled).Select(s => s.path).ToArray();

View File

@@ -115,7 +115,6 @@ namespace UnityBuilderAction.Input
}
}
#if UNITY_6000_0_OR_NEWER
private static void SetDebugSymbols(string enumValueName)
{
// UnityEditor.Android.UserBuildSettings and Unity.Android.Types.DebugSymbolLevel are part of the Unity Android module.
@@ -145,6 +144,5 @@ namespace UnityBuilderAction.Input
}
levelProp.SetValue(null, enumValue);
}
#endif
}
}

View File

@@ -21,19 +21,6 @@ namespace UnityBuilderAction.Input
EditorApplication.Exit(110);
}
#if UNITY_6000_0_OR_NEWER
var buildProfileSupport = true;
#else
var buildProfileSupport = false;
#endif // UNITY_6000_0_OR_NEWER
string buildProfile;
if (buildProfileSupport && validatedOptions.TryGetValue("activeBuildProfile", out buildProfile)) {
if (validatedOptions.ContainsKey("buildTarget")) {
Console.WriteLine("Extra argument -buildTarget");
EditorApplication.Exit(122);
}
} else {
string buildTarget;
if (!validatedOptions.TryGetValue("buildTarget", out buildTarget)) {
Console.WriteLine("Missing argument -buildTarget");
@@ -44,7 +31,6 @@ namespace UnityBuilderAction.Input
Console.WriteLine(buildTarget + " is not a defined " + typeof(BuildTarget).Name);
EditorApplication.Exit(121);
}
}
string customBuildPath;
if (!validatedOptions.TryGetValue("customBuildPath", out customBuildPath)) {

74492
dist/index.js generated vendored

File diff suppressed because one or more lines are too long

2
dist/index.js.map generated vendored

File diff suppressed because one or more lines are too long

401
dist/licenses.txt generated vendored
View File

@@ -6865,6 +6865,76 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
@deno/shim-deno
MIT
MIT License
Copyright 2021-2022 the Deno authors
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@deno/shim-deno-test
MIT
MIT License
Copyright 2021-2022 the Deno authors
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@fastify/busboy
MIT
Copyright Brian White. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
IN THE SOFTWARE.
@kubernetes/client-node
Apache-2.0
Apache License
@@ -13750,6 +13820,210 @@ Apache License
See the License for the specific language governing permissions and
limitations under the License.
@smithy/util-body-length-browser
Apache-2.0
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
@smithy/util-body-length-node
Apache-2.0
Apache License
@@ -17052,31 +17326,6 @@ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
IN THE SOFTWARE.
cross-fetch
MIT
The MIT License (MIT)
Copyright (c) 2017 Leonardo Quixadá
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
cross-spawn
MIT
The MIT License (MIT)
@@ -17569,56 +17818,6 @@ The above copyright notice and this permission notice shall be included in all c
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
graphql
MIT
MIT License
Copyright (c) GraphQL Contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
graphql-request
MIT
MIT License
Copyright (c) 2022 Jason Kuhrt
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
har-schema
ISC
Copyright (c) 2015, Ahmad Nassri <ahmad@ahmadnassri.com>
@@ -19412,33 +19611,6 @@ The above copyright notice and this permission notice shall be included in all c
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
shell-quote
MIT
The MIT License
Copyright (c) 2013 James Halliday (mail@substack.net)
Permission is hereby granted, free of charge,
to any person obtaining a copy of this software and
associated documentation files (the "Software"), to
deal in the Software without restriction, including
without limitation the rights to use, copy, modify,
merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom
the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice
shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR
ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
shelljs
BSD-3-Clause
Copyright (c) 2012, Artur Adib <arturadib@gmail.com>
@@ -19773,6 +19945,31 @@ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
undici
MIT
MIT License
Copyright (c) Matteo Collina and Undici contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
unity-changeset
MIT
MIT License

View File

@@ -4,69 +4,21 @@
echo "Changing to \"$ACTIVATE_LICENSE_PATH\" directory."
pushd "$ACTIVATE_LICENSE_PATH"
if [[ -n "$UNITY_SERIAL" && -n "$UNITY_EMAIL" && -n "$UNITY_PASSWORD" ]]; then
#
# SERIAL LICENSE MODE
#
# This will activate unity, using the serial activation process.
#
echo "Requesting activation"
echo "Requesting activation"
# Activate license
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/MacOS/Unity \
-logFile - \
-batchmode \
-nographics \
-quit \
-serial "$UNITY_SERIAL" \
-username "$UNITY_EMAIL" \
-password "$UNITY_PASSWORD" \
-projectPath "$ACTIVATE_LICENSE_PATH"
# Activate license
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/MacOS/Unity \
-logFile - \
-batchmode \
-nographics \
-quit \
-serial "$UNITY_SERIAL" \
-username "$UNITY_EMAIL" \
-password "$UNITY_PASSWORD" \
-projectPath "$ACTIVATE_LICENSE_PATH"
# Store the exit code from the verify command
UNITY_EXIT_CODE=$?
elif [[ -n "$UNITY_LICENSING_SERVER" ]]; then
#
# Custom Unity License Server
#
echo "Adding licensing server config"
mkdir -p "$UNITY_LICENSE_PATH/config/"
cp "$ACTION_FOLDER/unity-config/services-config.json" "$UNITY_LICENSE_PATH/config/services-config.json"
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/Frameworks/UnityLicensingClient.app/Contents/MacOS/Unity.Licensing.Client \
--acquire-floating > license.txt
# Store the exit code from the verify command
UNITY_EXIT_CODE=$?
if [ $UNITY_EXIT_CODE -eq 0 ]; then
PARSEDFILE=$(grep -oE '\"[^"]*\"' < license.txt | tr -d '"')
export FLOATING_LICENSE
FLOATING_LICENSE=$(sed -n 2p <<< "$PARSEDFILE")
FLOATING_LICENSE_TIMEOUT=$(sed -n 4p <<< "$PARSEDFILE")
echo "Acquired floating license: \"$FLOATING_LICENSE\" with timeout $FLOATING_LICENSE_TIMEOUT"
fi
else
#
# NO LICENSE ACTIVATION STRATEGY MATCHED
#
# This will exit since no activation strategies could be matched.
#
echo "License activation strategy could not be determined."
echo ""
echo "Visit https://game.ci/docs/github/activation for more"
echo "details on how to set up one of the possible activation strategies."
echo "::error ::No valid license activation strategy could be determined. Make sure to provide UNITY_EMAIL, UNITY_PASSWORD, and either a UNITY_SERIAL \
or UNITY_LICENSE. Otherwise please use UNITY_LICENSING_SERVER. See more info at https://game.ci/docs/github/activation"
# Immediately exit as no UNITY_EXIT_CODE can be derived.
exit 1;
fi
# Store the exit code from the verify command
UNITY_EXIT_CODE=$?
#
# Display information about the result

View File

@@ -149,13 +149,14 @@ echo ""
$( [ "${MANUAL_EXIT}" == "true" ] || echo "-quit" ) \
-batchmode \
$( [ "${ENABLE_GPU}" == "true" ] || echo "-nographics" ) \
-username "$UNITY_EMAIL" \
-password "$UNITY_PASSWORD" \
-customBuildName "$BUILD_NAME" \
-projectPath "$UNITY_PROJECT_PATH" \
$( [ -z "$BUILD_PROFILE" ] && echo "-buildTarget $BUILD_TARGET") \
-buildTarget "$BUILD_TARGET" \
-customBuildTarget "$BUILD_TARGET" \
-customBuildPath "$CUSTOM_BUILD_PATH" \
-customBuildProfile "$BUILD_PROFILE" \
${BUILD_PROFILE:+-activeBuildProfile} ${BUILD_PROFILE:+"$BUILD_PROFILE"} \
-executeMethod "$BUILD_METHOD" \
-buildVersion "$VERSION" \
-androidVersionCode "$ANDROID_VERSION_CODE" \

View File

@@ -4,29 +4,15 @@
echo "Changing to \"$ACTIVATE_LICENSE_PATH\" directory."
pushd "$ACTIVATE_LICENSE_PATH"
if [[ -n "$UNITY_LICENSING_SERVER" ]]; then
#
# Return any floating license used.
#
echo "Returning floating license: \"$FLOATING_LICENSE\""
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/Frameworks/UnityLicensingClient.app/Contents/MacOS/Unity.Licensing.Client \
--return-floating "$FLOATING_LICENSE"
elif [[ -n "$UNITY_SERIAL" ]]; then
#
# SERIAL LICENSE MODE
#
# This will return the license that is currently in use.
#
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/MacOS/Unity \
-logFile - \
-batchmode \
-nographics \
-quit \
-username "$UNITY_EMAIL" \
-password "$UNITY_PASSWORD" \
-returnlicense \
-projectPath "$ACTIVATE_LICENSE_PATH"
fi
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/MacOS/Unity \
-logFile - \
-batchmode \
-nographics \
-quit \
-username "$UNITY_EMAIL" \
-password "$UNITY_PASSWORD" \
-returnlicense \
-projectPath "$ACTIVATE_LICENSE_PATH"
# Return to previous working directory
popd

View File

@@ -68,18 +68,14 @@ elif [[ -n "$UNITY_LICENSING_SERVER" ]]; then
echo "Adding licensing server config"
/opt/unity/Editor/Data/Resources/Licensing/Client/Unity.Licensing.Client --acquire-floating > license.txt #is this accessible in a env variable?
PARSEDFILE=$(grep -oP '\".*?\"' < license.txt | tr -d '"')
export FLOATING_LICENSE
FLOATING_LICENSE=$(sed -n 2p <<< "$PARSEDFILE")
FLOATING_LICENSE_TIMEOUT=$(sed -n 4p <<< "$PARSEDFILE")
echo "Acquired floating license: \"$FLOATING_LICENSE\" with timeout $FLOATING_LICENSE_TIMEOUT"
# Store the exit code from the verify command
UNITY_EXIT_CODE=$?
if [ $UNITY_EXIT_CODE -eq 0 ]; then
PARSEDFILE=$(grep -oP '\".*?\"' < license.txt | tr -d '"')
export FLOATING_LICENSE
FLOATING_LICENSE=$(sed -n 2p <<< "$PARSEDFILE")
FLOATING_LICENSE_TIMEOUT=$(sed -n 4p <<< "$PARSEDFILE")
echo "Acquired floating license: \"$FLOATING_LICENSE\" with timeout $FLOATING_LICENSE_TIMEOUT"
fi
else
#
# NO LICENSE ACTIVATION STRATEGY MATCHED

View File

@@ -125,11 +125,10 @@ unity-editor \
$( [ "${MANUAL_EXIT}" == "true" ] || echo "-quit" ) \
-customBuildName "$BUILD_NAME" \
-projectPath "$UNITY_PROJECT_PATH" \
$( [ -z "$BUILD_PROFILE" ] && echo "-buildTarget $BUILD_TARGET" ) \
-buildTarget "$BUILD_TARGET" \
-customBuildTarget "$BUILD_TARGET" \
-customBuildPath "$CUSTOM_BUILD_PATH" \
-customBuildProfile "$BUILD_PROFILE" \
${BUILD_PROFILE:+-activeBuildProfile} ${BUILD_PROFILE:+"$BUILD_PROFILE"} \
-executeMethod "$BUILD_METHOD" \
-buildVersion "$VERSION" \
-androidVersionCode "$ANDROID_VERSION_CODE" \

View File

@@ -166,6 +166,7 @@ $unityArgs = @(
"-customBuildName", "`"$Env:BUILD_NAME`"",
"-projectPath", "`"$Env:UNITY_PROJECT_PATH`"",
"-executeMethod", "`"$Env:BUILD_METHOD`"",
"-buildTarget", "`"$Env:BUILD_TARGET`"",
"-customBuildTarget", "`"$Env:BUILD_TARGET`"",
"-customBuildPath", "`"$Env:CUSTOM_BUILD_PATH`"",
"-customBuildProfile", "`"$Env:BUILD_PROFILE`"",
@@ -180,13 +181,6 @@ $unityArgs = @(
"-logfile", "-"
) + $customParametersArray
if (-not $Env:BUILD_PROFILE) {
$unityArgs += @("-buildTarget", "`"$Env:BUILD_TARGET`"")
}
if ($Env:BUILD_PROFILE) {
$unityArgs += @("-activeBuildProfile", "`"$Env:BUILD_PROFILE`"")
}
# Remove null items as that will fail the Start-Process call
$unityArgs = $unityArgs | Where-Object { $_ -ne $null }

View File

@@ -18,9 +18,6 @@ regsvr32 C:\ProgramData\Microsoft\VisualStudio\Setup\x64\Microsoft.VisualStudio.
# Kill the regsvr process
Get-Process -Name regsvr32 | ForEach-Object { Stop-Process -Id $_.Id -Force }
# Install Visual C++ 2013 Redistributables
. "c:\steps\install_vcredist13.ps1"
# Setup Git Credentials
. "c:\steps\set_gitcredential.ps1"

View File

@@ -1,11 +0,0 @@
# For some reason, Unity is failing in github actions windows runners
# due to missing Visual C++ 2013 redistributables.
# This script downloads and installs the required redistributables.
Write-Output ""
Write-Output "#########################################################"
Write-Output "# Installing Visual C++ Redistributables (2013) #"
Write-Output "#########################################################"
Write-Output ""
choco install vcredist2013 -y --no-progress

View File

@@ -1,11 +0,0 @@
const base = require('./jest.config.js');
module.exports = {
...base,
forceExit: true,
detectOpenHandles: true,
testTimeout: 120000,
maxWorkers: 1,
};

View File

@@ -25,6 +25,6 @@ module.exports = {
// An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
modulePathIgnorePatterns: ['<rootDir>/lib/', '<rootDir>/dist/'],
// Use jest.setup.js to polyfill fetch for all tests
setupFiles: ['<rootDir>/jest.setup.js'],
// A list of paths to modules that run some code to configure or set up the testing framework before each test
setupFilesAfterEnv: ['<rootDir>/src/jest.setup.ts'],
};

View File

@@ -1,2 +0,0 @@
const fetch = require('node-fetch');
global.fetch = fetch;

View File

@@ -12,18 +12,17 @@
"lint": "prettier --check \"src/**/*.{js,ts}\" && eslint src/**/*.ts",
"format": "prettier --write \"src/**/*.{js,ts}\"",
"cli": "yarn ts-node src/index.ts -m cli",
"gcp-secrets-tests": "cross-env providerStrategy=aws orchestratorTests=true inputPullCommand=\"gcp-secret-manager\" populateOverride=true pullInputList=UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD yarn test -i -t \"orchestrator\"",
"gcp-secrets-cli": "cross-env orchestratorTests=true USE_IL2CPP=false inputPullCommand=\"gcp-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
"aws-secrets-cli": "cross-env orchestratorTests=true inputPullCommand=\"aws-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
"gcp-secrets-tests": "cross-env providerStrategy=aws cloudRunnerTests=true inputPullCommand=\"gcp-secret-manager\" populateOverride=true pullInputList=UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD yarn test -i -t \"cloud runner\"",
"gcp-secrets-cli": "cross-env cloudRunnerTests=true USE_IL2CPP=false inputPullCommand=\"gcp-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
"aws-secrets-cli": "cross-env cloudRunnerTests=true inputPullCommand=\"aws-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
"cli-aws": "cross-env providerStrategy=aws yarn run test-cli",
"cli-k8s": "cross-env providerStrategy=k8s yarn run test-cli",
"test-cli": "cross-env orchestratorTests=true yarn ts-node src/index.ts -m cli --projectPath test-project",
"test-cli": "cross-env cloudRunnerTests=true yarn ts-node src/index.ts -m cli --projectPath test-project",
"test": "jest",
"test:ci": "jest --config=jest.ci.config.js --runInBand",
"test-i": "cross-env orchestratorTests=true yarn test -i -t \"orchestrator\"",
"test-i": "cross-env cloudRunnerTests=true yarn test -i -t \"cloud runner\"",
"test-i-*": "yarn run test-i-aws && yarn run test-i-k8s",
"test-i-aws": "cross-env orchestratorTests=true providerStrategy=aws yarn test -i -t \"orchestrator\"",
"test-i-k8s": "cross-env orchestratorTests=true providerStrategy=k8s yarn test -i -t \"orchestrator\""
"test-i-aws": "cross-env cloudRunnerTests=true providerStrategy=aws yarn test -i -t \"cloud runner\"",
"test-i-k8s": "cross-env cloudRunnerTests=true providerStrategy=k8s yarn test -i -t \"cloud runner\""
},
"engines": {
"node": ">=18.x"
@@ -50,9 +49,8 @@
"nanoid": "^3.3.1",
"reflect-metadata": "^0.1.13",
"semver": "^7.5.2",
"shell-quote": "^1.8.3",
"ts-md5": "^1.3.1",
"unity-changeset": "^3.1.0",
"unity-changeset": "^2.0.0",
"uuid": "^9.0.0",
"yaml": "^2.2.2"
},
@@ -76,7 +74,6 @@
"jest-fail-on-console": "^3.0.2",
"js-yaml": "^4.1.0",
"lefthook": "^1.6.1",
"node-fetch": "2",
"prettier": "^2.5.1",
"ts-jest": "^27.1.3",
"ts-node": "10.8.1",
@@ -87,4 +84,4 @@
"node": "20.5.1",
"yarn": "1.22.19"
}
}
}

View File

@@ -1,606 +0,0 @@
/**
* Integration wiring tests for enterprise features in index.ts
*
* These tests verify the conditional gating logic in runMain():
* - Each enterprise feature is only invoked when its gate condition is met
* - Services are NOT called when their feature is disabled (the default)
* - The order of operations is correct (restore before build, save after build)
*/
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
import { BuildParameters } from './model';
// ---------------------------------------------------------------------------
// Service mocks — must be declared before importing index.ts (jest hoists them)
// ---------------------------------------------------------------------------
const mockChildWorkspaceService = {
buildConfig: jest.fn().mockReturnValue({ enabled: true, workspaceName: 'Test' }),
initializeWorkspace: jest.fn().mockReturnValue(false),
getWorkspaceSize: jest.fn().mockReturnValue('0 B'),
saveWorkspace: jest.fn(),
};
const mockSubmoduleProfileService = {
createInitPlan: jest.fn().mockResolvedValue([]),
execute: jest.fn().mockResolvedValue(''),
};
const mockLfsAgentService = {
configure: jest.fn().mockResolvedValue(''),
};
const mockLocalCacheService = {
resolveCacheRoot: jest.fn().mockReturnValue('/cache'),
generateCacheKey: jest.fn().mockReturnValue('key-1'),
restoreLfsCache: jest.fn().mockResolvedValue(true),
restoreLibraryCache: jest.fn().mockResolvedValue(true),
saveLibraryCache: jest.fn().mockResolvedValue(''),
saveLfsCache: jest.fn().mockResolvedValue(''),
};
const mockGitHooksService = {
installHooks: jest.fn().mockResolvedValue(''),
configureSkipList: jest.fn().mockReturnValue({ LEFTHOOK_EXCLUDE: 'pre-commit' }),
};
// Mock the dynamic import() targets — jest.mock with factory functions.
// The services are imported dynamically via `await import(...)` in index.ts,
// so we mock the module path and return the mock objects as named exports.
jest.mock('./model/orchestrator/services/cache/child-workspace-service', () => ({
ChildWorkspaceService: mockChildWorkspaceService,
}));
jest.mock('./model/orchestrator/services/submodule/submodule-profile-service', () => ({
SubmoduleProfileService: mockSubmoduleProfileService,
}));
jest.mock('./model/orchestrator/services/lfs/lfs-agent-service', () => ({
LfsAgentService: mockLfsAgentService,
}));
jest.mock('./model/orchestrator/services/cache/local-cache-service', () => ({
LocalCacheService: mockLocalCacheService,
}));
jest.mock('./model/orchestrator/services/hooks/git-hooks-service', () => ({
GitHooksService: mockGitHooksService,
}));
// Mock all non-enterprise dependencies to isolate the wiring logic
jest.mock('@actions/core');
jest.mock('./model', () => ({
Action: {
checkCompatibility: jest.fn(),
workspace: '/workspace',
actionFolder: '/action',
},
BuildParameters: {
create: jest.fn(),
},
Cache: {
verify: jest.fn(),
},
Orchestrator: {
run: jest.fn().mockResolvedValue(''),
},
Docker: {
run: jest.fn().mockResolvedValue(0),
},
ImageTag: jest.fn().mockImplementation(() => ({
toString: () => 'mock-image:latest',
})),
Output: {
setBuildVersion: jest.fn().mockResolvedValue(''),
setAndroidVersionCode: jest.fn().mockResolvedValue(''),
setEngineExitCode: jest.fn().mockResolvedValue(''),
},
}));
jest.mock('./model/cli/cli', () => ({
Cli: {
InitCliMode: jest.fn().mockReturnValue(false),
},
}));
jest.mock('./model/mac-builder', () => ({
__esModule: true,
default: {
run: jest.fn().mockResolvedValue(0),
},
}));
jest.mock('./model/platform-setup', () => ({
__esModule: true,
default: {
setup: jest.fn().mockResolvedValue(''),
},
}));
const mockedBuildParametersCreate = BuildParameters.create as jest.Mock;
interface EnterpriseBuildParametersOverrides {
providerStrategy?: string;
childWorkspacesEnabled?: boolean;
childWorkspaceName?: string;
childWorkspaceCacheRoot?: string;
childWorkspacePreserveGit?: boolean;
childWorkspaceSeparateLibrary?: boolean;
submoduleProfilePath?: string;
submoduleVariantPath?: string;
submoduleToken?: string;
gitPrivateToken?: string;
lfsTransferAgent?: string;
lfsTransferAgentArgs?: string;
lfsStoragePaths?: string;
localCacheEnabled?: boolean;
localCacheRoot?: string;
localCacheLibrary?: boolean;
localCacheLfs?: boolean;
gitHooksEnabled?: boolean;
gitHooksSkipList?: string;
gitHooksRunBeforeBuild?: string;
}
function createMockBuildParameters(overrides: EnterpriseBuildParametersOverrides = {}) {
return {
// Required base properties
providerStrategy: 'local',
targetPlatform: 'StandaloneLinux64',
editorVersion: '2021.3.1f1',
buildVersion: '1.0.0',
androidVersionCode: '1',
projectPath: '.',
branch: 'main',
runnerTempPath: '/tmp',
// Enterprise features - all disabled by default
childWorkspacesEnabled: false,
childWorkspaceName: '',
childWorkspaceCacheRoot: '',
childWorkspacePreserveGit: true,
childWorkspaceSeparateLibrary: true,
submoduleProfilePath: '',
submoduleVariantPath: '',
submoduleToken: '',
gitPrivateToken: '',
lfsTransferAgent: '',
lfsTransferAgentArgs: '',
lfsStoragePaths: '',
localCacheEnabled: false,
localCacheRoot: '',
localCacheLibrary: true,
localCacheLfs: false,
gitHooksEnabled: false,
gitHooksSkipList: '',
gitHooksRunBeforeBuild: '',
...overrides,
};
}
/**
* The entry point (runMain) is invoked by importing index.ts.
* Since it calls `runMain()` at module scope, we need to re-import it
* for each test. jest.isolateModules() handles this.
*/
async function runIndex(overrides: EnterpriseBuildParametersOverrides = {}): Promise<void> {
mockedBuildParametersCreate.mockResolvedValue(createMockBuildParameters(overrides));
return new Promise<void>((resolve) => {
jest.isolateModules(() => {
require('./index');
// runMain() is async; give it a tick to complete
// We use setImmediate to ensure all microtasks from the dynamic imports resolve
});
// Allow all promises and microtasks to settle
setTimeout(resolve, 100);
});
}
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
describe('index.ts enterprise feature wiring', () => {
const originalPlatform = process.platform;
const originalEnvironment = { ...process.env };
beforeEach(() => {
jest.clearAllMocks();
process.env.GITHUB_WORKSPACE = '/workspace';
// Force linux platform so Docker.run is used (not MacBuilder)
Object.defineProperty(process, 'platform', { value: 'linux' });
});
afterEach(() => {
Object.defineProperty(process, 'platform', { value: originalPlatform });
process.env = { ...originalEnvironment };
});
// -----------------------------------------------------------------------
// GitHooksService gating
// -----------------------------------------------------------------------
describe('GitHooksService gating', () => {
it('should NOT call GitHooksService when gitHooksEnabled is false (default)', async () => {
await runIndex({ gitHooksEnabled: false });
expect(mockGitHooksService.installHooks).not.toHaveBeenCalled();
expect(mockGitHooksService.configureSkipList).not.toHaveBeenCalled();
});
it('should call installHooks when gitHooksEnabled is true', async () => {
await runIndex({ gitHooksEnabled: true });
expect(mockGitHooksService.installHooks).toHaveBeenCalledWith('/workspace');
});
it('should call configureSkipList when gitHooksEnabled and gitHooksSkipList is set', async () => {
await runIndex({
gitHooksEnabled: true,
gitHooksSkipList: 'pre-commit,pre-push',
});
expect(mockGitHooksService.configureSkipList).toHaveBeenCalledWith(['pre-commit', 'pre-push']);
});
it('should NOT call configureSkipList when gitHooksSkipList is empty', async () => {
await runIndex({
gitHooksEnabled: true,
gitHooksSkipList: '',
});
expect(mockGitHooksService.installHooks).toHaveBeenCalled();
expect(mockGitHooksService.configureSkipList).not.toHaveBeenCalled();
});
});
// -----------------------------------------------------------------------
// LocalCacheService gating
// -----------------------------------------------------------------------
describe('LocalCacheService gating', () => {
it('should NOT call LocalCacheService when localCacheEnabled is false (default)', async () => {
await runIndex({ localCacheEnabled: false });
expect(mockLocalCacheService.resolveCacheRoot).not.toHaveBeenCalled();
expect(mockLocalCacheService.generateCacheKey).not.toHaveBeenCalled();
expect(mockLocalCacheService.restoreLibraryCache).not.toHaveBeenCalled();
expect(mockLocalCacheService.restoreLfsCache).not.toHaveBeenCalled();
expect(mockLocalCacheService.saveLibraryCache).not.toHaveBeenCalled();
expect(mockLocalCacheService.saveLfsCache).not.toHaveBeenCalled();
});
it('should call restore and save operations when localCacheEnabled is true', async () => {
await runIndex({
localCacheEnabled: true,
localCacheLibrary: true,
localCacheLfs: true,
});
expect(mockLocalCacheService.resolveCacheRoot).toHaveBeenCalled();
expect(mockLocalCacheService.generateCacheKey).toHaveBeenCalled();
expect(mockLocalCacheService.restoreLibraryCache).toHaveBeenCalled();
expect(mockLocalCacheService.restoreLfsCache).toHaveBeenCalled();
expect(mockLocalCacheService.saveLibraryCache).toHaveBeenCalled();
expect(mockLocalCacheService.saveLfsCache).toHaveBeenCalled();
});
it('should only cache Library when localCacheLibrary is true and localCacheLfs is false', async () => {
await runIndex({
localCacheEnabled: true,
localCacheLibrary: true,
localCacheLfs: false,
});
expect(mockLocalCacheService.restoreLibraryCache).toHaveBeenCalled();
expect(mockLocalCacheService.restoreLfsCache).not.toHaveBeenCalled();
expect(mockLocalCacheService.saveLibraryCache).toHaveBeenCalled();
expect(mockLocalCacheService.saveLfsCache).not.toHaveBeenCalled();
});
it('should only cache LFS when localCacheLfs is true and localCacheLibrary is false', async () => {
await runIndex({
localCacheEnabled: true,
localCacheLibrary: false,
localCacheLfs: true,
});
expect(mockLocalCacheService.restoreLibraryCache).not.toHaveBeenCalled();
expect(mockLocalCacheService.restoreLfsCache).toHaveBeenCalled();
expect(mockLocalCacheService.saveLibraryCache).not.toHaveBeenCalled();
expect(mockLocalCacheService.saveLfsCache).toHaveBeenCalled();
});
});
// -----------------------------------------------------------------------
// ChildWorkspaceService gating
// -----------------------------------------------------------------------
describe('ChildWorkspaceService gating', () => {
it('should NOT call ChildWorkspaceService when childWorkspacesEnabled is false (default)', async () => {
await runIndex({ childWorkspacesEnabled: false });
expect(mockChildWorkspaceService.buildConfig).not.toHaveBeenCalled();
expect(mockChildWorkspaceService.initializeWorkspace).not.toHaveBeenCalled();
expect(mockChildWorkspaceService.saveWorkspace).not.toHaveBeenCalled();
});
it('should NOT call ChildWorkspaceService when childWorkspacesEnabled is true but childWorkspaceName is empty', async () => {
await runIndex({
childWorkspacesEnabled: true,
childWorkspaceName: '',
});
expect(mockChildWorkspaceService.buildConfig).not.toHaveBeenCalled();
});
it('should call buildConfig, initializeWorkspace, and saveWorkspace when enabled with a name', async () => {
mockChildWorkspaceService.buildConfig.mockReturnValue({ enabled: true, workspaceName: 'TurnOfWar' });
await runIndex({
childWorkspacesEnabled: true,
childWorkspaceName: 'TurnOfWar',
childWorkspaceCacheRoot: '/cache/workspaces',
});
expect(mockChildWorkspaceService.buildConfig).toHaveBeenCalledWith(
expect.objectContaining({
childWorkspacesEnabled: true,
childWorkspaceName: 'TurnOfWar',
}),
);
expect(mockChildWorkspaceService.initializeWorkspace).toHaveBeenCalled();
expect(mockChildWorkspaceService.getWorkspaceSize).toHaveBeenCalled();
expect(mockChildWorkspaceService.saveWorkspace).toHaveBeenCalled();
});
});
// -----------------------------------------------------------------------
// SubmoduleProfileService gating
// -----------------------------------------------------------------------
describe('SubmoduleProfileService gating', () => {
it('should NOT call SubmoduleProfileService when submoduleProfilePath is empty (default)', async () => {
await runIndex({ submoduleProfilePath: '' });
expect(mockSubmoduleProfileService.createInitPlan).not.toHaveBeenCalled();
expect(mockSubmoduleProfileService.execute).not.toHaveBeenCalled();
});
it('should call createInitPlan and execute when submoduleProfilePath is set', async () => {
await runIndex({
submoduleProfilePath: '/path/to/profile.yml',
submoduleVariantPath: '',
submoduleToken: 'my-token',
});
expect(mockSubmoduleProfileService.createInitPlan).toHaveBeenCalledWith('/path/to/profile.yml', '', '/workspace');
expect(mockSubmoduleProfileService.execute).toHaveBeenCalled();
});
it('should pass variant path when provided', async () => {
await runIndex({
submoduleProfilePath: '/path/to/profile.yml',
submoduleVariantPath: '/path/to/variant.yml',
});
expect(mockSubmoduleProfileService.createInitPlan).toHaveBeenCalledWith(
'/path/to/profile.yml',
'/path/to/variant.yml',
'/workspace',
);
});
it('should use submoduleToken for auth, falling back to gitPrivateToken', async () => {
await runIndex({
submoduleProfilePath: '/path/to/profile.yml',
submoduleToken: '',
gitPrivateToken: 'fallback-token',
});
expect(mockSubmoduleProfileService.execute).toHaveBeenCalledWith(
expect.anything(),
'/workspace',
'fallback-token',
);
});
it('should prefer submoduleToken over gitPrivateToken', async () => {
await runIndex({
submoduleProfilePath: '/path/to/profile.yml',
submoduleToken: 'specific-token',
gitPrivateToken: 'fallback-token',
});
expect(mockSubmoduleProfileService.execute).toHaveBeenCalledWith(
expect.anything(),
'/workspace',
'specific-token',
);
});
});
// -----------------------------------------------------------------------
// LfsAgentService gating
// -----------------------------------------------------------------------
describe('LfsAgentService gating', () => {
it('should NOT call LfsAgentService when lfsTransferAgent is empty (default)', async () => {
await runIndex({ lfsTransferAgent: '' });
expect(mockLfsAgentService.configure).not.toHaveBeenCalled();
});
it('should call configure when lfsTransferAgent is set', async () => {
await runIndex({
lfsTransferAgent: '/tools/elastic-git-storage',
lfsTransferAgentArgs: '--verbose',
lfsStoragePaths: '/path/a;/path/b',
});
expect(mockLfsAgentService.configure).toHaveBeenCalledWith(
'/tools/elastic-git-storage',
'--verbose',
['/path/a', '/path/b'],
'/workspace',
);
});
it('should pass empty array when lfsStoragePaths is empty', async () => {
await runIndex({
lfsTransferAgent: '/tools/agent',
lfsStoragePaths: '',
});
expect(mockLfsAgentService.configure).toHaveBeenCalledWith('/tools/agent', '', [], '/workspace');
});
});
// -----------------------------------------------------------------------
// Order of operations (restore before build, save after build)
// -----------------------------------------------------------------------
describe('order of operations', () => {
it('should execute restore operations before build and save operations after build', async () => {
const callOrder: string[] = [];
// Track call order for each relevant operation
mockChildWorkspaceService.buildConfig.mockReturnValue({ enabled: true, workspaceName: 'Test' });
mockChildWorkspaceService.initializeWorkspace.mockImplementation(() => {
callOrder.push('child-workspace-restore');
return false;
});
mockChildWorkspaceService.getWorkspaceSize.mockImplementation(() => {
callOrder.push('child-workspace-size');
return '0 B';
});
mockSubmoduleProfileService.createInitPlan.mockImplementation(async () => {
callOrder.push('submodule-profile-plan');
return [];
});
mockSubmoduleProfileService.execute.mockImplementation(async () => {
callOrder.push('submodule-profile-execute');
});
mockLfsAgentService.configure.mockImplementation(async () => {
callOrder.push('lfs-agent-configure');
});
mockLocalCacheService.resolveCacheRoot.mockImplementation(() => {
callOrder.push('local-cache-resolve');
return '/cache';
});
mockLocalCacheService.generateCacheKey.mockImplementation(() => {
callOrder.push('local-cache-keygen');
return 'key-1';
});
mockLocalCacheService.restoreLfsCache.mockImplementation(async () => {
callOrder.push('local-cache-restore-lfs');
return true;
});
mockLocalCacheService.restoreLibraryCache.mockImplementation(async () => {
callOrder.push('local-cache-restore-library');
return true;
});
mockGitHooksService.installHooks.mockImplementation(async () => {
callOrder.push('git-hooks-install');
});
mockLocalCacheService.saveLibraryCache.mockImplementation(async () => {
callOrder.push('local-cache-save-library');
});
mockLocalCacheService.saveLfsCache.mockImplementation(async () => {
callOrder.push('local-cache-save-lfs');
});
mockChildWorkspaceService.saveWorkspace.mockImplementation(() => {
callOrder.push('child-workspace-save');
});
await runIndex({
childWorkspacesEnabled: true,
childWorkspaceName: 'TurnOfWar',
submoduleProfilePath: '/profile.yml',
lfsTransferAgent: '/tools/agent',
localCacheEnabled: true,
localCacheLfs: true,
localCacheLibrary: true,
gitHooksEnabled: true,
});
// Verify restore operations happen before save operations.
// The expected order from index.ts is:
// 1. Child workspace restore
// 2. Submodule profile init
// 3. LFS agent configure
// 4. Local cache restore (LFS then Library)
// 5. Git hooks install
// 6. [BUILD happens here - Docker.run or MacBuilder.run]
// 7. Local cache save (Library then LFS)
// 8. Child workspace save
const restoreOps = [
'child-workspace-restore',
'submodule-profile-plan',
'submodule-profile-execute',
'lfs-agent-configure',
'local-cache-restore-lfs',
'local-cache-restore-library',
'git-hooks-install',
];
const saveOps = ['local-cache-save-library', 'local-cache-save-lfs', 'child-workspace-save'];
// All restore ops should appear before all save ops
for (const restoreOp of restoreOps) {
if (!callOrder.includes(restoreOp)) continue; // Skip if the operation wasn't called
for (const saveOp of saveOps) {
if (!callOrder.includes(saveOp)) continue;
expect(callOrder.indexOf(restoreOp)).toBeLessThan(callOrder.indexOf(saveOp));
}
}
// Child workspace save should be last
if (callOrder.includes('child-workspace-save') && callOrder.includes('local-cache-save-lfs')) {
expect(callOrder.indexOf('local-cache-save-lfs')).toBeLessThan(callOrder.indexOf('child-workspace-save'));
}
});
});
// -----------------------------------------------------------------------
// Non-local provider strategy
// -----------------------------------------------------------------------
describe('non-local provider strategy', () => {
it('should skip all enterprise features when providerStrategy is not local', async () => {
await runIndex({
providerStrategy: 'aws',
childWorkspacesEnabled: true,
childWorkspaceName: 'Test',
submoduleProfilePath: '/profile.yml',
lfsTransferAgent: '/tools/agent',
localCacheEnabled: true,
gitHooksEnabled: true,
});
// None of the enterprise services should be called because
// they are inside the `if (providerStrategy === 'local')` block
expect(mockChildWorkspaceService.buildConfig).not.toHaveBeenCalled();
expect(mockSubmoduleProfileService.createInitPlan).not.toHaveBeenCalled();
expect(mockLfsAgentService.configure).not.toHaveBeenCalled();
expect(mockLocalCacheService.resolveCacheRoot).not.toHaveBeenCalled();
expect(mockGitHooksService.installHooks).not.toHaveBeenCalled();
});
});
});

View File

@@ -1,6 +1,5 @@
import * as core from '@actions/core';
import path from 'node:path';
import { Action, BuildParameters, Cache, Orchestrator, Docker, ImageTag, Output } from './model';
import { Action, BuildParameters, Cache, CloudRunner, Docker, ImageTag, Output } from './model';
import { Cli } from './model/cli/cli';
import MacBuilder from './model/mac-builder';
import PlatformSetup from './model/platform-setup';
@@ -24,94 +23,6 @@ async function runMain() {
if (buildParameters.providerStrategy === 'local') {
core.info('Building locally');
// Child workspace isolation - restore cached workspace before any other setup
let childWorkspaceConfig: any;
if (buildParameters.childWorkspacesEnabled && buildParameters.childWorkspaceName) {
const { ChildWorkspaceService } = await import('./model/orchestrator/services/cache/child-workspace-service');
const cacheRoot =
buildParameters.childWorkspaceCacheRoot ||
path.join(buildParameters.runnerTempPath || process.env.RUNNER_TEMP || '', 'game-ci-workspaces');
childWorkspaceConfig = ChildWorkspaceService.buildConfig({
childWorkspacesEnabled: buildParameters.childWorkspacesEnabled,
childWorkspaceName: buildParameters.childWorkspaceName,
childWorkspaceCacheRoot: cacheRoot,
childWorkspacePreserveGit: buildParameters.childWorkspacePreserveGit,
childWorkspaceSeparateLibrary: buildParameters.childWorkspaceSeparateLibrary,
});
const projectFullPath = path.join(workspace, buildParameters.projectPath);
const restored = ChildWorkspaceService.initializeWorkspace(projectFullPath, childWorkspaceConfig);
core.info(
`Child workspace "${buildParameters.childWorkspaceName}": ${
restored ? 'restored from cache' : 'starting fresh'
}`,
);
// Log workspace size for resource tracking
const size = ChildWorkspaceService.getWorkspaceSize(projectFullPath);
core.info(`Child workspace size after restore: ${size}`);
}
// Submodule profile initialization
if (buildParameters.submoduleProfilePath) {
const { SubmoduleProfileService } = await import(
'./model/orchestrator/services/submodule/submodule-profile-service'
);
core.info('Initializing submodules from profile...');
const plan = await SubmoduleProfileService.createInitPlan(
buildParameters.submoduleProfilePath,
buildParameters.submoduleVariantPath,
workspace,
);
await SubmoduleProfileService.execute(
plan,
workspace,
buildParameters.submoduleToken || buildParameters.gitPrivateToken,
);
}
// Configure custom LFS transfer agent
if (buildParameters.lfsTransferAgent) {
const { LfsAgentService } = await import('./model/orchestrator/services/lfs/lfs-agent-service');
core.info('Configuring custom LFS transfer agent...');
await LfsAgentService.configure(
buildParameters.lfsTransferAgent,
buildParameters.lfsTransferAgentArgs,
buildParameters.lfsStoragePaths ? buildParameters.lfsStoragePaths.split(';') : [],
workspace,
);
}
// Local build caching - restore
let cacheRoot = '';
let cacheKey = '';
if (buildParameters.localCacheEnabled) {
const { LocalCacheService } = await import('./model/orchestrator/services/cache/local-cache-service');
cacheRoot = LocalCacheService.resolveCacheRoot(buildParameters);
cacheKey = LocalCacheService.generateCacheKey(
buildParameters.targetPlatform,
buildParameters.editorVersion,
buildParameters.branch || '',
);
if (buildParameters.localCacheLfs) {
await LocalCacheService.restoreLfsCache(workspace, cacheRoot, cacheKey);
}
if (buildParameters.localCacheLibrary) {
const projectFullPath = path.join(workspace, buildParameters.projectPath);
await LocalCacheService.restoreLibraryCache(projectFullPath, cacheRoot, cacheKey);
}
}
// Git hooks — opt-in only. When disabled (default), do not touch hooks at all.
if (buildParameters.gitHooksEnabled) {
const { GitHooksService } = await import('./model/orchestrator/services/hooks/git-hooks-service');
await GitHooksService.installHooks(workspace);
if (buildParameters.gitHooksSkipList) {
const environment = GitHooksService.configureSkipList(buildParameters.gitHooksSkipList.split(','));
Object.assign(process.env, environment);
}
}
await PlatformSetup.setup(buildParameters, actionFolder);
exitCode =
process.platform === 'darwin'
@@ -121,31 +32,8 @@ async function runMain() {
actionFolder,
...buildParameters,
});
// Local build caching - save
if (buildParameters.localCacheEnabled) {
const { LocalCacheService } = await import('./model/orchestrator/services/cache/local-cache-service');
if (buildParameters.localCacheLibrary) {
const projectFullPath = path.join(workspace, buildParameters.projectPath);
await LocalCacheService.saveLibraryCache(projectFullPath, cacheRoot, cacheKey);
}
if (buildParameters.localCacheLfs) {
await LocalCacheService.saveLfsCache(workspace, cacheRoot, cacheKey);
}
}
// Child workspace isolation - save workspace for next run
if (childWorkspaceConfig && childWorkspaceConfig.enabled) {
const { ChildWorkspaceService } = await import('./model/orchestrator/services/cache/child-workspace-service');
const projectFullPath = path.join(workspace, buildParameters.projectPath);
const preSaveSize = ChildWorkspaceService.getWorkspaceSize(projectFullPath);
core.info(`Child workspace size before save: ${preSaveSize}`);
ChildWorkspaceService.saveWorkspace(projectFullPath, childWorkspaceConfig);
core.info(`Child workspace "${buildParameters.childWorkspaceName}" saved to cache`);
}
} else {
await Orchestrator.run(buildParameters, baseImage.toString());
await CloudRunner.run(buildParameters, baseImage.toString());
exitCode = 0;
}

View File

@@ -1,13 +1,22 @@
// Integration test for exercising real GitHub check creation and updates.
import Orchestrator from '../model/orchestrator/orchestrator';
import { BuildParameters } from '../model';
import CloudRunner from '../model/cloud-runner/cloud-runner';
import UnityVersioning from '../model/unity-versioning';
import { Cli } from '../model/cli/cli';
import GitHub from '../model/github';
import { TIMEOUT_INFINITE, createParameters } from '../test-utils/orchestrator-test-helpers';
import { OptionValues } from 'commander';
export const TIMEOUT_INFINITE = 1e9;
async function createParameters(overrides?: OptionValues) {
if (overrides) Cli.options = overrides;
return BuildParameters.create();
}
const runIntegration = process.env.RUN_GITHUB_INTEGRATION_TESTS === 'true';
const describeOrSkip = runIntegration ? describe : describe.skip;
describeOrSkip('Orchestrator Github Checks Integration', () => {
describeOrSkip('Cloud Runner Github Checks Integration', () => {
it(
'creates and updates a real GitHub check',
async () => {
@@ -15,10 +24,10 @@ describeOrSkip('Orchestrator Github Checks Integration', () => {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.read('test-project'),
asyncOrchestrator: `true`,
asyncCloudRunner: `true`,
githubChecks: `true`,
});
await Orchestrator.setup(buildParameter);
await CloudRunner.setup(buildParameter);
const checkId = await GitHub.createGitHubCheck(`integration create`);
expect(checkId).not.toEqual('');
await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `integration`);

View File

@@ -1,3 +0,0 @@
import { fetch as undiciFetch, Headers, Request, Response } from 'undici';
Object.assign(globalThis, { fetch: undiciFetch, Headers, Request, Response });

View File

@@ -1,5 +1,15 @@
import failOnConsole from 'jest-fail-on-console';
// Polyfill fetch for the Node.js test environment.
// Jest runs tests inside a VM context where the global `fetch` is not
// automatically provided, even on Node 18+. Octokit requires a `fetch`
// implementation, so we provide one using undici's implementation.
// This ensures tests that interact with Octokit do not throw when
// constructing the client.
import { fetch as undiciFetch, Headers, Request, Response } from 'undici';
Object.assign(globalThis, { fetch: undiciFetch, Headers, Request, Response });
// Fail when console logs something inside a test - use spyOn instead
failOnConsole({
shouldFailOnWarn: true,

View File

@@ -1,7 +1,7 @@
import { customAlphabet } from 'nanoid';
import AndroidVersioning from './android-versioning';
import OrchestratorConstants from './orchestrator/options/orchestrator-constants';
import OrchestratorBuildGuid from './orchestrator/options/orchestrator-guid';
import CloudRunnerConstants from './cloud-runner/options/cloud-runner-constants';
import CloudRunnerBuildGuid from './cloud-runner/options/cloud-runner-guid';
import Input from './input';
import Platform from './platform';
import UnityVersioning from './unity-versioning';
@@ -10,8 +10,8 @@ import { GitRepoReader } from './input-readers/git-repo';
import { GithubCliReader } from './input-readers/github-cli';
import { Cli } from './cli/cli';
import GitHub from './github';
import OrchestratorOptions from './orchestrator/options/orchestrator-options';
import Orchestrator from './orchestrator/orchestrator';
import CloudRunnerOptions from './cloud-runner/options/cloud-runner-options';
import CloudRunner from './cloud-runner/cloud-runner';
import * as core from '@actions/core';
class BuildParameters {
@@ -56,18 +56,9 @@ class BuildParameters {
public providerStrategy!: string;
public gitPrivateToken!: string;
public awsStackName!: string;
public awsEndpoint?: string;
public awsCloudFormationEndpoint?: string;
public awsEcsEndpoint?: string;
public awsKinesisEndpoint?: string;
public awsCloudWatchLogsEndpoint?: string;
public awsS3Endpoint?: string;
public storageProvider!: string;
public rcloneRemote!: string;
public kubeConfig!: string;
public containerMemory!: string;
public containerCpu!: string;
public containerNamespace!: string;
public kubeVolumeSize!: string;
public kubeVolume!: string;
public kubeStorageClass!: string;
@@ -84,13 +75,11 @@ class BuildParameters {
public runNumber!: string;
public branch!: string;
public githubRepo!: string;
public orchestratorRepoName!: string;
public cloneDepth!: number;
public gitSha!: string;
public logId!: string;
public buildGuid!: string;
public orchestratorBranch!: string;
public orchestratorDebug!: boolean | undefined;
public cloudRunnerBranch!: string;
public cloudRunnerDebug!: boolean | undefined;
public buildPlatform!: string | undefined;
public isCliMode!: boolean;
public maxRetainedWorkspaces!: number;
@@ -106,30 +95,9 @@ class BuildParameters {
public cacheUnityInstallationOnMac!: boolean;
public unityHubVersionOnMac!: string;
public dockerWorkspacePath!: string;
public submoduleProfilePath!: string;
public submoduleVariantPath!: string;
public submoduleToken!: string;
public localCacheEnabled!: boolean;
public localCacheRoot!: string;
public localCacheLibrary!: boolean;
public localCacheLfs!: boolean;
public childWorkspacesEnabled!: boolean;
public childWorkspaceName!: string;
public childWorkspaceCacheRoot!: string;
public childWorkspacePreserveGit!: boolean;
public childWorkspaceSeparateLibrary!: boolean;
public lfsTransferAgent!: string;
public lfsTransferAgentArgs!: string;
public lfsStoragePaths!: string;
public gitHooksEnabled!: boolean;
public gitHooksSkipList!: string;
public gitHooksRunBeforeBuild!: string;
public providerExecutable!: string;
public middleware!: string;
public middlewareFiles!: string[];
public static shouldUseRetainedWorkspaceMode(buildParameters: BuildParameters) {
return buildParameters.maxRetainedWorkspaces > 0 && Orchestrator.lockedWorkspace !== ``;
return buildParameters.maxRetainedWorkspaces > 0 && CloudRunner.lockedWorkspace !== ``;
}
static async create(): Promise<BuildParameters> {
@@ -214,76 +182,44 @@ class BuildParameters {
dockerIsolationMode: Input.dockerIsolationMode,
containerRegistryRepository: Input.containerRegistryRepository,
containerRegistryImageVersion: Input.containerRegistryImageVersion,
providerStrategy: OrchestratorOptions.providerStrategy,
buildPlatform: OrchestratorOptions.buildPlatform,
kubeConfig: OrchestratorOptions.kubeConfig,
containerMemory: OrchestratorOptions.containerMemory,
containerCpu: OrchestratorOptions.containerCpu,
containerNamespace: OrchestratorOptions.containerNamespace,
kubeVolumeSize: OrchestratorOptions.kubeVolumeSize,
kubeVolume: OrchestratorOptions.kubeVolume,
postBuildContainerHooks: OrchestratorOptions.postBuildContainerHooks,
preBuildContainerHooks: OrchestratorOptions.preBuildContainerHooks,
customJob: OrchestratorOptions.customJob,
providerStrategy: CloudRunnerOptions.providerStrategy,
buildPlatform: CloudRunnerOptions.buildPlatform,
kubeConfig: CloudRunnerOptions.kubeConfig,
containerMemory: CloudRunnerOptions.containerMemory,
containerCpu: CloudRunnerOptions.containerCpu,
kubeVolumeSize: CloudRunnerOptions.kubeVolumeSize,
kubeVolume: CloudRunnerOptions.kubeVolume,
postBuildContainerHooks: CloudRunnerOptions.postBuildContainerHooks,
preBuildContainerHooks: CloudRunnerOptions.preBuildContainerHooks,
customJob: CloudRunnerOptions.customJob,
runNumber: Input.runNumber,
branch: Input.branch.replace('/head', '') || (await GitRepoReader.GetBranch()),
orchestratorBranch: OrchestratorOptions.orchestratorBranch.split('/').reverse()[0],
orchestratorDebug: OrchestratorOptions.orchestratorDebug,
githubRepo: (Input.githubRepo ?? (await GitRepoReader.GetRemote())) || OrchestratorOptions.orchestratorRepoName,
orchestratorRepoName: OrchestratorOptions.orchestratorRepoName,
cloneDepth: Number.parseInt(OrchestratorOptions.cloneDepth),
cloudRunnerBranch: CloudRunnerOptions.cloudRunnerBranch.split('/').reverse()[0],
cloudRunnerDebug: CloudRunnerOptions.cloudRunnerDebug,
githubRepo: (Input.githubRepo ?? (await GitRepoReader.GetRemote())) || 'game-ci/unity-builder',
isCliMode: Cli.isCliMode,
awsStackName: OrchestratorOptions.awsStackName,
awsEndpoint: OrchestratorOptions.awsEndpoint,
awsCloudFormationEndpoint: OrchestratorOptions.awsCloudFormationEndpoint,
awsEcsEndpoint: OrchestratorOptions.awsEcsEndpoint,
awsKinesisEndpoint: OrchestratorOptions.awsKinesisEndpoint,
awsCloudWatchLogsEndpoint: OrchestratorOptions.awsCloudWatchLogsEndpoint,
awsS3Endpoint: OrchestratorOptions.awsS3Endpoint,
storageProvider: OrchestratorOptions.storageProvider,
rcloneRemote: OrchestratorOptions.rcloneRemote,
awsStackName: CloudRunnerOptions.awsStackName,
gitSha: Input.gitSha,
logId: customAlphabet(OrchestratorConstants.alphabet, 9)(),
buildGuid: OrchestratorBuildGuid.generateGuid(Input.runNumber, Input.targetPlatform),
commandHooks: OrchestratorOptions.commandHooks,
inputPullCommand: OrchestratorOptions.inputPullCommand,
pullInputList: OrchestratorOptions.pullInputList,
kubeStorageClass: OrchestratorOptions.kubeStorageClass,
cacheKey: OrchestratorOptions.cacheKey,
maxRetainedWorkspaces: Number.parseInt(OrchestratorOptions.maxRetainedWorkspaces),
useLargePackages: OrchestratorOptions.useLargePackages,
useCompressionStrategy: OrchestratorOptions.useCompressionStrategy,
garbageMaxAge: OrchestratorOptions.garbageMaxAge,
githubChecks: OrchestratorOptions.githubChecks,
asyncWorkflow: OrchestratorOptions.asyncOrchestrator,
githubCheckId: OrchestratorOptions.githubCheckId,
finalHooks: OrchestratorOptions.finalHooks,
skipLfs: OrchestratorOptions.skipLfs,
skipCache: OrchestratorOptions.skipCache,
logId: customAlphabet(CloudRunnerConstants.alphabet, 9)(),
buildGuid: CloudRunnerBuildGuid.generateGuid(Input.runNumber, Input.targetPlatform),
commandHooks: CloudRunnerOptions.commandHooks,
inputPullCommand: CloudRunnerOptions.inputPullCommand,
pullInputList: CloudRunnerOptions.pullInputList,
kubeStorageClass: CloudRunnerOptions.kubeStorageClass,
cacheKey: CloudRunnerOptions.cacheKey,
maxRetainedWorkspaces: Number.parseInt(CloudRunnerOptions.maxRetainedWorkspaces),
useLargePackages: CloudRunnerOptions.useLargePackages,
useCompressionStrategy: CloudRunnerOptions.useCompressionStrategy,
garbageMaxAge: CloudRunnerOptions.garbageMaxAge,
githubChecks: CloudRunnerOptions.githubChecks,
asyncWorkflow: CloudRunnerOptions.asyncCloudRunner,
githubCheckId: CloudRunnerOptions.githubCheckId,
finalHooks: CloudRunnerOptions.finalHooks,
skipLfs: CloudRunnerOptions.skipLfs,
skipCache: CloudRunnerOptions.skipCache,
cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac,
unityHubVersionOnMac: Input.unityHubVersionOnMac,
dockerWorkspacePath: Input.dockerWorkspacePath,
submoduleProfilePath: Input.submoduleProfilePath,
submoduleVariantPath: Input.submoduleVariantPath,
submoduleToken: Input.submoduleToken,
localCacheEnabled: Input.localCacheEnabled,
localCacheRoot: Input.localCacheRoot,
localCacheLibrary: Input.localCacheLibrary,
localCacheLfs: Input.localCacheLfs,
childWorkspacesEnabled: Input.childWorkspacesEnabled,
childWorkspaceName: Input.childWorkspaceName,
childWorkspaceCacheRoot: Input.childWorkspaceCacheRoot,
childWorkspacePreserveGit: Input.childWorkspacePreserveGit,
childWorkspaceSeparateLibrary: Input.childWorkspaceSeparateLibrary,
lfsTransferAgent: Input.lfsTransferAgent,
lfsTransferAgentArgs: Input.lfsTransferAgentArgs,
lfsStoragePaths: Input.lfsStoragePaths,
gitHooksEnabled: Input.gitHooksEnabled,
gitHooksSkipList: Input.gitHooksSkipList,
gitHooksRunBeforeBuild: Input.gitHooksRunBeforeBuild,
providerExecutable: Input.providerExecutable,
middleware: OrchestratorOptions.middleware,
middlewareFiles: OrchestratorOptions.middlewareFiles,
};
}

View File

@@ -1,19 +1,17 @@
import { Command } from 'commander-ts';
import { BuildParameters, Orchestrator, ImageTag, Input } from '..';
import { BuildParameters, CloudRunner, ImageTag, Input } from '..';
import * as core from '@actions/core';
import { ActionYamlReader } from '../input-readers/action-yaml';
import OrchestratorLogger from '../orchestrator/services/core/orchestrator-logger';
import OrchestratorQueryOverride from '../orchestrator/options/orchestrator-query-override';
import CloudRunnerLogger from '../cloud-runner/services/core/cloud-runner-logger';
import CloudRunnerQueryOverride from '../cloud-runner/options/cloud-runner-query-override';
import { CliFunction, CliFunctionsRepository } from './cli-functions-repository';
import { Caching } from '../orchestrator/remote-client/caching';
import { LfsHashing } from '../orchestrator/services/utility/lfs-hashing';
import { RemoteClient } from '../orchestrator/remote-client';
import OrchestratorOptionsReader from '../orchestrator/options/orchestrator-options-reader';
import { Caching } from '../cloud-runner/remote-client/caching';
import { LfsHashing } from '../cloud-runner/services/utility/lfs-hashing';
import { RemoteClient } from '../cloud-runner/remote-client';
import CloudRunnerOptionsReader from '../cloud-runner/options/cloud-runner-options-reader';
import GitHub from '../github';
import { OptionValues } from 'commander';
import { InputKey } from '../input';
import { SubmoduleProfileService } from '../orchestrator/services/submodule/submodule-profile-service';
import { LfsAgentService } from '../orchestrator/services/lfs/lfs-agent-service';
export class Cli {
public static options: OptionValues | undefined;
@@ -38,7 +36,7 @@ export class Cli {
const program = new Command();
program.version('0.0.1');
const properties = OrchestratorOptionsReader.GetProperties();
const properties = CloudRunnerOptionsReader.GetProperties();
const actionYamlReader: ActionYamlReader = new ActionYamlReader();
for (const element of properties) {
program.option(`--${element} <${element}>`, actionYamlReader.GetActionYamlValue(element));
@@ -55,11 +53,6 @@ export class Cli {
program.option('--artifactName <artifactName>', 'caching artifact name');
program.option('--select <select>', 'select a particular resource');
program.option('--logFile <logFile>', 'output to log file (log stream only)');
program.option('--profilePath <profilePath>', 'path to submodule profile YAML');
program.option('--variantPath <variantPath>', 'path to submodule variant YAML');
program.option('--agentPath <agentPath>', 'path to custom LFS transfer agent');
program.option('--agentArgs <agentArgs>', 'arguments for custom LFS transfer agent');
program.option('--storagePaths <storagePaths>', 'semicolon-separated storage paths for LFS agent');
program.parse(process.argv);
Cli.options = program.opts();
@@ -69,23 +62,23 @@ export class Cli {
static async RunCli(): Promise<void> {
GitHub.githubInputEnabled = false;
if (Cli.options!['populateOverride'] === `true`) {
await OrchestratorQueryOverride.PopulateQueryOverrideInput();
await CloudRunnerQueryOverride.PopulateQueryOverrideInput();
}
if (Cli.options!['logInput']) {
Cli.logInput();
}
const results = CliFunctionsRepository.GetCliFunctions(Cli.options?.mode);
OrchestratorLogger.log(`Entrypoint: ${results.key}`);
CloudRunnerLogger.log(`Entrypoint: ${results.key}`);
Cli.options!.versioning = 'None';
Orchestrator.buildParameters = await BuildParameters.create();
Orchestrator.buildParameters.buildGuid = process.env.BUILD_GUID || ``;
OrchestratorLogger.log(`Build Params:
${JSON.stringify(Orchestrator.buildParameters, undefined, 4)}
CloudRunner.buildParameters = await BuildParameters.create();
CloudRunner.buildParameters.buildGuid = process.env.BUILD_GUID || ``;
CloudRunnerLogger.log(`Build Params:
${JSON.stringify(CloudRunner.buildParameters, undefined, 4)}
`);
Orchestrator.lockedWorkspace = process.env.LOCKED_WORKSPACE || ``;
OrchestratorLogger.log(`Locked Workspace: ${Orchestrator.lockedWorkspace}`);
await Orchestrator.setup(Orchestrator.buildParameters);
CloudRunner.lockedWorkspace = process.env.LOCKED_WORKSPACE || ``;
CloudRunnerLogger.log(`Locked Workspace: ${CloudRunner.lockedWorkspace}`);
await CloudRunner.setup(CloudRunner.buildParameters);
return await results.target[results.propertyKey](Cli.options);
}
@@ -94,7 +87,7 @@ export class Cli {
private static logInput() {
core.info(`\n`);
core.info(`INPUT:`);
const properties = OrchestratorOptionsReader.GetProperties();
const properties = CloudRunnerOptionsReader.GetProperties();
for (const element of properties) {
if (
element in Input &&
@@ -111,28 +104,28 @@ export class Cli {
core.info(`\n`);
}
@CliFunction(`cli-build`, `runs a orchestrator build`)
@CliFunction(`cli-build`, `runs a cloud runner build`)
public static async CLIBuild(): Promise<string> {
const buildParameter = await BuildParameters.create();
const baseImage = new ImageTag(buildParameter);
return (await Orchestrator.run(buildParameter, baseImage.toString())).BuildResults;
return (await CloudRunner.run(buildParameter, baseImage.toString())).BuildResults;
}
@CliFunction(`async-workflow`, `runs a orchestrator build`)
@CliFunction(`async-workflow`, `runs a cloud runner build`)
public static async asyncronousWorkflow(): Promise<string> {
const buildParameter = await BuildParameters.create();
const baseImage = new ImageTag(buildParameter);
await Orchestrator.setup(buildParameter);
await CloudRunner.setup(buildParameter);
return (await Orchestrator.run(buildParameter, baseImage.toString())).BuildResults;
return (await CloudRunner.run(buildParameter, baseImage.toString())).BuildResults;
}
@CliFunction(`checks-update`, `runs a orchestrator build`)
@CliFunction(`checks-update`, `runs a cloud runner build`)
public static async checksUpdate() {
const buildParameter = await BuildParameters.create();
await Orchestrator.setup(buildParameter);
await CloudRunner.setup(buildParameter);
const input = JSON.parse(process.env.CHECKS_UPDATE || ``);
core.info(`Checks Update ${process.env.CHECKS_UPDATE}`);
if (input.mode === `create`) {
@@ -146,18 +139,18 @@ export class Cli {
public static async GarbageCollect(): Promise<string> {
const buildParameter = await BuildParameters.create();
await Orchestrator.setup(buildParameter);
await CloudRunner.setup(buildParameter);
return await Orchestrator.Provider.garbageCollect(``, false, 0, false, false);
return await CloudRunner.Provider.garbageCollect(``, false, 0, false, false);
}
@CliFunction(`list-resources`, `lists active resources`)
public static async ListResources(): Promise<string[]> {
const buildParameter = await BuildParameters.create();
await Orchestrator.setup(buildParameter);
const result = await Orchestrator.Provider.listResources();
OrchestratorLogger.log(JSON.stringify(result, undefined, 4));
await CloudRunner.setup(buildParameter);
const result = await CloudRunner.Provider.listResources();
CloudRunnerLogger.log(JSON.stringify(result, undefined, 4));
return result.map((x) => x.Name);
}
@@ -166,39 +159,17 @@ export class Cli {
public static async ListWorfklow(): Promise<string[]> {
const buildParameter = await BuildParameters.create();
await Orchestrator.setup(buildParameter);
await CloudRunner.setup(buildParameter);
return (await Orchestrator.Provider.listWorkflow()).map((x) => x.Name);
return (await CloudRunner.Provider.listWorkflow()).map((x) => x.Name);
}
@CliFunction(`watch`, `follows logs of a running workflow`)
public static async Watch(): Promise<string> {
const buildParameter = await BuildParameters.create();
await Orchestrator.setup(buildParameter);
await CloudRunner.setup(buildParameter);
return await Orchestrator.Provider.watchWorkflow();
}
@CliFunction(`submodule-init`, `initializes submodules from a YAML profile`)
public static async SubmoduleInit(): Promise<void> {
const profilePath = Cli.options!['profilePath'];
const variantPath = Cli.options!['variantPath'] || '';
if (!profilePath) {
throw new Error('--profilePath is required for submodule-init');
}
const plan = await SubmoduleProfileService.createInitPlan(profilePath, variantPath, process.cwd());
await SubmoduleProfileService.execute(plan, process.cwd());
}
@CliFunction(`lfs-agent-configure`, `configures a custom LFS transfer agent`)
public static async LfsAgentConfigure(): Promise<void> {
const agentPath = Cli.options!['agentPath'];
if (!agentPath) {
throw new Error('--agentPath is required for lfs-agent-configure');
}
const agentArgs = Cli.options!['agentArgs'] || '';
const storagePaths = (Cli.options!['storagePaths'] || '').split(';').filter(Boolean);
await LfsAgentService.configure(agentPath, agentArgs, storagePaths, process.cwd());
return await CloudRunner.Provider.watchWorkflow();
}
}

View File

@@ -0,0 +1,187 @@
import AwsBuildPlatform from './providers/aws';
import { BuildParameters, Input } from '..';
import Kubernetes from './providers/k8s';
import CloudRunnerLogger from './services/core/cloud-runner-logger';
import { CloudRunnerStepParameters } from './options/cloud-runner-step-parameters';
import { WorkflowCompositionRoot } from './workflows/workflow-composition-root';
import { CloudRunnerError } from './error/cloud-runner-error';
import { TaskParameterSerializer } from './services/core/task-parameter-serializer';
import * as core from '@actions/core';
import CloudRunnerSecret from './options/cloud-runner-secret';
import { ProviderInterface } from './providers/provider-interface';
import CloudRunnerEnvironmentVariable from './options/cloud-runner-environment-variable';
import TestCloudRunner from './providers/test';
import LocalCloudRunner from './providers/local';
import LocalDockerCloudRunner from './providers/docker';
import GitHub from '../github';
import SharedWorkspaceLocking from './services/core/shared-workspace-locking';
import { FollowLogStreamService } from './services/core/follow-log-stream-service';
import CloudRunnerResult from './services/core/cloud-runner-result';
class CloudRunner {
public static Provider: ProviderInterface;
public static buildParameters: BuildParameters;
private static defaultSecrets: CloudRunnerSecret[];
private static cloudRunnerEnvironmentVariables: CloudRunnerEnvironmentVariable[];
static lockedWorkspace: string = ``;
public static readonly retainedWorkspacePrefix: string = `retained-workspace`;
public static get isCloudRunnerEnvironment() {
return process.env[`GITHUB_ACTIONS`] !== `true`;
}
public static get isCloudRunnerAsyncEnvironment() {
return process.env[`ASYNC_WORKFLOW`] === `true`;
}
public static async setup(buildParameters: BuildParameters) {
CloudRunnerLogger.setup();
CloudRunnerLogger.log(`Setting up cloud runner`);
CloudRunner.buildParameters = buildParameters;
if (CloudRunner.buildParameters.githubCheckId === ``) {
CloudRunner.buildParameters.githubCheckId = await GitHub.createGitHubCheck(CloudRunner.buildParameters.buildGuid);
}
CloudRunner.setupSelectedBuildPlatform();
CloudRunner.defaultSecrets = TaskParameterSerializer.readDefaultSecrets();
CloudRunner.cloudRunnerEnvironmentVariables =
TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameters);
if (GitHub.githubInputEnabled) {
const buildParameterPropertyNames = Object.getOwnPropertyNames(buildParameters);
for (const element of CloudRunner.cloudRunnerEnvironmentVariables) {
// CloudRunnerLogger.log(`Cloud Runner output ${Input.ToEnvVarFormat(element.name)} = ${element.value}`);
core.setOutput(Input.ToEnvVarFormat(element.name), element.value);
}
for (const element of buildParameterPropertyNames) {
// CloudRunnerLogger.log(`Cloud Runner output ${Input.ToEnvVarFormat(element)} = ${buildParameters[element]}`);
core.setOutput(Input.ToEnvVarFormat(element), buildParameters[element]);
}
core.setOutput(
Input.ToEnvVarFormat(`buildArtifact`),
`build-${CloudRunner.buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}`,
);
}
FollowLogStreamService.Reset();
}
private static setupSelectedBuildPlatform() {
CloudRunnerLogger.log(`Cloud Runner platform selected ${CloudRunner.buildParameters.providerStrategy}`);
switch (CloudRunner.buildParameters.providerStrategy) {
case 'k8s':
CloudRunner.Provider = new Kubernetes(CloudRunner.buildParameters);
break;
case 'aws':
CloudRunner.Provider = new AwsBuildPlatform(CloudRunner.buildParameters);
break;
case 'test':
CloudRunner.Provider = new TestCloudRunner();
break;
case 'local-docker':
CloudRunner.Provider = new LocalDockerCloudRunner();
break;
case 'local-system':
CloudRunner.Provider = new LocalCloudRunner();
break;
}
}
static async run(buildParameters: BuildParameters, baseImage: string) {
if (baseImage.includes(`undefined`)) {
throw new Error(`baseImage is undefined`);
}
await CloudRunner.setup(buildParameters);
await CloudRunner.Provider.setupWorkflow(
CloudRunner.buildParameters.buildGuid,
CloudRunner.buildParameters,
CloudRunner.buildParameters.branch,
CloudRunner.defaultSecrets,
);
try {
if (buildParameters.maxRetainedWorkspaces > 0) {
CloudRunner.lockedWorkspace = SharedWorkspaceLocking.NewWorkspaceName();
const result = await SharedWorkspaceLocking.GetLockedWorkspace(
CloudRunner.lockedWorkspace,
CloudRunner.buildParameters.buildGuid,
CloudRunner.buildParameters,
);
if (result) {
CloudRunnerLogger.logLine(`Using retained workspace ${CloudRunner.lockedWorkspace}`);
CloudRunner.cloudRunnerEnvironmentVariables = [
...CloudRunner.cloudRunnerEnvironmentVariables,
{ name: `LOCKED_WORKSPACE`, value: CloudRunner.lockedWorkspace },
];
} else {
CloudRunnerLogger.log(`Max retained workspaces reached ${buildParameters.maxRetainedWorkspaces}`);
buildParameters.maxRetainedWorkspaces = 0;
CloudRunner.lockedWorkspace = ``;
}
}
await CloudRunner.updateStatusWithBuildParameters();
const output = await new WorkflowCompositionRoot().run(
new CloudRunnerStepParameters(
baseImage,
CloudRunner.cloudRunnerEnvironmentVariables,
CloudRunner.defaultSecrets,
),
);
await CloudRunner.Provider.cleanupWorkflow(
CloudRunner.buildParameters,
CloudRunner.buildParameters.branch,
CloudRunner.defaultSecrets,
);
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
if (buildParameters.asyncWorkflow && this.isCloudRunnerEnvironment && this.isCloudRunnerAsyncEnvironment) {
await GitHub.updateGitHubCheck(CloudRunner.buildParameters.buildGuid, `success`, `success`, `completed`);
}
if (BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
const workspace = CloudRunner.lockedWorkspace || ``;
await SharedWorkspaceLocking.ReleaseWorkspace(
workspace,
CloudRunner.buildParameters.buildGuid,
CloudRunner.buildParameters,
);
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(workspace, CloudRunner.buildParameters);
if (isLocked) {
throw new Error(
`still locked after releasing ${await SharedWorkspaceLocking.GetAllLocksForWorkspace(
workspace,
buildParameters,
)}`,
);
}
CloudRunner.lockedWorkspace = ``;
}
await GitHub.triggerWorkflowOnComplete(CloudRunner.buildParameters.finalHooks);
if (buildParameters.constantGarbageCollection) {
CloudRunner.Provider.garbageCollect(``, true, buildParameters.garbageMaxAge, true, true);
}
return new CloudRunnerResult(buildParameters, output, true, true, false);
} catch (error: any) {
CloudRunnerLogger.log(JSON.stringify(error, undefined, 4));
await GitHub.updateGitHubCheck(
CloudRunner.buildParameters.buildGuid,
`Failed - Error ${error?.message || error}`,
`failure`,
`completed`,
);
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
await CloudRunnerError.handleException(error, CloudRunner.buildParameters, CloudRunner.defaultSecrets);
throw error;
}
}
private static async updateStatusWithBuildParameters() {
const content = { ...CloudRunner.buildParameters };
content.gitPrivateToken = ``;
content.unitySerial = ``;
content.unityEmail = ``;
content.unityPassword = ``;
const jsonContent = JSON.stringify(content, undefined, 4);
await GitHub.updateGitHubCheck(jsonContent, CloudRunner.buildParameters.buildGuid);
}
}
export default CloudRunner;

View File

@@ -0,0 +1,15 @@
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import * as core from '@actions/core';
import CloudRunner from '../cloud-runner';
import CloudRunnerSecret from '../options/cloud-runner-secret';
import BuildParameters from '../../build-parameters';
export class CloudRunnerError {
public static async handleException(error: unknown, buildParameters: BuildParameters, secrets: CloudRunnerSecret[]) {
CloudRunnerLogger.error(JSON.stringify(error, undefined, 4));
core.setFailed('Cloud Runner failed');
if (CloudRunner.Provider !== undefined) {
await CloudRunner.Provider.cleanupWorkflow(buildParameters, buildParameters.branch, secrets);
}
}
}

View File

@@ -0,0 +1,4 @@
class CloudRunnerConstants {
static alphabet = '0123456789abcdefghijklmnopqrstuvwxyz';
}
export default CloudRunnerConstants;

View File

@@ -0,0 +1,5 @@
class CloudRunnerEnvironmentVariable {
public name!: string;
public value!: string;
}
export default CloudRunnerEnvironmentVariable;

View File

@@ -0,0 +1,90 @@
import path from 'node:path';
import CloudRunnerOptions from './cloud-runner-options';
import CloudRunner from '../cloud-runner';
import BuildParameters from '../../build-parameters';
export class CloudRunnerFolders {
public static readonly repositoryFolder = 'repo';
public static ToLinuxFolder(folder: string) {
return folder.replace(/\\/g, `/`);
}
// Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
public static get uniqueCloudRunnerJobFolderAbsolute(): string {
return CloudRunner.buildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)
? path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.lockedWorkspace)
: path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.buildParameters.buildGuid);
}
public static get cacheFolderForAllFull(): string {
return path.join('/', CloudRunnerFolders.buildVolumeFolder, CloudRunnerFolders.cacheFolder);
}
public static get cacheFolderForCacheKeyFull(): string {
return path.join(
'/',
CloudRunnerFolders.buildVolumeFolder,
CloudRunnerFolders.cacheFolder,
CloudRunner.buildParameters.cacheKey,
);
}
public static get builderPathAbsolute(): string {
return path.join(
CloudRunnerOptions.useSharedBuilder
? `/${CloudRunnerFolders.buildVolumeFolder}`
: CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute,
`builder`,
);
}
public static get repoPathAbsolute(): string {
return path.join(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute, CloudRunnerFolders.repositoryFolder);
}
public static get projectPathAbsolute(): string {
return path.join(CloudRunnerFolders.repoPathAbsolute, CloudRunner.buildParameters.projectPath);
}
public static get libraryFolderAbsolute(): string {
return path.join(CloudRunnerFolders.projectPathAbsolute, `Library`);
}
public static get projectBuildFolderAbsolute(): string {
return path.join(CloudRunnerFolders.repoPathAbsolute, CloudRunner.buildParameters.buildPath);
}
public static get lfsFolderAbsolute(): string {
return path.join(CloudRunnerFolders.repoPathAbsolute, `.git`, `lfs`);
}
public static get purgeRemoteCaching(): boolean {
return process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined;
}
public static get lfsCacheFolderFull() {
return path.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `lfs`);
}
public static get libraryCacheFolderFull() {
return path.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `Library`);
}
public static get unityBuilderRepoUrl(): string {
return `https://${CloudRunner.buildParameters.gitPrivateToken}@github.com/game-ci/unity-builder.git`;
}
public static get targetBuildRepoUrl(): string {
return `https://${CloudRunner.buildParameters.gitPrivateToken}@github.com/${CloudRunner.buildParameters.githubRepo}.git`;
}
public static get buildVolumeFolder() {
return 'data';
}
public static get cacheFolder() {
return 'cache';
}
}

View File

@@ -1,11 +1,11 @@
import { customAlphabet } from 'nanoid';
import OrchestratorConstants from './orchestrator-constants';
import CloudRunnerConstants from './cloud-runner-constants';
class OrchestratorNamespace {
class CloudRunnerNamespace {
static generateGuid(runNumber: string | number, platform: string) {
const nanoid = customAlphabet(OrchestratorConstants.alphabet, 4);
const nanoid = customAlphabet(CloudRunnerConstants.alphabet, 4);
return `${runNumber}-${platform.toLowerCase().replace('standalone', '')}-${nanoid()}`;
}
}
export default OrchestratorNamespace;
export default CloudRunnerNamespace;

View File

@@ -0,0 +1,10 @@
import Input from '../../input';
import CloudRunnerOptions from './cloud-runner-options';
class CloudRunnerOptionsReader {
static GetProperties() {
return [...Object.getOwnPropertyNames(Input), ...Object.getOwnPropertyNames(CloudRunnerOptions)];
}
}
export default CloudRunnerOptionsReader;

View File

@@ -0,0 +1,283 @@
import { Cli } from '../../cli/cli';
import CloudRunnerQueryOverride from './cloud-runner-query-override';
import GitHub from '../../github';
import * as core from '@actions/core';
class CloudRunnerOptions {
// ### ### ###
// Input Handling
// ### ### ###
public static getInput(query: string): string | undefined {
if (GitHub.githubInputEnabled) {
const coreInput = core.getInput(query);
if (coreInput && coreInput !== '') {
return coreInput;
}
}
const alternativeQuery = CloudRunnerOptions.ToEnvVarFormat(query);
// Query input sources
if (Cli.query(query, alternativeQuery)) {
return Cli.query(query, alternativeQuery);
}
if (CloudRunnerQueryOverride.query(query, alternativeQuery)) {
return CloudRunnerQueryOverride.query(query, alternativeQuery);
}
if (process.env[query] !== undefined) {
return process.env[query];
}
if (alternativeQuery !== query && process.env[alternativeQuery] !== undefined) {
return process.env[alternativeQuery];
}
}
public static ToEnvVarFormat(input: string): string {
if (input.toUpperCase() === input) {
return input;
}
return input
.replace(/([A-Z])/g, ' $1')
.trim()
.toUpperCase()
.replace(/ /g, '_');
}
// ### ### ###
// Provider parameters
// ### ### ###
static get region(): string {
return CloudRunnerOptions.getInput('region') || 'eu-west-2';
}
// ### ### ###
// GitHub parameters
// ### ### ###
static get githubChecks(): boolean {
const value = CloudRunnerOptions.getInput('githubChecks');
return value === `true` || false;
}
static get githubCheckId(): string {
return CloudRunnerOptions.getInput('githubCheckId') || ``;
}
static get githubOwner(): string {
return CloudRunnerOptions.getInput('githubOwner') || CloudRunnerOptions.githubRepo?.split(`/`)[0] || '';
}
static get githubRepoName(): string {
return CloudRunnerOptions.getInput('githubRepoName') || CloudRunnerOptions.githubRepo?.split(`/`)[1] || '';
}
static get finalHooks(): string[] {
return CloudRunnerOptions.getInput('finalHooks')?.split(',') || [];
}
// ### ### ###
// Git syncronization parameters
// ### ### ###
static get githubRepo(): string | undefined {
return CloudRunnerOptions.getInput('GITHUB_REPOSITORY') || CloudRunnerOptions.getInput('GITHUB_REPO') || undefined;
}
static get branch(): string {
if (CloudRunnerOptions.getInput(`GITHUB_REF`)) {
return (
CloudRunnerOptions.getInput(`GITHUB_REF`)?.replace('refs/', '').replace(`head/`, '').replace(`heads/`, '') || ``
);
} else if (CloudRunnerOptions.getInput('branch')) {
return CloudRunnerOptions.getInput('branch') || ``;
} else {
return '';
}
}
// ### ### ###
// Cloud Runner parameters
// ### ### ###
static get buildPlatform(): string {
const input = CloudRunnerOptions.getInput('buildPlatform');
if (input && input !== '') {
return input;
}
if (CloudRunnerOptions.providerStrategy !== 'local') {
return 'linux';
}
return process.platform;
}
static get cloudRunnerBranch(): string {
return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'main';
}
static get providerStrategy(): string {
const provider =
CloudRunnerOptions.getInput('cloudRunnerCluster') || CloudRunnerOptions.getInput('providerStrategy');
if (Cli.isCliMode) {
return provider || 'aws';
}
return provider || 'local';
}
static get containerCpu(): string {
return CloudRunnerOptions.getInput('containerCpu') || `1024`;
}
static get containerMemory(): string {
return CloudRunnerOptions.getInput('containerMemory') || `3072`;
}
static get customJob(): string {
return CloudRunnerOptions.getInput('customJob') || '';
}
// ### ### ###
// Custom commands from files parameters
// ### ### ###
static get containerHookFiles(): string[] {
return CloudRunnerOptions.getInput('containerHookFiles')?.split(`,`) || [];
}
static get commandHookFiles(): string[] {
return CloudRunnerOptions.getInput('commandHookFiles')?.split(`,`) || [];
}
// ### ### ###
// Custom commands from yaml parameters
// ### ### ###
static get commandHooks(): string {
return CloudRunnerOptions.getInput('commandHooks') || '';
}
static get postBuildContainerHooks(): string {
return CloudRunnerOptions.getInput('postBuildContainerHooks') || '';
}
static get preBuildContainerHooks(): string {
return CloudRunnerOptions.getInput('preBuildContainerHooks') || '';
}
// ### ### ###
// Input override handling
// ### ### ###
static get pullInputList(): string[] {
return CloudRunnerOptions.getInput('pullInputList')?.split(`,`) || [];
}
static get inputPullCommand(): string {
const value = CloudRunnerOptions.getInput('inputPullCommand');
if (value === 'gcp-secret-manager') {
return 'gcloud secrets versions access 1 --secret="{0}"';
} else if (value === 'aws-secret-manager') {
return 'aws secretsmanager get-secret-value --secret-id {0}';
}
return value || '';
}
// ### ### ###
// Aws
// ### ### ###
static get awsStackName() {
return CloudRunnerOptions.getInput('awsStackName') || 'game-ci';
}
// ### ### ###
// K8s
// ### ### ###
static get kubeConfig(): string {
return CloudRunnerOptions.getInput('kubeConfig') || '';
}
static get kubeVolume(): string {
return CloudRunnerOptions.getInput('kubeVolume') || '';
}
static get kubeVolumeSize(): string {
return CloudRunnerOptions.getInput('kubeVolumeSize') || '25Gi';
}
static get kubeStorageClass(): string {
return CloudRunnerOptions.getInput('kubeStorageClass') || '';
}
// ### ### ###
// Caching
// ### ### ###
static get cacheKey(): string {
return CloudRunnerOptions.getInput('cacheKey') || CloudRunnerOptions.branch;
}
// ### ### ###
// Utility Parameters
// ### ### ###
static get cloudRunnerDebug(): boolean {
return (
CloudRunnerOptions.getInput(`cloudRunnerTests`) === `true` ||
CloudRunnerOptions.getInput(`cloudRunnerDebug`) === `true` ||
CloudRunnerOptions.getInput(`cloudRunnerDebugTree`) === `true` ||
CloudRunnerOptions.getInput(`cloudRunnerDebugEnv`) === `true` ||
false
);
}
static get skipLfs(): boolean {
return CloudRunnerOptions.getInput(`skipLfs`) === `true`;
}
static get skipCache(): boolean {
return CloudRunnerOptions.getInput(`skipCache`) === `true`;
}
public static get asyncCloudRunner(): boolean {
return CloudRunnerOptions.getInput('asyncCloudRunner') === 'true';
}
public static get useLargePackages(): boolean {
return CloudRunnerOptions.getInput(`useLargePackages`) === `true`;
}
public static get useSharedBuilder(): boolean {
return CloudRunnerOptions.getInput(`useSharedBuilder`) === `true`;
}
public static get useCompressionStrategy(): boolean {
return CloudRunnerOptions.getInput(`useCompressionStrategy`) === `true`;
}
public static get useCleanupCron(): boolean {
return (CloudRunnerOptions.getInput(`useCleanupCron`) || 'true') === 'true';
}
// ### ### ###
// Retained Workspace
// ### ### ###
public static get maxRetainedWorkspaces(): string {
return CloudRunnerOptions.getInput(`maxRetainedWorkspaces`) || `0`;
}
// ### ### ###
// Garbage Collection
// ### ### ###
static get garbageMaxAge(): number {
return Number(CloudRunnerOptions.getInput(`garbageMaxAge`)) || 24;
}
}
export default CloudRunnerOptions;

View File

@@ -0,0 +1,67 @@
import Input from '../../input';
import { GenericInputReader } from '../../input-readers/generic-input-reader';
import CloudRunnerOptions from './cloud-runner-options';
const formatFunction = (value: string, arguments_: any[]) => {
for (const element of arguments_) {
value = value.replace(`{${element.key}}`, element.value);
}
return value;
};
class CloudRunnerQueryOverride {
static queryOverrides: { [key: string]: string } | undefined;
// TODO accept premade secret sources or custom secret source definition yamls
public static query(key: string, alternativeKey: string) {
if (CloudRunnerQueryOverride.queryOverrides && CloudRunnerQueryOverride.queryOverrides[key] !== undefined) {
return CloudRunnerQueryOverride.queryOverrides[key];
}
if (
CloudRunnerQueryOverride.queryOverrides &&
alternativeKey &&
CloudRunnerQueryOverride.queryOverrides[alternativeKey] !== undefined
) {
return CloudRunnerQueryOverride.queryOverrides[alternativeKey];
}
return;
}
private static shouldUseOverride(query: string) {
if (CloudRunnerOptions.inputPullCommand !== '') {
if (CloudRunnerOptions.pullInputList.length > 0) {
const doesInclude =
CloudRunnerOptions.pullInputList.includes(query) ||
CloudRunnerOptions.pullInputList.includes(Input.ToEnvVarFormat(query));
return doesInclude ? true : false;
} else {
return true;
}
}
}
private static async queryOverride(query: string) {
if (!this.shouldUseOverride(query)) {
throw new Error(`Should not be trying to run override query on ${query}`);
}
return await GenericInputReader.Run(
formatFunction(CloudRunnerOptions.inputPullCommand, [{ key: 0, value: query }]),
);
}
public static async PopulateQueryOverrideInput() {
const queries = CloudRunnerOptions.pullInputList;
CloudRunnerQueryOverride.queryOverrides = {};
for (const element of queries) {
if (CloudRunnerQueryOverride.shouldUseOverride(element)) {
CloudRunnerQueryOverride.queryOverrides[element] = await CloudRunnerQueryOverride.queryOverride(element);
}
}
}
}
export default CloudRunnerQueryOverride;

View File

@@ -1,6 +1,6 @@
class OrchestratorSecret {
class CloudRunnerSecret {
public ParameterKey!: string;
public EnvironmentVariable!: string;
public ParameterValue!: string;
}
export default OrchestratorSecret;
export default CloudRunnerSecret;

View File

@@ -0,0 +1,3 @@
export class CloudRunnerStatics {
public static readonly logPrefix = `Cloud-Runner`;
}

View File

@@ -0,0 +1,13 @@
import CloudRunnerEnvironmentVariable from './cloud-runner-environment-variable';
import CloudRunnerSecret from './cloud-runner-secret';
export class CloudRunnerStepParameters {
public image: string;
public environment: CloudRunnerEnvironmentVariable[];
public secrets: CloudRunnerSecret[];
constructor(image: string, environmentVariables: CloudRunnerEnvironmentVariable[], secrets: CloudRunnerSecret[]) {
this.image = image;
this.environment = environmentVariables;
this.secrets = secrets;
}
}

View File

@@ -1,18 +1,14 @@
import OrchestratorLogger from '../../services/core/orchestrator-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import * as core from '@actions/core';
import {
CloudFormation,
CreateStackCommand,
// eslint-disable-next-line import/named
CreateStackCommandInput,
DescribeStacksCommand,
// eslint-disable-next-line import/named
DescribeStacksCommandInput,
ListStacksCommand,
// eslint-disable-next-line import/named
Parameter,
UpdateStackCommand,
// eslint-disable-next-line import/named
UpdateStackCommandInput,
waitUntilStackCreateComplete,
waitUntilStackUpdateComplete,
@@ -20,17 +16,6 @@ import {
import { BaseStackFormation } from './cloud-formations/base-stack-formation';
import crypto from 'node:crypto';
const DEFAULT_STACK_WAIT_TIME_SECONDS = 600;
function getStackWaitTime(): number {
const overrideValue = Number(process.env.ORCHESTRATOR_AWS_STACK_WAIT_TIME ?? '');
if (!Number.isNaN(overrideValue) && overrideValue > 0) {
return overrideValue;
}
return DEFAULT_STACK_WAIT_TIME_SECONDS;
}
export class AWSBaseStack {
constructor(baseStackName: string) {
this.baseStackName = baseStackName;
@@ -39,7 +24,6 @@ export class AWSBaseStack {
async setupBaseStack(CF: CloudFormation) {
const baseStackName = this.baseStackName;
const stackWaitTimeSeconds = getStackWaitTime();
const baseStack = BaseStackFormation.formation;
@@ -70,39 +54,18 @@ export class AWSBaseStack {
};
const stacks = await CF.send(
new ListStacksCommand({
StackStatusFilter: [
'CREATE_IN_PROGRESS',
'UPDATE_IN_PROGRESS',
'UPDATE_COMPLETE',
'CREATE_COMPLETE',
'ROLLBACK_COMPLETE',
],
}),
new ListStacksCommand({ StackStatusFilter: ['UPDATE_COMPLETE', 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE'] }),
);
const stackNames = stacks.StackSummaries?.map((x) => x.StackName) || [];
const stackExists: boolean = stackNames.includes(baseStackName);
const stackExists: Boolean = stackNames.includes(baseStackName) || false;
const describeStack = async () => {
return await CF.send(new DescribeStacksCommand(describeStackInput));
};
try {
if (!stackExists) {
OrchestratorLogger.log(`${baseStackName} stack does not exist (${JSON.stringify(stackNames)})`);
let created = false;
try {
await CF.send(new CreateStackCommand(createStackInput));
created = true;
} catch (error: any) {
const message = `${error?.name ?? ''} ${error?.message ?? ''}`;
if (message.includes('AlreadyExistsException')) {
OrchestratorLogger.log(`Base stack already exists, continuing with describe`);
} else {
throw error;
}
}
if (created) {
OrchestratorLogger.log(`created stack (version: ${parametersHash})`);
}
CloudRunnerLogger.log(`${baseStackName} stack does not exist (${JSON.stringify(stackNames)})`);
await CF.send(new CreateStackCommand(createStackInput));
CloudRunnerLogger.log(`created stack (version: ${parametersHash})`);
}
const CFState = await describeStack();
let stack = CFState.Stacks?.[0];
@@ -112,35 +75,32 @@ export class AWSBaseStack {
const stackVersion = stack.Parameters?.find((x) => x.ParameterKey === 'Version')?.ParameterValue;
if (stack.StackStatus === 'CREATE_IN_PROGRESS') {
OrchestratorLogger.log(
`Waiting up to ${stackWaitTimeSeconds}s for '${baseStackName}' CloudFormation creation to finish`,
);
await waitUntilStackCreateComplete(
{
client: CF,
maxWaitTime: stackWaitTimeSeconds,
maxWaitTime: 200,
},
describeStackInput,
);
}
if (stackExists) {
OrchestratorLogger.log(`Base stack exists (version: ${stackVersion}, local version: ${parametersHash})`);
CloudRunnerLogger.log(`Base stack exists (version: ${stackVersion}, local version: ${parametersHash})`);
if (parametersHash !== stackVersion) {
OrchestratorLogger.log(`Attempting update of base stack`);
CloudRunnerLogger.log(`Attempting update of base stack`);
try {
await CF.send(new UpdateStackCommand(updateInput));
} catch (error: any) {
if (error['message'].includes('No updates are to be performed')) {
OrchestratorLogger.log(`No updates are to be performed`);
CloudRunnerLogger.log(`No updates are to be performed`);
} else {
OrchestratorLogger.log(`Update Failed (Stack name: ${baseStackName})`);
OrchestratorLogger.log(error['message']);
CloudRunnerLogger.log(`Update Failed (Stack name: ${baseStackName})`);
CloudRunnerLogger.log(error['message']);
}
OrchestratorLogger.log(`Continuing...`);
CloudRunnerLogger.log(`Continuing...`);
}
} else {
OrchestratorLogger.log(`No update required`);
CloudRunnerLogger.log(`No update required`);
}
stack = (await describeStack()).Stacks?.[0];
if (!stack) {
@@ -149,19 +109,16 @@ export class AWSBaseStack {
);
}
if (stack.StackStatus === 'UPDATE_IN_PROGRESS') {
OrchestratorLogger.log(
`Waiting up to ${stackWaitTimeSeconds}s for '${baseStackName}' CloudFormation update to finish`,
);
await waitUntilStackUpdateComplete(
{
client: CF,
maxWaitTime: stackWaitTimeSeconds,
maxWaitTime: 200,
},
describeStackInput,
);
}
}
OrchestratorLogger.log('base stack is now ready');
CloudRunnerLogger.log('base stack is now ready');
} catch (error) {
core.error(JSON.stringify(await describeStack(), undefined, 4));
throw error;

View File

@@ -21,7 +21,6 @@ export class AWSCloudFormationTemplates {
public static getSecretDefinitionTemplate(p1: string, p2: string) {
return `
Secrets:
- Name: '${p1}'
ValueFrom: !Ref ${p2}Secret
`;

View File

@@ -1,16 +1,16 @@
import OrchestratorLogger from '../../services/core/orchestrator-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { CloudFormation, DescribeStackEventsCommand } from '@aws-sdk/client-cloudformation';
import * as core from '@actions/core';
import Orchestrator from '../../orchestrator';
import CloudRunner from '../../cloud-runner';
export class AWSError {
static async handleStackCreationFailure(error: any, CF: CloudFormation, taskDefStackName: string) {
OrchestratorLogger.log('aws error: ');
CloudRunnerLogger.log('aws error: ');
core.error(JSON.stringify(error, undefined, 4));
if (Orchestrator.buildParameters.orchestratorDebug) {
OrchestratorLogger.log('Getting events and resources for task stack');
if (CloudRunner.buildParameters.cloudRunnerDebug) {
CloudRunnerLogger.log('Getting events and resources for task stack');
const events = (await CF.send(new DescribeStackEventsCommand({ StackName: taskDefStackName }))).StackEvents;
OrchestratorLogger.log(JSON.stringify(events, undefined, 4));
CloudRunnerLogger.log(JSON.stringify(events, undefined, 4));
}
}
}

View File

@@ -1,34 +1,22 @@
import {
CloudFormation,
CreateStackCommand,
// eslint-disable-next-line import/named
CreateStackCommandInput,
DescribeStackResourcesCommand,
DescribeStacksCommand,
ListStacksCommand,
waitUntilStackCreateComplete,
} from '@aws-sdk/client-cloudformation';
import OrchestratorAWSTaskDef from './orchestrator-aws-task-def';
import OrchestratorSecret from '../../options/orchestrator-secret';
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import { AWSCloudFormationTemplates } from './aws-cloud-formation-templates';
import OrchestratorLogger from '../../services/core/orchestrator-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { AWSError } from './aws-error';
import Orchestrator from '../../orchestrator';
import CloudRunner from '../../cloud-runner';
import { CleanupCronFormation } from './cloud-formations/cleanup-cron-formation';
import OrchestratorOptions from '../../options/orchestrator-options';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation';
const DEFAULT_STACK_WAIT_TIME_SECONDS = 600;
function getStackWaitTime(): number {
const overrideValue = Number(process.env.ORCHESTRATOR_AWS_STACK_WAIT_TIME ?? '');
if (!Number.isNaN(overrideValue) && overrideValue > 0) {
return overrideValue;
}
return DEFAULT_STACK_WAIT_TIME_SECONDS;
}
export class AWSJobStack {
private baseStackName: string;
constructor(baseStackName: string) {
@@ -43,23 +31,23 @@ export class AWSJobStack {
commands: string,
mountdir: string,
workingdir: string,
secrets: OrchestratorSecret[],
): Promise<OrchestratorAWSTaskDef> {
secrets: CloudRunnerSecret[],
): Promise<CloudRunnerAWSTaskDef> {
const taskDefStackName = `${this.baseStackName}-${buildGuid}`;
let taskDefCloudFormation = AWSCloudFormationTemplates.readTaskCloudFormationTemplate();
taskDefCloudFormation = taskDefCloudFormation.replace(
`ContainerCpu:
Default: 1024`,
`ContainerCpu:
Default: ${Number.parseInt(Orchestrator.buildParameters.containerCpu)}`,
Default: ${Number.parseInt(CloudRunner.buildParameters.containerCpu)}`,
);
taskDefCloudFormation = taskDefCloudFormation.replace(
`ContainerMemory:
Default: 2048`,
`ContainerMemory:
Default: ${Number.parseInt(Orchestrator.buildParameters.containerMemory)}`,
Default: ${Number.parseInt(CloudRunner.buildParameters.containerMemory)}`,
);
if (!OrchestratorOptions.asyncOrchestrator) {
if (!CloudRunnerOptions.asyncCloudRunner) {
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
taskDefCloudFormation,
'# template resources logstream',
@@ -133,8 +121,8 @@ export class AWSJobStack {
},
...secretsMappedToCloudFormationParameters,
];
OrchestratorLogger.log(
`Starting AWS job with memory: ${Orchestrator.buildParameters.containerMemory} cpu: ${Orchestrator.buildParameters.containerCpu}`,
CloudRunnerLogger.log(
`Starting AWS job with memory: ${CloudRunner.buildParameters.containerMemory} cpu: ${CloudRunner.buildParameters.containerCpu}`,
);
let previousStackExists = true;
while (previousStackExists) {
@@ -147,7 +135,7 @@ export class AWSJobStack {
const element = stacks.StackSummaries[index];
if (element.StackName === taskDefStackName && element.StackStatus !== 'DELETE_COMPLETE') {
previousStackExists = true;
OrchestratorLogger.log(`Previous stack still exists: ${JSON.stringify(element)}`);
CloudRunnerLogger.log(`Previous stack still exists: ${JSON.stringify(element)}`);
await new Promise((promise) => setTimeout(promise, 5000));
}
}
@@ -159,15 +147,12 @@ export class AWSJobStack {
Parameters: parameters,
};
try {
const stackWaitTimeSeconds = getStackWaitTime();
OrchestratorLogger.log(
`Creating job aws formation ${taskDefStackName} (waiting up to ${stackWaitTimeSeconds}s for completion)`,
);
CloudRunnerLogger.log(`Creating job aws formation ${taskDefStackName}`);
await CF.send(new CreateStackCommand(createStackInput));
await waitUntilStackCreateComplete(
{
client: CF,
maxWaitTime: stackWaitTimeSeconds,
maxWaitTime: 200,
},
{ StackName: taskDefStackName },
);
@@ -201,7 +186,7 @@ export class AWSJobStack {
},
{
ParameterKey: 'BUILDGUID',
ParameterValue: Orchestrator.buildParameters.buildGuid,
ParameterValue: CloudRunner.buildParameters.buildGuid,
},
{
ParameterKey: 'EnvironmentName',
@@ -209,9 +194,9 @@ export class AWSJobStack {
},
],
};
if (OrchestratorOptions.useCleanupCron) {
if (CloudRunnerOptions.useCleanupCron) {
try {
OrchestratorLogger.log(`Creating job cleanup formation`);
CloudRunnerLogger.log(`Creating job cleanup formation`);
await CF.send(new CreateStackCommand(createCleanupStackInput));
// await CF.waitFor('stackCreateComplete', { StackName: createCleanupStackInput.StackName }).promise();

View File

@@ -1,61 +1,38 @@
import { DescribeTasksCommand, RunTaskCommand, waitUntilTasksRunning } from '@aws-sdk/client-ecs';
import { DescribeStreamCommand, GetRecordsCommand, GetShardIteratorCommand } from '@aws-sdk/client-kinesis';
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
import {
DescribeTasksCommand,
ECS,
RunTaskCommand,
RunTaskCommandInput,
Task,
waitUntilTasksRunning,
} from '@aws-sdk/client-ecs';
import {
DescribeStreamCommand,
DescribeStreamCommandOutput,
GetRecordsCommand,
GetRecordsCommandOutput,
GetShardIteratorCommand,
Kinesis,
} from '@aws-sdk/client-kinesis';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import * as core from '@actions/core';
import OrchestratorAWSTaskDef from './orchestrator-aws-task-def';
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
import * as zlib from 'node:zlib';
import OrchestratorLogger from '../../services/core/orchestrator-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { Input } from '../../..';
import Orchestrator from '../../orchestrator';
import CloudRunner from '../../cloud-runner';
import { CommandHookService } from '../../services/hooks/command-hook-service';
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
import OrchestratorOptions from '../../options/orchestrator-options';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import GitHub from '../../../github';
import { AwsClientFactory } from './aws-client-factory';
class AWSTaskRunner {
public static ECS: ECS;
public static Kinesis: Kinesis;
private static readonly encodedUnderscore = `$252F`;
/**
* Transform localhost endpoints to host.docker.internal for container environments.
* When LocalStack is used, ECS tasks run in Docker containers that need to reach
* LocalStack on the host machine via host.docker.internal.
*/
private static transformEndpointsForContainer(
environment: OrchestratorEnvironmentVariable[],
): OrchestratorEnvironmentVariable[] {
const endpointEnvironmentNames = new Set([
'AWS_S3_ENDPOINT',
'AWS_ENDPOINT',
'AWS_CLOUD_FORMATION_ENDPOINT',
'AWS_ECS_ENDPOINT',
'AWS_KINESIS_ENDPOINT',
'AWS_CLOUD_WATCH_LOGS_ENDPOINT',
'INPUT_AWSS3ENDPOINT',
'INPUT_AWSENDPOINT',
]);
return environment.map((x) => {
let value = x.value;
if (
typeof value === 'string' &&
endpointEnvironmentNames.has(x.name) &&
(value.startsWith('http://localhost') || value.startsWith('http://127.0.0.1'))
) {
// Replace localhost with host.docker.internal so ECS containers can access host services
value = value
.replace('http://localhost', 'http://host.docker.internal')
.replace('http://127.0.0.1', 'http://host.docker.internal');
OrchestratorLogger.log(`AWS TaskRunner: Replaced localhost with host.docker.internal for ${x.name}: ${value}`);
}
return { name: x.name, value };
});
}
static async runTask(
taskDef: OrchestratorAWSTaskDef,
environment: OrchestratorEnvironmentVariable[],
taskDef: CloudRunnerAWSTaskDef,
environment: CloudRunnerEnvironmentVariable[],
commands: string,
): Promise<{ output: string; shouldCleanup: boolean }> {
const cluster = taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ECSCluster')?.PhysicalResourceId || '';
@@ -70,9 +47,6 @@ class AWSTaskRunner {
const streamName =
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'KinesisStream')?.PhysicalResourceId || '';
// Transform localhost endpoints for container environment
const transformedEnvironment = AWSTaskRunner.transformEndpointsForContainer(environment);
const runParameters = {
cluster,
taskDefinition,
@@ -81,8 +55,8 @@ class AWSTaskRunner {
containerOverrides: [
{
name: taskDef.taskDefStackName,
environment: transformedEnvironment,
command: ['-c', CommandHookService.ApplyHooksToCommands(commands, Orchestrator.buildParameters)],
environment,
command: ['-c', CommandHookService.ApplyHooksToCommands(commands, CloudRunner.buildParameters)],
},
],
},
@@ -97,49 +71,45 @@ class AWSTaskRunner {
};
if (JSON.stringify(runParameters.overrides.containerOverrides).length > 8192) {
OrchestratorLogger.log(JSON.stringify(runParameters.overrides.containerOverrides, undefined, 4));
CloudRunnerLogger.log(JSON.stringify(runParameters.overrides.containerOverrides, undefined, 4));
throw new Error(`Container Overrides length must be at most 8192`);
}
const task = await AwsClientFactory.getECS().send(new RunTaskCommand(runParameters as any));
const task = await AWSTaskRunner.ECS.send(new RunTaskCommand(runParameters as RunTaskCommandInput));
const taskArn = task.tasks?.[0].taskArn || '';
OrchestratorLogger.log('Orchestrator job is starting');
CloudRunnerLogger.log('Cloud runner job is starting');
await AWSTaskRunner.waitUntilTaskRunning(taskArn, cluster);
OrchestratorLogger.log(
`Orchestrator job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Async:${
OrchestratorOptions.asyncOrchestrator
CloudRunnerLogger.log(
`Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Async:${
CloudRunnerOptions.asyncCloudRunner
}`,
);
if (OrchestratorOptions.asyncOrchestrator) {
if (CloudRunnerOptions.asyncCloudRunner) {
const shouldCleanup: boolean = false;
const output: string = '';
OrchestratorLogger.log(`Watch Orchestrator To End: false`);
CloudRunnerLogger.log(`Watch Cloud Runner To End: false`);
return { output, shouldCleanup };
}
OrchestratorLogger.log(`Streaming...`);
CloudRunnerLogger.log(`Streaming...`);
const { output, shouldCleanup } = await this.streamLogsUntilTaskStops(cluster, taskArn, streamName);
let exitCode;
let containerState;
let taskData;
while (exitCode === undefined) {
await new Promise((resolve) => setTimeout(resolve, 10000));
await new Promise((resolve) => resolve(10000));
taskData = await AWSTaskRunner.describeTasks(cluster, taskArn);
const containers = taskData?.containers as any[] | undefined;
if (!containers || containers.length === 0) {
continue;
}
containerState = containers[0];
containerState = taskData.containers?.[0];
exitCode = containerState?.exitCode;
}
OrchestratorLogger.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`);
CloudRunnerLogger.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`);
if (exitCode === undefined) {
OrchestratorLogger.logWarning(`Undefined exitcode for container`);
CloudRunnerLogger.logWarning(`Undefined exitcode for container`);
}
const wasSuccessful = exitCode === 0;
if (wasSuccessful) {
OrchestratorLogger.log(`Orchestrator job has finished successfully`);
CloudRunnerLogger.log(`Cloud runner job has finished successfully`);
return { output, shouldCleanup };
}
@@ -155,18 +125,19 @@ class AWSTaskRunner {
try {
await waitUntilTasksRunning(
{
client: AwsClientFactory.getECS(),
maxWaitTime: 300,
minDelay: 5,
maxDelay: 30,
client: AWSTaskRunner.ECS,
maxWaitTime: 120,
},
{ tasks: [taskArn], cluster },
);
} catch (error_) {
const error = error_ as Error;
await new Promise((resolve) => setTimeout(resolve, 3000));
const taskAfterError = await AWSTaskRunner.describeTasks(cluster, taskArn);
OrchestratorLogger.log(`Orchestrator job has ended ${taskAfterError?.containers?.[0]?.lastStatus}`);
CloudRunnerLogger.log(
`Cloud runner job has ended ${
(await AWSTaskRunner.describeTasks(cluster, taskArn)).containers?.[0].lastStatus
}`,
);
core.setFailed(error);
core.error(error);
@@ -174,42 +145,22 @@ class AWSTaskRunner {
}
static async describeTasks(clusterName: string, taskArn: string) {
const maxAttempts = 10;
let delayMs = 1000;
const maxDelayMs = 60000;
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
const tasks = await AwsClientFactory.getECS().send(
new DescribeTasksCommand({ cluster: clusterName, tasks: [taskArn] }),
);
if (tasks.tasks?.[0]) {
return tasks.tasks?.[0];
}
throw new Error('No task found');
} catch (error: any) {
const isThrottle = error?.name === 'ThrottlingException' || /rate exceeded/i.test(String(error?.message));
if (!isThrottle || attempt === maxAttempts) {
throw error;
}
const jitterMs = Math.floor(Math.random() * Math.min(1000, delayMs));
const sleepMs = delayMs + jitterMs;
OrchestratorLogger.log(
`AWS throttled DescribeTasks (attempt ${attempt}/${maxAttempts}), backing off ${sleepMs}ms (${delayMs} + jitter ${jitterMs})`,
);
await new Promise((r) => setTimeout(r, sleepMs));
delayMs = Math.min(delayMs * 2, maxDelayMs);
}
const tasks = await AWSTaskRunner.ECS.send(new DescribeTasksCommand({ cluster: clusterName, tasks: [taskArn] }));
if (tasks.tasks?.[0]) {
return tasks.tasks?.[0];
} else {
throw new Error('No task found');
}
}
static async streamLogsUntilTaskStops(clusterName: string, taskArn: string, kinesisStreamName: string) {
await new Promise((resolve) => setTimeout(resolve, 3000));
OrchestratorLogger.log(`Streaming...`);
CloudRunnerLogger.log(`Streaming...`);
const stream = await AWSTaskRunner.getLogStream(kinesisStreamName);
let iterator = await AWSTaskRunner.getLogIterator(stream);
const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${Orchestrator.buildParameters.awsStackName}${AWSTaskRunner.encodedUnderscore}${Orchestrator.buildParameters.awsStackName}-${Orchestrator.buildParameters.buildGuid}`;
OrchestratorLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`);
const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsStackName}${AWSTaskRunner.encodedUnderscore}${CloudRunner.buildParameters.awsStackName}-${CloudRunner.buildParameters.buildGuid}`;
CloudRunnerLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`);
await GitHub.updateGitHubCheck(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`, ``);
let shouldReadLogs = true;
let shouldCleanup = true;
@@ -219,9 +170,6 @@ class AWSTaskRunner {
await new Promise((resolve) => setTimeout(resolve, 1500));
const taskData = await AWSTaskRunner.describeTasks(clusterName, taskArn);
({ timestamp, shouldReadLogs } = AWSTaskRunner.checkStreamingShouldContinue(taskData, timestamp, shouldReadLogs));
if (taskData?.lastStatus !== 'RUNNING') {
await new Promise((resolve) => setTimeout(resolve, 3500));
}
({ iterator, shouldReadLogs, output, shouldCleanup } = await AWSTaskRunner.handleLogStreamIteration(
iterator,
shouldReadLogs,
@@ -239,22 +187,7 @@ class AWSTaskRunner {
output: string,
shouldCleanup: boolean,
) {
let records: any;
try {
records = await AwsClientFactory.getKinesis().send(new GetRecordsCommand({ ShardIterator: iterator }));
} catch (error: any) {
const isThrottle = error?.name === 'ThrottlingException' || /rate exceeded/i.test(String(error?.message));
if (isThrottle) {
const baseBackoffMs = 1000;
const jitterMs = Math.floor(Math.random() * 1000);
const sleepMs = baseBackoffMs + jitterMs;
OrchestratorLogger.log(`AWS throttled GetRecords, backing off ${sleepMs}ms (1000 + jitter ${jitterMs})`);
await new Promise((r) => setTimeout(r, sleepMs));
return { iterator, shouldReadLogs, output, shouldCleanup };
}
throw error;
}
const records = await AWSTaskRunner.Kinesis.send(new GetRecordsCommand({ ShardIterator: iterator }));
iterator = records.NextShardIterator || '';
({ shouldReadLogs, output, shouldCleanup } = AWSTaskRunner.logRecords(
records,
@@ -267,27 +200,27 @@ class AWSTaskRunner {
return { iterator, shouldReadLogs, output, shouldCleanup };
}
private static checkStreamingShouldContinue(taskData: any, timestamp: number, shouldReadLogs: boolean) {
private static checkStreamingShouldContinue(taskData: Task, timestamp: number, shouldReadLogs: boolean) {
if (taskData?.lastStatus === 'UNKNOWN') {
OrchestratorLogger.log('## Orchestrator job unknwon');
CloudRunnerLogger.log('## Cloud runner job unknwon');
}
if (taskData?.lastStatus !== 'RUNNING') {
if (timestamp === 0) {
OrchestratorLogger.log('## Orchestrator job stopped, streaming end of logs');
CloudRunnerLogger.log('## Cloud runner job stopped, streaming end of logs');
timestamp = Date.now();
}
if (timestamp !== 0 && Date.now() - timestamp > 30000) {
OrchestratorLogger.log('## Orchestrator status is not RUNNING for 30 seconds, last query for logs');
CloudRunnerLogger.log('## Cloud runner status is not RUNNING for 30 seconds, last query for logs');
shouldReadLogs = false;
}
OrchestratorLogger.log(`## Status of job: ${taskData.lastStatus}`);
CloudRunnerLogger.log(`## Status of job: ${taskData.lastStatus}`);
}
return { timestamp, shouldReadLogs };
}
private static logRecords(
records: any,
records: GetRecordsCommandOutput,
iterator: string,
shouldReadLogs: boolean,
output: string,
@@ -315,13 +248,13 @@ class AWSTaskRunner {
}
private static async getLogStream(kinesisStreamName: string) {
return await AwsClientFactory.getKinesis().send(new DescribeStreamCommand({ StreamName: kinesisStreamName }));
return await AWSTaskRunner.Kinesis.send(new DescribeStreamCommand({ StreamName: kinesisStreamName }));
}
private static async getLogIterator(stream: any) {
private static async getLogIterator(stream: DescribeStreamCommandOutput) {
return (
(
await AwsClientFactory.getKinesis().send(
await AWSTaskRunner.Kinesis.send(
new GetShardIteratorCommand({
ShardIteratorType: 'TRIM_HORIZON',
StreamName: stream.StreamDescription?.StreamName ?? '',

View File

@@ -1,7 +1,7 @@
import Orchestrator from '../../../orchestrator';
import CloudRunner from '../../../cloud-runner';
export class TaskDefinitionFormation {
public static readonly description: string = `Game CI Orchestrator Task Stack`;
public static readonly description: string = `Game CI Cloud Runner Task Stack`;
public static get formation(): string {
return `AWSTemplateFormatVersion: 2010-09-09
Description: ${TaskDefinitionFormation.description}
@@ -29,11 +29,11 @@ Parameters:
Default: 80
Description: What port number the application inside the docker container is binding to
ContainerCpu:
Default: ${Orchestrator.buildParameters.containerCpu}
Default: ${CloudRunner.buildParameters.containerCpu}
Type: Number
Description: How much CPU to give the container. 1024 is 1 CPU
ContainerMemory:
Default: ${Orchestrator.buildParameters.containerMemory}
Default: ${CloudRunner.buildParameters.containerMemory}
Type: Number
Description: How much memory in megabytes to give the container
BUILDGUID:
@@ -127,7 +127,8 @@ Resources:
- SourceVolume: efs-data
ContainerPath: !Ref EFSMountDirectory
ReadOnly: false
# template secrets p3 - container def
Secrets:
# template secrets p3 - container def
LogConfiguration:
LogDriver: awslogs
Options:

View File

@@ -1,10 +1,9 @@
// eslint-disable-next-line import/named
import { StackResource } from '@aws-sdk/client-cloudformation';
class OrchestratorAWSTaskDef {
class CloudRunnerAWSTaskDef {
public taskDefStackName!: string;
public taskDefCloudFormation!: string;
public taskDefResources: StackResource[] | undefined;
public baseResources: StackResource[] | undefined;
}
export default OrchestratorAWSTaskDef;
export default CloudRunnerAWSTaskDef;

View File

@@ -1,11 +1,13 @@
import { CloudFormation, DeleteStackCommand, waitUntilStackDeleteComplete } from '@aws-sdk/client-cloudformation';
import OrchestratorSecret from '../../options/orchestrator-secret';
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
import OrchestratorAWSTaskDef from './orchestrator-aws-task-def';
import { ECS as ECSClient } from '@aws-sdk/client-ecs';
import { Kinesis } from '@aws-sdk/client-kinesis';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
import AwsTaskRunner from './aws-task-runner';
import { ProviderInterface } from '../provider-interface';
import BuildParameters from '../../../build-parameters';
import OrchestratorLogger from '../../services/core/orchestrator-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { AWSJobStack as AwsJobStack } from './aws-job-stack';
import { AWSBaseStack as AwsBaseStack } from './aws-base-stack';
import { Input } from '../../..';
@@ -13,20 +15,7 @@ import { GarbageCollectionService } from './services/garbage-collection-service'
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
import { TaskService } from './services/task-service';
import OrchestratorOptions from '../../options/orchestrator-options';
import { AwsClientFactory } from './aws-client-factory';
import ResourceTracking from '../../services/core/resource-tracking';
const DEFAULT_STACK_WAIT_TIME_SECONDS = 600;
function getStackWaitTime(): number {
const overrideValue = Number(process.env.ORCHESTRATOR_AWS_STACK_WAIT_TIME ?? '');
if (!Number.isNaN(overrideValue) && overrideValue > 0) {
return overrideValue;
}
return DEFAULT_STACK_WAIT_TIME_SECONDS;
}
import CloudRunnerOptions from '../../options/cloud-runner-options';
class AWSBuildEnvironment implements ProviderInterface {
private baseStackName: string;
@@ -88,7 +77,7 @@ class AWSBuildEnvironment implements ProviderInterface {
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {
process.env.AWS_REGION = Input.region;
const CF = AwsClientFactory.getCloudFormation();
const CF = new CloudFormation({ region: Input.region });
await new AwsBaseStack(this.baseStackName).setupBaseStack(CF);
}
@@ -98,16 +87,15 @@ class AWSBuildEnvironment implements ProviderInterface {
commands: string,
mountdir: string,
workingdir: string,
environment: OrchestratorEnvironmentVariable[],
secrets: OrchestratorSecret[],
environment: CloudRunnerEnvironmentVariable[],
secrets: CloudRunnerSecret[],
): Promise<string> {
process.env.AWS_REGION = Input.region;
ResourceTracking.logAllocationSummary('aws workflow');
await ResourceTracking.logDiskUsageSnapshot('aws workflow (host)');
AwsClientFactory.getECS();
const CF = AwsClientFactory.getCloudFormation();
AwsClientFactory.getKinesis();
OrchestratorLogger.log(`AWS Region: ${CF.config.region}`);
const ECS = new ECSClient({ region: Input.region });
const CF = new CloudFormation({ region: Input.region });
AwsTaskRunner.ECS = ECS;
AwsTaskRunner.Kinesis = new Kinesis({ region: Input.region });
CloudRunnerLogger.log(`AWS Region: ${CF.config.region}`);
const entrypoint = ['/bin/sh'];
const startTimeMs = Date.now();
const taskDef = await new AwsJobStack(this.baseStackName).setupCloudFormations(
@@ -124,37 +112,36 @@ class AWSBuildEnvironment implements ProviderInterface {
let postRunTaskTimeMs;
try {
const postSetupStacksTimeMs = Date.now();
OrchestratorLogger.log(`Setup job time: ${Math.floor((postSetupStacksTimeMs - startTimeMs) / 1000)}s`);
CloudRunnerLogger.log(`Setup job time: ${Math.floor((postSetupStacksTimeMs - startTimeMs) / 1000)}s`);
const { output, shouldCleanup } = await AwsTaskRunner.runTask(taskDef, environment, commands);
postRunTaskTimeMs = Date.now();
OrchestratorLogger.log(`Run job time: ${Math.floor((postRunTaskTimeMs - postSetupStacksTimeMs) / 1000)}s`);
CloudRunnerLogger.log(`Run job time: ${Math.floor((postRunTaskTimeMs - postSetupStacksTimeMs) / 1000)}s`);
if (shouldCleanup) {
await this.cleanupResources(CF, taskDef);
}
const postCleanupTimeMs = Date.now();
if (postRunTaskTimeMs !== undefined)
OrchestratorLogger.log(`Cleanup job time: ${Math.floor((postCleanupTimeMs - postRunTaskTimeMs) / 1000)}s`);
CloudRunnerLogger.log(`Cleanup job time: ${Math.floor((postCleanupTimeMs - postRunTaskTimeMs) / 1000)}s`);
return output;
} catch (error) {
OrchestratorLogger.log(`error running task ${error}`);
CloudRunnerLogger.log(`error running task ${error}`);
await this.cleanupResources(CF, taskDef);
throw error;
}
}
async cleanupResources(CF: CloudFormation, taskDef: OrchestratorAWSTaskDef) {
const stackWaitTimeSeconds = getStackWaitTime();
OrchestratorLogger.log(`Cleanup starting (waiting up to ${stackWaitTimeSeconds}s for stack deletion)`);
async cleanupResources(CF: CloudFormation, taskDef: CloudRunnerAWSTaskDef) {
CloudRunnerLogger.log('Cleanup starting');
await CF.send(new DeleteStackCommand({ StackName: taskDef.taskDefStackName }));
if (OrchestratorOptions.useCleanupCron) {
if (CloudRunnerOptions.useCleanupCron) {
await CF.send(new DeleteStackCommand({ StackName: `${taskDef.taskDefStackName}-cleanup` }));
}
await waitUntilStackDeleteComplete(
{
client: CF,
maxWaitTime: stackWaitTimeSeconds,
maxWaitTime: 200,
},
{
StackName: taskDef.taskDefStackName,
@@ -163,14 +150,14 @@ class AWSBuildEnvironment implements ProviderInterface {
await waitUntilStackDeleteComplete(
{
client: CF,
maxWaitTime: stackWaitTimeSeconds,
maxWaitTime: 200,
},
{
StackName: `${taskDef.taskDefStackName}-cleanup`,
},
);
OrchestratorLogger.log(`Deleted Stack: ${taskDef.taskDefStackName}`);
OrchestratorLogger.log('Cleanup complete');
CloudRunnerLogger.log(`Deleted Stack: ${taskDef.taskDefStackName}`);
CloudRunnerLogger.log('Cleanup complete');
}
}
export default AWSBuildEnvironment;

View File

@@ -1,10 +1,14 @@
import { DeleteStackCommand, DescribeStackResourcesCommand } from '@aws-sdk/client-cloudformation';
import { DeleteLogGroupCommand } from '@aws-sdk/client-cloudwatch-logs';
import { StopTaskCommand } from '@aws-sdk/client-ecs';
import {
CloudFormation,
DeleteStackCommand,
DeleteStackCommandInput,
DescribeStackResourcesCommand,
} from '@aws-sdk/client-cloudformation';
import { CloudWatchLogs, DeleteLogGroupCommand } from '@aws-sdk/client-cloudwatch-logs';
import { ECS, StopTaskCommand } from '@aws-sdk/client-ecs';
import Input from '../../../../input';
import OrchestratorLogger from '../../../services/core/orchestrator-logger';
import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
import { TaskService } from './task-service';
import { AwsClientFactory } from '../aws-client-factory';
export class GarbageCollectionService {
static isOlderThan1day(date: Date) {
@@ -15,9 +19,9 @@ export class GarbageCollectionService {
public static async cleanup(deleteResources = false, OneDayOlderOnly: boolean = false) {
process.env.AWS_REGION = Input.region;
const CF = AwsClientFactory.getCloudFormation();
const ecs = AwsClientFactory.getECS();
const cwl = AwsClientFactory.getCloudWatchLogs();
const CF = new CloudFormation({ region: Input.region });
const ecs = new ECS({ region: Input.region });
const cwl = new CloudWatchLogs({ region: Input.region });
const taskDefinitionsInUse = new Array();
const tasks = await TaskService.getTasks();
@@ -25,7 +29,7 @@ export class GarbageCollectionService {
const { taskElement, element } = task;
taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.createdAt!))) {
OrchestratorLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
CloudRunnerLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
await ecs.send(new StopTaskCommand({ task: taskElement.taskArn || '', cluster: element }));
}
}
@@ -37,7 +41,7 @@ export class GarbageCollectionService {
(x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId),
)
) {
OrchestratorLogger.log(`Skipping ${element.StackName} - active task was running not deleting`);
CloudRunnerLogger.log(`Skipping ${element.StackName} - active task was running not deleting`);
return;
}
@@ -47,13 +51,14 @@ export class GarbageCollectionService {
(!OneDayOlderOnly || (element.CreationTime && GarbageCollectionService.isOlderThan1day(element.CreationTime)))
) {
if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') {
OrchestratorLogger.log(`Skipping ${element.StackName} ignore list`);
CloudRunnerLogger.log(`Skipping ${element.StackName} ignore list`);
return;
}
OrchestratorLogger.log(`Deleting ${element.StackName}`);
await CF.send(new DeleteStackCommand({ StackName: element.StackName }));
CloudRunnerLogger.log(`Deleting ${element.StackName}`);
const deleteStackInput: DeleteStackCommandInput = { StackName: element.StackName };
await CF.send(new DeleteStackCommand(deleteStackInput));
}
}
const logGroups = await TaskService.getLogGroups();
@@ -62,14 +67,14 @@ export class GarbageCollectionService {
deleteResources &&
(!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.creationTime!)))
) {
OrchestratorLogger.log(`Deleting ${element.logGroupName}`);
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
await cwl.send(new DeleteLogGroupCommand({ logGroupName: element.logGroupName || '' }));
}
}
const locks = await TaskService.getLocks();
for (const element of locks) {
OrchestratorLogger.log(`Lock: ${element.Key}`);
CloudRunnerLogger.log(`Lock: ${element.Key}`);
}
}
}

View File

@@ -1,22 +1,31 @@
import {
CloudFormation,
DescribeStackResourcesCommand,
DescribeStacksCommand,
ListStacksCommand,
StackSummary,
} from '@aws-sdk/client-cloudformation';
import type { StackSummary } from '@aws-sdk/client-cloudformation';
// eslint-disable-next-line import/named
import { DescribeLogGroupsCommand, DescribeLogGroupsCommandInput } from '@aws-sdk/client-cloudwatch-logs';
import type { LogGroup } from '@aws-sdk/client-cloudwatch-logs';
import { DescribeTasksCommand, ListClustersCommand, ListTasksCommand } from '@aws-sdk/client-ecs';
import type { Task } from '@aws-sdk/client-ecs';
import { ListObjectsV2Command } from '@aws-sdk/client-s3';
import {
CloudWatchLogs,
DescribeLogGroupsCommand,
DescribeLogGroupsCommandInput,
LogGroup,
} from '@aws-sdk/client-cloudwatch-logs';
import {
DescribeTasksCommand,
DescribeTasksCommandInput,
ECS,
ListClustersCommand,
ListTasksCommand,
ListTasksCommandInput,
Task,
} from '@aws-sdk/client-ecs';
import { ListObjectsCommand, ListObjectsCommandInput, S3 } from '@aws-sdk/client-s3';
import Input from '../../../../input';
import OrchestratorLogger from '../../../services/core/orchestrator-logger';
import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
import AwsTaskRunner from '../aws-task-runner';
import Orchestrator from '../../../orchestrator';
import { AwsClientFactory } from '../aws-client-factory';
import SharedWorkspaceLocking from '../../../services/core/shared-workspace-locking';
import CloudRunner from '../../../cloud-runner';
export class TaskService {
static async watch() {
@@ -29,27 +38,27 @@ export class TaskService {
return output;
}
public static async getCloudFormationJobStacks(): Promise<StackSummary[]> {
public static async getCloudFormationJobStacks() {
const result: StackSummary[] = [];
OrchestratorLogger.log(``);
OrchestratorLogger.log(`List Cloud Formation Stacks`);
CloudRunnerLogger.log(``);
CloudRunnerLogger.log(`List Cloud Formation Stacks`);
process.env.AWS_REGION = Input.region;
const CF = AwsClientFactory.getCloudFormation();
const CF = new CloudFormation({ region: Input.region });
const stacks =
(await CF.send(new ListStacksCommand({}))).StackSummaries?.filter(
(_x) =>
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription,
) || [];
OrchestratorLogger.log(``);
OrchestratorLogger.log(`Cloud Formation Stacks ${stacks.length}`);
CloudRunnerLogger.log(``);
CloudRunnerLogger.log(`Cloud Formation Stacks ${stacks.length}`);
for (const element of stacks) {
if (!element.CreationTime) {
OrchestratorLogger.log(`${element.StackName} due to undefined CreationTime`);
CloudRunnerLogger.log(`${element.StackName} due to undefined CreationTime`);
}
const ageDate: Date = new Date(Date.now() - (element.CreationTime?.getTime() ?? 0));
OrchestratorLogger.log(
CloudRunnerLogger.log(
`Task Stack ${element.StackName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
@@ -61,79 +70,69 @@ export class TaskService {
(_x) =>
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription,
) || [];
OrchestratorLogger.log(``);
OrchestratorLogger.log(`Base Stacks ${baseStacks.length}`);
CloudRunnerLogger.log(``);
CloudRunnerLogger.log(`Base Stacks ${baseStacks.length}`);
for (const element of baseStacks) {
if (!element.CreationTime) {
OrchestratorLogger.log(`${element.StackName} due to undefined CreationTime`);
CloudRunnerLogger.log(`${element.StackName} due to undefined CreationTime`);
}
const ageDate: Date = new Date(Date.now() - (element.CreationTime?.getTime() ?? 0));
OrchestratorLogger.log(
CloudRunnerLogger.log(
`Task Stack ${element.StackName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
result.push(element);
}
OrchestratorLogger.log(``);
CloudRunnerLogger.log(``);
return result;
}
public static async getTasks(): Promise<{ taskElement: Task; element: string }[]> {
public static async getTasks() {
const result: { taskElement: Task; element: string }[] = [];
OrchestratorLogger.log(``);
OrchestratorLogger.log(`List Tasks`);
CloudRunnerLogger.log(``);
CloudRunnerLogger.log(`List Tasks`);
process.env.AWS_REGION = Input.region;
const ecs = AwsClientFactory.getECS();
const clusters: string[] = [];
{
let nextToken: string | undefined;
do {
const clusterResponse = await ecs.send(new ListClustersCommand({ nextToken }));
clusters.push(...(clusterResponse.clusterArns ?? []));
nextToken = clusterResponse.nextToken;
} while (nextToken);
}
OrchestratorLogger.log(`Task Clusters ${clusters.length}`);
const ecs = new ECS({ region: Input.region });
const clusters = (await ecs.send(new ListClustersCommand({}))).clusterArns || [];
CloudRunnerLogger.log(`Task Clusters ${clusters.length}`);
for (const element of clusters) {
const taskArns: string[] = [];
{
let nextToken: string | undefined;
do {
const taskResponse = await ecs.send(new ListTasksCommand({ cluster: element, nextToken }));
taskArns.push(...(taskResponse.taskArns ?? []));
nextToken = taskResponse.nextToken;
} while (nextToken);
}
if (taskArns.length > 0) {
const describeInput = { tasks: taskArns, cluster: element };
const input: ListTasksCommandInput = {
cluster: element,
};
const list = (await ecs.send(new ListTasksCommand(input))).taskArns || [];
if (list.length > 0) {
const describeInput: DescribeTasksCommandInput = { tasks: list, cluster: element };
const describeList = (await ecs.send(new DescribeTasksCommand(describeInput))).tasks || [];
if (describeList.length === 0) {
OrchestratorLogger.log(`No Tasks`);
CloudRunnerLogger.log(`No Tasks`);
continue;
}
OrchestratorLogger.log(`Tasks ${describeList.length}`);
CloudRunnerLogger.log(`Tasks ${describeList.length}`);
for (const taskElement of describeList) {
if (taskElement === undefined) {
continue;
}
taskElement.overrides = {};
taskElement.attachments = [];
if (taskElement.createdAt === undefined) {
OrchestratorLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
CloudRunnerLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
continue;
}
result.push({ taskElement, element });
}
}
}
OrchestratorLogger.log(``);
CloudRunnerLogger.log(``);
return result;
}
public static async awsDescribeJob(job: string) {
process.env.AWS_REGION = Input.region;
const CF = AwsClientFactory.getCloudFormation();
const CF = new CloudFormation({ region: Input.region });
try {
const stack =
(await CF.send(new ListStacksCommand({}))).StackSummaries?.find((_x) => _x.StackName === job) || undefined;
@@ -143,7 +142,7 @@ export class TaskService {
throw new Error('stack not defined');
}
if (!stack.CreationTime) {
OrchestratorLogger.log(`${stack.StackName} due to undefined CreationTime`);
CloudRunnerLogger.log(`${stack.StackName} due to undefined CreationTime`);
}
const ageDate: Date = new Date(Date.now() - (stack.CreationTime?.getTime() ?? 0));
const message = `
@@ -153,43 +152,40 @@ export class TaskService {
${JSON.stringify(stackInfo, undefined, 4)}
${JSON.stringify(stackInfo2, undefined, 4)}
`;
OrchestratorLogger.log(message);
CloudRunnerLogger.log(message);
return message;
} catch (error) {
OrchestratorLogger.error(
CloudRunnerLogger.error(
`Failed to describe job ${job}: ${error instanceof Error ? error.message : String(error)}`,
);
throw error;
}
}
public static async getLogGroups(): Promise<LogGroup[]> {
const result: LogGroup[] = [];
public static async getLogGroups() {
const result: Array<LogGroup> = [];
process.env.AWS_REGION = Input.region;
const cwl = AwsClientFactory.getCloudWatchLogs();
const ecs = new CloudWatchLogs();
let logStreamInput: DescribeLogGroupsCommandInput = {
/* logGroupNamePrefix: 'game-ci' */
};
let logGroupsDescribe = await cwl.send(new DescribeLogGroupsCommand(logStreamInput));
let logGroupsDescribe = await ecs.send(new DescribeLogGroupsCommand(logStreamInput));
const logGroups = logGroupsDescribe.logGroups || [];
while (logGroupsDescribe.nextToken) {
logStreamInput = {
/* logGroupNamePrefix: 'game-ci',*/
nextToken: logGroupsDescribe.nextToken,
};
logGroupsDescribe = await cwl.send(new DescribeLogGroupsCommand(logStreamInput));
logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
logGroupsDescribe = await ecs.send(new DescribeLogGroupsCommand(logStreamInput));
logGroups.push(...(logGroupsDescribe?.logGroups || []));
}
OrchestratorLogger.log(`Log Groups ${logGroups.length}`);
CloudRunnerLogger.log(`Log Groups ${logGroups.length}`);
for (const element of logGroups) {
if (element.creationTime === undefined) {
OrchestratorLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
continue;
}
const ageDate: Date = new Date(Date.now() - element.creationTime);
OrchestratorLogger.log(
CloudRunnerLogger.log(
`Task Stack ${element.logGroupName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
@@ -199,22 +195,15 @@ export class TaskService {
return result;
}
public static async getLocks(): Promise<Array<{ Key: string }>> {
public static async getLocks() {
process.env.AWS_REGION = Input.region;
if (Orchestrator.buildParameters.storageProvider === 'rclone') {
// eslint-disable-next-line no-unused-vars
type ListObjectsFunction = (prefix: string) => Promise<string[]>;
const objects = await (SharedWorkspaceLocking as unknown as { listObjects: ListObjectsFunction }).listObjects('');
return objects.map((x: string) => ({ Key: x }));
}
const s3 = AwsClientFactory.getS3();
const listRequest = {
Bucket: Orchestrator.buildParameters.awsStackName,
const s3 = new S3({ region: Input.region });
const listRequest: ListObjectsCommandInput = {
Bucket: CloudRunner.buildParameters.awsStackName,
};
const results = await s3.send(new ListObjectsV2Command(listRequest));
const results = await s3.send(new ListObjectsCommand(listRequest));
return (results.Contents || []).map((object) => ({ Key: object.Key || '' }));
return results.Contents || [];
}
}

View File

@@ -1,20 +1,20 @@
import BuildParameters from '../../../build-parameters';
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
import OrchestratorLogger from '../../services/core/orchestrator-logger';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { ProviderInterface } from '../provider-interface';
import OrchestratorSecret from '../../options/orchestrator-secret';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import Docker from '../../../docker';
import { Action } from '../../..';
import { writeFileSync } from 'node:fs';
import Orchestrator from '../../orchestrator';
import CloudRunner from '../../cloud-runner';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
import { OrchestratorSystem } from '../../services/core/orchestrator-system';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
import * as fs from 'node:fs';
import { CommandHookService } from '../../services/hooks/command-hook-service';
import { StringKeyValuePair } from '../../../shared-types';
class LocalDockerOrchestrator implements ProviderInterface {
class LocalDockerCloudRunner implements ProviderInterface {
public buildParameters!: BuildParameters;
listResources(): Promise<ProviderResource[]> {
@@ -50,15 +50,15 @@ class LocalDockerOrchestrator implements ProviderInterface {
const { workspace } = Action;
if (
fs.existsSync(
`${workspace}/orchestrator-cache/cache/build/build-${buildParameters.buildGuid}.tar${
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
`${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}`,
)
) {
await OrchestratorSystem.Run(`ls ${workspace}/orchestrator-cache/cache/build/`);
await OrchestratorSystem.Run(
`rm -r ${workspace}/orchestrator-cache/cache/build/build-${buildParameters.buildGuid}.tar${
Orchestrator.buildParameters.useCompressionStrategy ? '.lz4' : ''
await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache/cache/build/`);
await CloudRunnerSystem.Run(
`rm -r ${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}`,
);
}
@@ -80,47 +80,22 @@ class LocalDockerOrchestrator implements ProviderInterface {
commands: string,
mountdir: string,
workingdir: string,
environment: OrchestratorEnvironmentVariable[],
secrets: OrchestratorSecret[],
environment: CloudRunnerEnvironmentVariable[],
secrets: CloudRunnerSecret[],
): Promise<string> {
OrchestratorLogger.log(buildGuid);
OrchestratorLogger.log(commands);
CloudRunnerLogger.log(buildGuid);
CloudRunnerLogger.log(commands);
const { workspace, actionFolder } = Action;
const content: StringKeyValuePair[] = [];
for (const x of secrets) {
content.push({ name: x.EnvironmentVariable, value: x.ParameterValue });
}
// Replace localhost with host.docker.internal for LocalStack endpoints (similar to K8s)
// This allows Docker containers to access LocalStack running on the host
const endpointEnvironmentNames = new Set([
'AWS_S3_ENDPOINT',
'AWS_ENDPOINT',
'AWS_CLOUD_FORMATION_ENDPOINT',
'AWS_ECS_ENDPOINT',
'AWS_KINESIS_ENDPOINT',
'AWS_CLOUD_WATCH_LOGS_ENDPOINT',
'INPUT_AWSS3ENDPOINT',
'INPUT_AWSENDPOINT',
]);
for (const x of environment) {
let value = x.value;
if (
typeof value === 'string' &&
endpointEnvironmentNames.has(x.name) &&
(value.startsWith('http://localhost') || value.startsWith('http://127.0.0.1'))
) {
// Replace localhost with host.docker.internal so containers can access host services
value = value
.replace('http://localhost', 'http://host.docker.internal')
.replace('http://127.0.0.1', 'http://host.docker.internal');
OrchestratorLogger.log(`Replaced localhost with host.docker.internal for ${x.name}: ${value}`);
}
content.push({ name: x.name, value });
content.push({ name: x.name, value: x.value });
}
// if (this.buildParameters?.orchestratorIntegrationTests) {
// if (this.buildParameters?.cloudRunnerIntegrationTests) {
// core.info(JSON.stringify(content, undefined, 4));
// core.info(JSON.stringify(secrets, undefined, 4));
// core.info(JSON.stringify(environment, undefined, 4));
@@ -137,33 +112,25 @@ class LocalDockerOrchestrator implements ProviderInterface {
// core.info(JSON.stringify({ workspace, actionFolder, ...this.buildParameters, ...content }, undefined, 4));
const entrypointFilePath = `start.sh`;
// Use #!/bin/sh for POSIX compatibility (Alpine-based images like rclone/rclone don't have bash)
const fileContents = `#!/bin/sh
const fileContents = `#!/bin/bash
set -e
mkdir -p /github/workspace/orchestrator-cache
mkdir -p /github/workspace/cloud-runner-cache
mkdir -p /data/cache
cp -a /github/workspace/orchestrator-cache/. ${sharedFolder}
cp -a /github/workspace/cloud-runner-cache/. ${sharedFolder}
${CommandHookService.ApplyHooksToCommands(commands, this.buildParameters)}
# Only copy cache directory, exclude retained workspaces to avoid running out of disk space
if [ -d "${sharedFolder}cache" ]; then
cp -a ${sharedFolder}cache/. /github/workspace/orchestrator-cache/cache/ || true
fi
# Copy test files from /data/ root to workspace for test assertions
# This allows tests to write files to /data/ and have them available in the workspace
find ${sharedFolder} -maxdepth 1 -type f -name "test-*" -exec cp -a {} /github/workspace/orchestrator-cache/ \\; || true
cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
`;
writeFileSync(`${workspace}/${entrypointFilePath}`, fileContents, {
flag: 'w',
});
if (Orchestrator.buildParameters.orchestratorDebug) {
OrchestratorLogger.log(`Running local-docker: \n ${fileContents}`);
if (CloudRunner.buildParameters.cloudRunnerDebug) {
CloudRunnerLogger.log(`Running local-docker: \n ${fileContents}`);
}
if (fs.existsSync(`${workspace}/orchestrator-cache`)) {
await OrchestratorSystem.Run(`ls ${workspace}/orchestrator-cache && du -sh ${workspace}/orchestrator-cache`);
if (fs.existsSync(`${workspace}/cloud-runner-cache`)) {
await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache && du -sh ${workspace}/cloud-runner-cache`);
}
const exitCode = await Docker.run(
image,
@@ -193,4 +160,4 @@ find ${sharedFolder} -maxdepth 1 -type f -name "test-*" -exec cp -a {} /github/w
return myOutput;
}
}
export default LocalDockerOrchestrator;
export default LocalDockerCloudRunner;

View File

@@ -2,22 +2,21 @@ import * as k8s from '@kubernetes/client-node';
import { BuildParameters } from '../../..';
import * as core from '@actions/core';
import { ProviderInterface } from '../provider-interface';
import OrchestratorSecret from '../../options/orchestrator-secret';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import KubernetesStorage from './kubernetes-storage';
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import KubernetesTaskRunner from './kubernetes-task-runner';
import KubernetesSecret from './kubernetes-secret';
import KubernetesJobSpecFactory from './kubernetes-job-spec-factory';
import KubernetesServiceAccount from './kubernetes-service-account';
import OrchestratorLogger from '../../services/core/orchestrator-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { CoreV1Api } from '@kubernetes/client-node';
import Orchestrator from '../../orchestrator';
import CloudRunner from '../../cloud-runner';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
import { KubernetesRole } from './kubernetes-role';
import { OrchestratorSystem } from '../../services/core/orchestrator-system';
import ResourceTracking from '../../services/core/resource-tracking';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
class Kubernetes implements ProviderInterface {
public static Instance: Kubernetes;
@@ -38,6 +37,7 @@ class Kubernetes implements ProviderInterface {
public serviceAccountName: string = '';
public ip: string = '';
// eslint-disable-next-line no-unused-vars
constructor(buildParameters: BuildParameters) {
Kubernetes.Instance = this;
this.kubeConfig = new k8s.KubeConfig();
@@ -46,8 +46,8 @@ class Kubernetes implements ProviderInterface {
this.kubeClientApps = this.kubeConfig.makeApiClient(k8s.AppsV1Api);
this.kubeClientBatch = this.kubeConfig.makeApiClient(k8s.BatchV1Api);
this.rbacAuthorizationV1Api = this.kubeConfig.makeApiClient(k8s.RbacAuthorizationV1Api);
this.namespace = buildParameters.containerNamespace ? buildParameters.containerNamespace : 'default';
OrchestratorLogger.log('Loaded default Kubernetes configuration for this environment');
this.namespace = 'default';
CloudRunnerLogger.log('Loaded default Kubernetes configuration for this environment');
}
async PushLogUpdate(logs: string) {
@@ -63,7 +63,7 @@ class Kubernetes implements ProviderInterface {
// logs to base64
logs = Buffer.from(logs).toString('base64');
const response = await OrchestratorSystem.Run(`curl -X POST -d "${logs}" ${url}`, false, true);
const response = await CloudRunnerSystem.Run(`curl -X POST -d "${logs}" ${url}`, false, true);
RemoteClientLogger.log(`Pushed logs to ${url} ${response}`);
}
@@ -133,19 +133,16 @@ class Kubernetes implements ProviderInterface {
commands: string,
mountdir: string,
workingdir: string,
environment: OrchestratorEnvironmentVariable[],
secrets: OrchestratorSecret[],
environment: CloudRunnerEnvironmentVariable[],
secrets: CloudRunnerSecret[],
): Promise<string> {
try {
OrchestratorLogger.log('Orchestrator K8s workflow!');
ResourceTracking.logAllocationSummary('k8s workflow');
await ResourceTracking.logDiskUsageSnapshot('k8s workflow (host)');
await ResourceTracking.logK3dNodeDiskUsage('k8s workflow (before job)');
CloudRunnerLogger.log('Cloud Runner K8s workflow!');
// Setup
const id =
BuildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(this.buildParameters)
? Orchestrator.lockedWorkspace
? CloudRunner.lockedWorkspace
: this.buildParameters.buildGuid;
this.pvcName = `unity-builder-pvc-${id}`;
await KubernetesStorage.createPersistentVolumeClaim(
@@ -159,134 +156,14 @@ class Kubernetes implements ProviderInterface {
this.jobName = `unity-builder-job-${this.buildGuid}`;
this.containerName = `main`;
await KubernetesSecret.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
// For tests, clean up old images before creating job to free space for image pull
// IMPORTANT: Preserve the Unity image to avoid re-pulling it
if (process.env['orchestratorTests'] === 'true') {
try {
OrchestratorLogger.log('Cleaning up old images in k3d node before pulling new image...');
const { OrchestratorSystem: OrchestratorSystemModule } = await import(
'../../services/core/orchestrator-system'
);
// Aggressive cleanup: remove stopped containers and non-Unity images
// IMPORTANT: Preserve Unity images (unityci/editor) to avoid re-pulling the 3.9GB image
const K3D_NODE_CONTAINERS = ['k3d-unity-builder-agent-0', 'k3d-unity-builder-server-0'];
const cleanupCommands: string[] = [];
for (const NODE of K3D_NODE_CONTAINERS) {
// Remove all stopped containers (this frees runtime space but keeps images)
cleanupCommands.push(
`docker exec ${NODE} sh -c "crictl rm --all 2>/dev/null || true" || true`,
`docker exec ${NODE} sh -c "for img in $(crictl images -q 2>/dev/null); do repo=$(crictl inspecti $img --format '{{.repo}}' 2>/dev/null || echo ''); if echo "$repo" | grep -qvE 'unityci/editor|unity'; then crictl rmi $img 2>/dev/null || true; fi; done" || true`,
`docker exec ${NODE} sh -c "crictl rmi --prune 2>/dev/null || true" || true`,
);
}
for (const cmd of cleanupCommands) {
try {
await OrchestratorSystemModule.Run(cmd, true, true);
} catch (cmdError) {
// Ignore individual command failures - cleanup is best effort
OrchestratorLogger.log(`Cleanup command failed (non-fatal): ${cmdError}`);
}
}
OrchestratorLogger.log('Cleanup completed (containers and non-Unity images removed, Unity images preserved)');
} catch (cleanupError) {
OrchestratorLogger.logWarning(`Failed to cleanup images before job creation: ${cleanupError}`);
// Continue anyway - image might already be cached
}
}
let output = '';
try {
// Before creating the job, verify we have the Unity image cached on the agent node
// If not cached, try to ensure it's available to avoid disk pressure during pull
if (process.env['orchestratorTests'] === 'true' && image.includes('unityci/editor')) {
try {
const { OrchestratorSystem: OrchestratorSystemModule2 } = await import(
'../../services/core/orchestrator-system'
);
// Check if image is cached on agent node (where pods run)
const agentImageCheck = await OrchestratorSystemModule2.Run(
`docker exec k3d-unity-builder-agent-0 sh -c "crictl images | grep -q unityci/editor && echo 'cached' || echo 'not_cached'" || echo 'not_cached'`,
true,
true,
);
if (agentImageCheck.includes('not_cached')) {
// Check if image is on server node
const serverImageCheck = await OrchestratorSystemModule2.Run(
`docker exec k3d-unity-builder-server-0 sh -c "crictl images | grep -q unityci/editor && echo 'cached' || echo 'not_cached'" || echo 'not_cached'`,
true,
true,
);
// Check available disk space on agent node
const diskInfo = await OrchestratorSystemModule2.Run(
'docker exec k3d-unity-builder-agent-0 sh -c "df -h /var/lib/rancher/k3s 2>/dev/null | tail -1 || df -h / 2>/dev/null | tail -1 || echo unknown" || echo unknown',
true,
true,
);
OrchestratorLogger.logWarning(
`Unity image not cached on agent node (where pods run). Server node: ${
serverImageCheck.includes('cached') ? 'has image' : 'no image'
}. Disk info: ${diskInfo.trim()}. Pod will attempt to pull image (3.9GB) which may fail due to disk pressure.`,
);
// If image is on server but not agent, log a warning
// NOTE: We don't attempt to pull here because:
// 1. Pulling a 3.9GB image can take several minutes and block the test
// 2. If there's not enough disk space, the pull will hang indefinitely
// 3. The pod will attempt to pull during scheduling anyway
// 4. If the pull fails, Kubernetes will provide proper error messages
if (serverImageCheck.includes('cached')) {
OrchestratorLogger.logWarning(
'Unity image exists on server node but not agent node. Pod will attempt to pull during scheduling. If pull fails due to disk pressure, ensure cleanup runs before this test.',
);
} else {
// Image not on either node - check if we have enough space to pull
// Extract available space from disk info
const availableSpaceMatch = diskInfo.match(/(\d+(?:\.\d+)?)\s*([gkm]?i?b)/i);
if (availableSpaceMatch) {
const availableValue = Number.parseFloat(availableSpaceMatch[1]);
const availableUnit = availableSpaceMatch[2].toUpperCase();
let availableGB = availableValue;
if (availableUnit.includes('M')) {
availableGB = availableValue / 1024;
} else if (availableUnit.includes('K')) {
availableGB = availableValue / (1024 * 1024);
}
// Unity image is ~3.9GB, need at least 4.5GB to be safe
if (availableGB < 4.5) {
OrchestratorLogger.logWarning(
`CRITICAL: Unity image not cached and only ${availableGB.toFixed(
2,
)}GB available. Image pull (3.9GB) will likely fail. Consider running cleanup or ensuring pre-pull step succeeds.`,
);
}
}
}
} else {
OrchestratorLogger.log('Unity image is cached on agent node - pod should start without pulling');
}
} catch (checkError) {
// Ignore check errors - continue with job creation
OrchestratorLogger.logWarning(`Failed to verify Unity image cache: ${checkError}`);
}
}
OrchestratorLogger.log('Job does not exist');
CloudRunnerLogger.log('Job does not exist');
await this.createJob(commands, image, mountdir, workingdir, environment, secrets);
OrchestratorLogger.log('Watching pod until running');
CloudRunnerLogger.log('Watching pod until running');
await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
OrchestratorLogger.log('Pod is running');
CloudRunnerLogger.log('Pod is running');
output += await KubernetesTaskRunner.runTask(
this.kubeConfig,
this.kubeClient,
@@ -296,9 +173,9 @@ class Kubernetes implements ProviderInterface {
this.namespace,
);
} catch (error: any) {
OrchestratorLogger.log(`error running k8s workflow ${error}`);
CloudRunnerLogger.log(`error running k8s workflow ${error}`);
await new Promise((resolve) => setTimeout(resolve, 3000));
OrchestratorLogger.log(
CloudRunnerLogger.log(
JSON.stringify(
(await this.kubeClient.listNamespacedEvent(this.namespace)).body.items
.map((x) => {
@@ -321,7 +198,7 @@ class Kubernetes implements ProviderInterface {
return output;
} catch (error) {
OrchestratorLogger.log('Running job failed');
CloudRunnerLogger.log('Running job failed');
core.error(JSON.stringify(error, undefined, 4));
// await this.cleanupTaskResources();
@@ -334,8 +211,8 @@ class Kubernetes implements ProviderInterface {
image: string,
mountdir: string,
workingdir: string,
environment: OrchestratorEnvironmentVariable[],
secrets: OrchestratorSecret[],
environment: CloudRunnerEnvironmentVariable[],
secrets: CloudRunnerSecret[],
) {
await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
const find = await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace);
@@ -359,8 +236,8 @@ class Kubernetes implements ProviderInterface {
image: string,
mountdir: string,
workingdir: string,
environment: OrchestratorEnvironmentVariable[],
secrets: OrchestratorSecret[],
environment: CloudRunnerEnvironmentVariable[],
secrets: CloudRunnerSecret[],
) {
for (let index = 0; index < 3; index++) {
try {
@@ -385,13 +262,13 @@ class Kubernetes implements ProviderInterface {
// await KubernetesRole.createRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api);
const result = await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
OrchestratorLogger.log(`Build job created`);
CloudRunnerLogger.log(`Build job created`);
await new Promise((promise) => setTimeout(promise, 5000));
OrchestratorLogger.log('Job created');
CloudRunnerLogger.log('Job created');
return result.body.metadata?.name;
} catch (error) {
OrchestratorLogger.log(`Error occured creating job: ${error}`);
CloudRunnerLogger.log(`Error occured creating job: ${error}`);
throw error;
}
}
@@ -403,26 +280,26 @@ class Kubernetes implements ProviderInterface {
}
async cleanupTaskResources() {
OrchestratorLogger.log('cleaning up');
CloudRunnerLogger.log('cleaning up');
try {
await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
await this.kubeClient.deleteNamespacedPod(this.podName, this.namespace);
await KubernetesRole.deleteRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api);
} catch (error: any) {
OrchestratorLogger.log(`Failed to cleanup`);
CloudRunnerLogger.log(`Failed to cleanup`);
if (error.response.body.reason !== `NotFound`) {
OrchestratorLogger.log(`Wasn't a not found error: ${error.response.body.reason}`);
CloudRunnerLogger.log(`Wasn't a not found error: ${error.response.body.reason}`);
throw error;
}
}
try {
await this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace);
} catch (error: any) {
OrchestratorLogger.log(`Failed to cleanup secret`);
OrchestratorLogger.log(error.response.body.reason);
CloudRunnerLogger.log(`Failed to cleanup secret`);
CloudRunnerLogger.log(error.response.body.reason);
}
OrchestratorLogger.log('cleaned up Secret, Job and Pod');
OrchestratorLogger.log('cleaning up finished');
CloudRunnerLogger.log('cleaned up Secret, Job and Pod');
CloudRunnerLogger.log('cleaning up finished');
}
async cleanupWorkflow(
@@ -435,14 +312,14 @@ class Kubernetes implements ProviderInterface {
if (BuildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
return;
}
OrchestratorLogger.log(`deleting PVC`);
CloudRunnerLogger.log(`deleting PVC`);
try {
await this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace);
await this.kubeClient.deleteNamespacedServiceAccount(this.serviceAccountName, this.namespace);
OrchestratorLogger.log('cleaned up PVC and Service Account');
CloudRunnerLogger.log('cleaned up PVC and Service Account');
} catch (error: any) {
OrchestratorLogger.log(`Cleanup failed ${JSON.stringify(error, undefined, 4)}`);
CloudRunnerLogger.log(`Cleanup failed ${JSON.stringify(error, undefined, 4)}`);
throw error;
}
}

View File

@@ -0,0 +1,127 @@
import { V1EnvVar, V1EnvVarSource, V1SecretKeySelector } from '@kubernetes/client-node';
import BuildParameters from '../../../build-parameters';
import { CommandHookService } from '../../services/hooks/command-hook-service';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import CloudRunner from '../../cloud-runner';
class KubernetesJobSpecFactory {
static getJobSpec(
command: string,
image: string,
mountdir: string,
workingDirectory: string,
environment: CloudRunnerEnvironmentVariable[],
secrets: CloudRunnerSecret[],
buildGuid: string,
buildParameters: BuildParameters,
secretName: string,
pvcName: string,
jobName: string,
k8s: any,
containerName: string,
ip: string = '',
) {
const job = new k8s.V1Job();
job.apiVersion = 'batch/v1';
job.kind = 'Job';
job.metadata = {
name: jobName,
labels: {
app: 'unity-builder',
buildGuid,
},
};
job.spec = {
ttlSecondsAfterFinished: 9999,
backoffLimit: 0,
template: {
spec: {
volumes: [
{
name: 'build-mount',
persistentVolumeClaim: {
claimName: pvcName,
},
},
],
containers: [
{
ttlSecondsAfterFinished: 9999,
name: containerName,
image,
command: ['/bin/sh'],
args: [
'-c',
`${CommandHookService.ApplyHooksToCommands(`${command}\nsleep 2m`, CloudRunner.buildParameters)}`,
],
workingDir: `${workingDirectory}`,
resources: {
requests: {
memory: `${Number.parseInt(buildParameters.containerMemory) / 1024}G` || '750M',
cpu: Number.parseInt(buildParameters.containerCpu) / 1024 || '1',
},
},
env: [
...environment.map((x) => {
const environmentVariable = new V1EnvVar();
environmentVariable.name = x.name;
environmentVariable.value = x.value;
return environmentVariable;
}),
...secrets.map((x) => {
const secret = new V1EnvVarSource();
secret.secretKeyRef = new V1SecretKeySelector();
secret.secretKeyRef.key = x.ParameterKey;
secret.secretKeyRef.name = secretName;
const environmentVariable = new V1EnvVar();
environmentVariable.name = x.EnvironmentVariable;
environmentVariable.valueFrom = secret;
return environmentVariable;
}),
{ name: 'LOG_SERVICE_IP', value: ip },
],
volumeMounts: [
{
name: 'build-mount',
mountPath: `${mountdir}`,
},
],
lifecycle: {
preStop: {
exec: {
command: [
`wait 60s;
cd /data/builder/action/steps;
chmod +x /return_license.sh;
/return_license.sh;`,
],
},
},
},
},
],
restartPolicy: 'Never',
},
},
};
if (process.env['CLOUD_RUNNER_MINIKUBE']) {
job.spec.template.spec.volumes[0] = {
name: 'build-mount',
hostPath: {
path: `/data`,
type: `Directory`,
},
};
}
job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '10Gi';
return job;
}
}
export default KubernetesJobSpecFactory;

View File

@@ -0,0 +1,23 @@
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { CoreV1Api } from '@kubernetes/client-node';
class KubernetesPods {
public static async IsPodRunning(podName: string, namespace: string, kubeClient: CoreV1Api) {
const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.filter((x) => podName === x.metadata?.name);
const running = pods.length > 0 && (pods[0].status?.phase === `Running` || pods[0].status?.phase === `Pending`);
const phase = pods[0]?.status?.phase || 'undefined status';
CloudRunnerLogger.log(`Getting pod status: ${phase}`);
if (phase === `Failed`) {
throw new Error(`K8s pod failed`);
}
return running;
}
public static async GetPodStatus(podName: string, namespace: string, kubeClient: CoreV1Api) {
const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.find((x) => podName === x.metadata?.name);
const phase = pods?.status?.phase || 'undefined status';
return phase;
}
}
export default KubernetesPods;

View File

@@ -1,12 +1,12 @@
import { CoreV1Api } from '@kubernetes/client-node';
import OrchestratorSecret from '../../options/orchestrator-secret';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import * as k8s from '@kubernetes/client-node';
import OrchestratorLogger from '../../services/core/orchestrator-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import * as base64 from 'base-64';
class KubernetesSecret {
static async createSecret(
secrets: OrchestratorSecret[],
secrets: CloudRunnerSecret[],
secretName: string,
namespace: string,
kubeClient: CoreV1Api,
@@ -23,20 +23,20 @@ class KubernetesSecret {
for (const buildSecret of secrets) {
secret.data[buildSecret.ParameterKey] = base64.encode(buildSecret.ParameterValue);
}
OrchestratorLogger.log(`Creating secret: ${secretName}`);
CloudRunnerLogger.log(`Creating secret: ${secretName}`);
const existingSecrets = await kubeClient.listNamespacedSecret(namespace);
const mappedSecrets = existingSecrets.body.items.map((x) => {
return x.metadata?.name || `no name`;
});
OrchestratorLogger.log(
CloudRunnerLogger.log(
`ExistsAlready: ${mappedSecrets.includes(secretName)} SecretsCount: ${mappedSecrets.length}`,
);
await new Promise((promise) => setTimeout(promise, 15000));
await kubeClient.createNamespacedSecret(namespace, secret);
OrchestratorLogger.log('Created secret');
CloudRunnerLogger.log('Created secret');
} catch (error) {
OrchestratorLogger.log(`Created secret failed ${error}`);
CloudRunnerLogger.log(`Created secret failed ${error}`);
throw new Error(`Failed to create kubernetes secret`);
}
}

View File

@@ -0,0 +1,116 @@
import { waitUntil } from 'async-wait-until';
import * as core from '@actions/core';
import * as k8s from '@kubernetes/client-node';
import BuildParameters from '../../../build-parameters';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { IncomingMessage } from 'node:http';
import GitHub from '../../../github';
class KubernetesStorage {
public static async createPersistentVolumeClaim(
buildParameters: BuildParameters,
pvcName: string,
kubeClient: k8s.CoreV1Api,
namespace: string,
) {
if (buildParameters.kubeVolume !== ``) {
CloudRunnerLogger.log(`Kube Volume was input was set ${buildParameters.kubeVolume} overriding ${pvcName}`);
pvcName = buildParameters.kubeVolume;
return;
}
const allPvc = (await kubeClient.listNamespacedPersistentVolumeClaim(namespace)).body.items;
const pvcList = allPvc.map((x) => x.metadata?.name);
CloudRunnerLogger.log(`Current PVCs in namespace ${namespace}`);
CloudRunnerLogger.log(JSON.stringify(pvcList, undefined, 4));
if (pvcList.includes(pvcName)) {
CloudRunnerLogger.log(`pvc ${pvcName} already exists`);
if (GitHub.githubInputEnabled) {
core.setOutput('volume', pvcName);
}
return;
}
CloudRunnerLogger.log(`Creating PVC ${pvcName} (does not exist)`);
const result = await KubernetesStorage.createPVC(pvcName, buildParameters, kubeClient, namespace);
await KubernetesStorage.handleResult(result, kubeClient, namespace, pvcName);
}
public static async getPVCPhase(kubeClient: k8s.CoreV1Api, name: string, namespace: string) {
try {
return (await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace)).body.status?.phase;
} catch (error) {
core.error('Failed to get PVC phase');
core.error(JSON.stringify(error, undefined, 4));
throw error;
}
}
public static async watchUntilPVCNotPending(kubeClient: k8s.CoreV1Api, name: string, namespace: string) {
try {
CloudRunnerLogger.log(`watch Until PVC Not Pending ${name} ${namespace}`);
CloudRunnerLogger.log(`${await this.getPVCPhase(kubeClient, name, namespace)}`);
await waitUntil(
async () => {
return (await this.getPVCPhase(kubeClient, name, namespace)) === 'Pending';
},
{
timeout: 750000,
intervalBetweenAttempts: 15000,
},
);
} catch (error: any) {
core.error('Failed to watch PVC');
core.error(error.toString());
core.error(
`PVC Body: ${JSON.stringify(
(await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace)).body,
undefined,
4,
)}`,
);
throw error;
}
}
private static async createPVC(
pvcName: string,
buildParameters: BuildParameters,
kubeClient: k8s.CoreV1Api,
namespace: string,
) {
const pvc = new k8s.V1PersistentVolumeClaim();
pvc.apiVersion = 'v1';
pvc.kind = 'PersistentVolumeClaim';
pvc.metadata = {
name: pvcName,
};
pvc.spec = {
accessModes: ['ReadWriteOnce'],
storageClassName: buildParameters.kubeStorageClass === '' ? 'standard' : buildParameters.kubeStorageClass,
resources: {
requests: {
storage: buildParameters.kubeVolumeSize,
},
},
};
const result = await kubeClient.createNamespacedPersistentVolumeClaim(namespace, pvc);
return result;
}
private static async handleResult(
result: { response: IncomingMessage; body: k8s.V1PersistentVolumeClaim },
kubeClient: k8s.CoreV1Api,
namespace: string,
pvcName: string,
) {
const name = result.body.metadata?.name || '';
CloudRunnerLogger.log(`PVC ${name} created`);
await this.watchUntilPVCNotPending(kubeClient, name, namespace);
CloudRunnerLogger.log(`PVC ${name} is ready and not pending`);
core.setOutput('volume', pvcName);
}
}
export default KubernetesStorage;

View File

@@ -0,0 +1,119 @@
import { CoreV1Api, KubeConfig } from '@kubernetes/client-node';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { waitUntil } from 'async-wait-until';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
import CloudRunner from '../../cloud-runner';
import KubernetesPods from './kubernetes-pods';
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
class KubernetesTaskRunner {
static readonly maxRetry: number = 3;
static lastReceivedMessage: string = ``;
static async runTask(
kubeConfig: KubeConfig,
kubeClient: CoreV1Api,
jobName: string,
podName: string,
containerName: string,
namespace: string,
) {
let output = '';
let shouldReadLogs = true;
let shouldCleanup = true;
let retriesAfterFinish = 0;
// eslint-disable-next-line no-constant-condition
while (true) {
await new Promise((resolve) => setTimeout(resolve, 3000));
CloudRunnerLogger.log(
`Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace} ${CloudRunner.buildParameters.kubeVolumeSize}/${CloudRunner.buildParameters.containerCpu}/${CloudRunner.buildParameters.containerMemory}`,
);
let extraFlags = ``;
extraFlags += (await KubernetesPods.IsPodRunning(podName, namespace, kubeClient))
? ` -f -c ${containerName}`
: ` --previous`;
const callback = (outputChunk: string) => {
output += outputChunk;
// split output chunk and handle per line
for (const chunk of outputChunk.split(`\n`)) {
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
chunk,
shouldReadLogs,
shouldCleanup,
output,
));
}
};
try {
await CloudRunnerSystem.Run(`kubectl logs ${podName}${extraFlags}`, false, true, callback);
} catch (error: any) {
await new Promise((resolve) => setTimeout(resolve, 3000));
const continueStreaming = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient);
CloudRunnerLogger.log(`K8s logging error ${error} ${continueStreaming}`);
if (continueStreaming) {
continue;
}
if (retriesAfterFinish < KubernetesTaskRunner.maxRetry) {
retriesAfterFinish++;
continue;
}
throw error;
}
if (FollowLogStreamService.DidReceiveEndOfTransmission) {
CloudRunnerLogger.log('end of log stream');
break;
}
}
return output;
}
static async watchUntilPodRunning(kubeClient: CoreV1Api, podName: string, namespace: string) {
let waitComplete: boolean = false;
let message = ``;
CloudRunnerLogger.log(`Watching ${podName} ${namespace}`);
await waitUntil(
async () => {
const status = await kubeClient.readNamespacedPodStatus(podName, namespace);
const phase = status?.body.status?.phase;
waitComplete = phase !== 'Pending';
message = `Phase:${status.body.status?.phase} \n Reason:${
status.body.status?.conditions?.[0].reason || ''
} \n Message:${status.body.status?.conditions?.[0].message || ''}`;
// CloudRunnerLogger.log(
// JSON.stringify(
// (await kubeClient.listNamespacedEvent(namespace)).body.items
// .map((x) => {
// return {
// message: x.message || ``,
// name: x.metadata.name || ``,
// reason: x.reason || ``,
// };
// })
// .filter((x) => x.name.includes(podName)),
// undefined,
// 4,
// ),
// );
if (waitComplete || phase !== 'Pending') return true;
return false;
},
{
timeout: 2000000,
intervalBetweenAttempts: 15000,
},
);
if (!waitComplete) {
CloudRunnerLogger.log(message);
}
return waitComplete;
}
}
export default KubernetesTaskRunner;

View File

@@ -1,14 +1,13 @@
import BuildParameters from '../../../build-parameters';
import { OrchestratorSystem } from '../../services/core/orchestrator-system';
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
import OrchestratorLogger from '../../services/core/orchestrator-logger';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { ProviderInterface } from '../provider-interface';
import OrchestratorSecret from '../../options/orchestrator-secret';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
import { quote } from 'shell-quote';
class LocalOrchestrator implements ProviderInterface {
class LocalCloudRunner implements ProviderInterface {
listResources(): Promise<ProviderResource[]> {
throw new Error('Method not implemented.');
}
@@ -59,29 +58,15 @@ class LocalOrchestrator implements ProviderInterface {
// eslint-disable-next-line no-unused-vars
workingdir: string,
// eslint-disable-next-line no-unused-vars
environment: OrchestratorEnvironmentVariable[],
environment: CloudRunnerEnvironmentVariable[],
// eslint-disable-next-line no-unused-vars
secrets: OrchestratorSecret[],
secrets: CloudRunnerSecret[],
): Promise<string> {
OrchestratorLogger.log(image);
OrchestratorLogger.log(buildGuid);
OrchestratorLogger.log(commands);
CloudRunnerLogger.log(image);
CloudRunnerLogger.log(buildGuid);
CloudRunnerLogger.log(commands);
// On Windows, many built-in hooks use POSIX shell syntax. Execute via bash if available.
if (process.platform === 'win32') {
const inline = commands
.replace(/\r/g, '')
.split('\n')
.filter((x) => x.trim().length > 0)
.join(' ; ');
// Use shell-quote to properly escape the command string, preventing command injection
const bashWrapped = `bash -lc ${quote([inline])}`;
return await OrchestratorSystem.Run(bashWrapped);
}
return await OrchestratorSystem.Run(commands);
return await CloudRunnerSystem.Run(commands);
}
}
export default LocalOrchestrator;
export default LocalCloudRunner;

View File

@@ -1,6 +1,6 @@
import BuildParameters from '../../build-parameters';
import OrchestratorEnvironmentVariable from '../options/orchestrator-environment-variable';
import OrchestratorSecret from '../options/orchestrator-secret';
import CloudRunnerEnvironmentVariable from '../options/cloud-runner-environment-variable';
import CloudRunnerSecret from '../options/cloud-runner-secret';
import { ProviderResource } from './provider-resource';
import { ProviderWorkflow } from './provider-workflow';
@@ -35,9 +35,9 @@ export interface ProviderInterface {
// eslint-disable-next-line no-unused-vars
workingdir: string,
// eslint-disable-next-line no-unused-vars
environment: OrchestratorEnvironmentVariable[],
environment: CloudRunnerEnvironmentVariable[],
// eslint-disable-next-line no-unused-vars
secrets: OrchestratorSecret[],
secrets: CloudRunnerSecret[],
): Promise<string>;
garbageCollect(
// eslint-disable-next-line no-unused-vars

View File

@@ -1,12 +1,12 @@
import BuildParameters from '../../../build-parameters';
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
import OrchestratorLogger from '../../services/core/orchestrator-logger';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { ProviderInterface } from '../provider-interface';
import OrchestratorSecret from '../../options/orchestrator-secret';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
class TestOrchestrator implements ProviderInterface {
class TestCloudRunner implements ProviderInterface {
listResources(): Promise<ProviderResource[]> {
throw new Error('Method not implemented.');
}
@@ -51,17 +51,17 @@ class TestOrchestrator implements ProviderInterface {
// eslint-disable-next-line no-unused-vars
workingdir: string,
// eslint-disable-next-line no-unused-vars
environment: OrchestratorEnvironmentVariable[],
environment: CloudRunnerEnvironmentVariable[],
// eslint-disable-next-line no-unused-vars
secrets: OrchestratorSecret[],
secrets: CloudRunnerSecret[],
): Promise<string> {
OrchestratorLogger.log(image);
OrchestratorLogger.log(buildGuid);
OrchestratorLogger.log(commands);
CloudRunnerLogger.log(image);
CloudRunnerLogger.log(buildGuid);
CloudRunnerLogger.log(commands);
return await new Promise((result) => {
result(commands);
});
}
}
export default TestOrchestrator;
export default TestCloudRunner;

View File

@@ -0,0 +1,181 @@
import { assert } from 'node:console';
import fs from 'node:fs';
import path from 'node:path';
import CloudRunner from '../cloud-runner';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import { CloudRunnerFolders } from '../options/cloud-runner-folders';
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
import { LfsHashing } from '../services/utility/lfs-hashing';
import { RemoteClientLogger } from './remote-client-logger';
import { Cli } from '../../cli/cli';
import { CliFunction } from '../../cli/cli-functions-repository';
// eslint-disable-next-line github/no-then
const fileExists = async (fpath: fs.PathLike) => !!(await fs.promises.stat(fpath).catch(() => false));
export class Caching {
@CliFunction(`cache-push`, `push to cache`)
static async cachePush() {
try {
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
CloudRunner.buildParameters = buildParameter;
await Caching.PushToCache(
Cli.options!['cachePushTo'],
Cli.options!['cachePushFrom'],
Cli.options!['artifactName'] || '',
);
} catch (error: any) {
CloudRunnerLogger.log(`${error}`);
}
}
@CliFunction(`cache-pull`, `pull from cache`)
static async cachePull() {
try {
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
CloudRunner.buildParameters = buildParameter;
await Caching.PullFromCache(
Cli.options!['cachePushFrom'],
Cli.options!['cachePushTo'],
Cli.options!['artifactName'] || '',
);
} catch (error: any) {
CloudRunnerLogger.log(`${error}`);
}
}
public static async PushToCache(cacheFolder: string, sourceFolder: string, cacheArtifactName: string) {
CloudRunnerLogger.log(`Pushing to cache ${sourceFolder}`);
cacheArtifactName = cacheArtifactName.replace(' ', '');
const startPath = process.cwd();
let compressionSuffix = '';
if (CloudRunner.buildParameters.useCompressionStrategy === true) {
compressionSuffix = `.lz4`;
}
CloudRunnerLogger.log(`Compression: ${CloudRunner.buildParameters.useCompressionStrategy} ${compressionSuffix}`);
try {
if (!(await fileExists(cacheFolder))) {
await CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
}
process.chdir(path.resolve(sourceFolder, '..'));
if (CloudRunner.buildParameters.cloudRunnerDebug === true) {
CloudRunnerLogger.log(
`Hashed cache folder ${await LfsHashing.hashAllFiles(sourceFolder)} ${sourceFolder} ${path.basename(
sourceFolder,
)}`,
);
}
const contents = await fs.promises.readdir(path.basename(sourceFolder));
CloudRunnerLogger.log(
`There is ${contents.length} files/dir in the source folder ${path.basename(sourceFolder)}`,
);
if (contents.length === 0) {
CloudRunnerLogger.log(
`Did not push source folder to cache because it was empty ${path.basename(sourceFolder)}`,
);
process.chdir(`${startPath}`);
return;
}
await CloudRunnerSystem.Run(
`tar -cf ${cacheArtifactName}.tar${compressionSuffix} "${path.basename(sourceFolder)}"`,
);
await CloudRunnerSystem.Run(`du ${cacheArtifactName}.tar${compressionSuffix}`);
assert(await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`), 'cache archive exists');
assert(await fileExists(path.basename(sourceFolder)), 'source folder exists');
await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar${compressionSuffix} ${cacheFolder}`);
RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`);
assert(
await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.tar${compressionSuffix}`),
'cache archive exists inside cache folder',
);
} catch (error) {
process.chdir(`${startPath}`);
throw error;
}
process.chdir(`${startPath}`);
}
public static async PullFromCache(cacheFolder: string, destinationFolder: string, cacheArtifactName: string = ``) {
CloudRunnerLogger.log(`Pulling from cache ${destinationFolder} ${CloudRunner.buildParameters.skipCache}`);
if (`${CloudRunner.buildParameters.skipCache}` === `true`) {
CloudRunnerLogger.log(`Skipping cache debugSkipCache is true`);
return;
}
cacheArtifactName = cacheArtifactName.replace(' ', '');
let compressionSuffix = '';
if (CloudRunner.buildParameters.useCompressionStrategy === true) {
compressionSuffix = `.lz4`;
}
const startPath = process.cwd();
RemoteClientLogger.log(`Caching for (lz4 ${compressionSuffix}) ${path.basename(destinationFolder)}`);
try {
if (!(await fileExists(cacheFolder))) {
await fs.promises.mkdir(cacheFolder);
}
if (!(await fileExists(destinationFolder))) {
await fs.promises.mkdir(destinationFolder);
}
const latestInBranch = await (
await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar${compressionSuffix}$ | head -1`)
)
.replace(/\n/g, ``)
.replace(`.tar${compressionSuffix}`, '');
process.chdir(cacheFolder);
const cacheSelection =
cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`))
? cacheArtifactName
: latestInBranch;
await CloudRunnerLogger.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`);
if (await fileExists(`${cacheSelection}.tar${compressionSuffix}`)) {
const resultsFolder = `results${CloudRunner.buildParameters.buildGuid}`;
await CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`);
RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar${compressionSuffix}`);
const fullResultsFolder = path.join(cacheFolder, resultsFolder);
await CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar${compressionSuffix} -C ${fullResultsFolder}`);
RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
assert(await fileExists(fullResultsFolder), `cache extraction results folder exists`);
const destinationParentFolder = path.resolve(destinationFolder, '..');
if (await fileExists(destinationFolder)) {
await fs.promises.rmdir(destinationFolder, { recursive: true });
}
await CloudRunnerSystem.Run(
`mv "${path.join(fullResultsFolder, path.basename(destinationFolder))}" "${destinationParentFolder}"`,
);
const contents = await fs.promises.readdir(
path.join(destinationParentFolder, path.basename(destinationFolder)),
);
CloudRunnerLogger.log(
`There is ${contents.length} files/dir in the cache pulled contents for ${path.basename(destinationFolder)}`,
);
} else {
RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`);
if (cacheSelection !== ``) {
RemoteClientLogger.logWarning(
`cache item ${cacheArtifactName}.tar${compressionSuffix} doesn't exist ${destinationFolder}`,
);
throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
}
}
} catch (error) {
process.chdir(startPath);
throw error;
}
process.chdir(startPath);
}
public static async handleCachePurging() {
if (process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined) {
RemoteClientLogger.log(`purging ${CloudRunnerFolders.purgeRemoteCaching}`);
fs.promises.rmdir(CloudRunnerFolders.cacheFolder, { recursive: true });
}
}
}

View File

@@ -0,0 +1,251 @@
import fs from 'node:fs';
import CloudRunner from '../cloud-runner';
import { CloudRunnerFolders } from '../options/cloud-runner-folders';
import { Caching } from './caching';
import { LfsHashing } from '../services/utility/lfs-hashing';
import { RemoteClientLogger } from './remote-client-logger';
import path from 'node:path';
import { assert } from 'node:console';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import { CliFunction } from '../../cli/cli-functions-repository';
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
import YAML from 'yaml';
import GitHub from '../../github';
import BuildParameters from '../../build-parameters';
import { Cli } from '../../cli/cli';
import CloudRunnerOptions from '../options/cloud-runner-options';
export class RemoteClient {
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
static async setupRemoteClient() {
CloudRunnerLogger.log(`bootstrap game ci cloud runner...`);
if (!(await RemoteClient.handleRetainedWorkspace())) {
await RemoteClient.bootstrapRepository();
}
await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
await RemoteClient.runCustomHookFiles(`before-build`);
}
@CliFunction('remote-cli-log-stream', `log stream from standard input`)
public static async remoteClientLogStream() {
const logFile = Cli.options!['logFile'];
process.stdin.resume();
process.stdin.setEncoding('utf8');
let lingeringLine = '';
process.stdin.on('data', (chunk) => {
const lines = chunk.toString().split('\n');
lines[0] = lingeringLine + lines[0];
lingeringLine = lines.pop() || '';
for (const element of lines) {
if (CloudRunnerOptions.providerStrategy !== 'k8s') {
CloudRunnerLogger.log(element);
} else {
fs.appendFileSync(logFile, element);
CloudRunnerLogger.log(element);
}
}
});
process.stdin.on('end', () => {
if (CloudRunnerOptions.providerStrategy !== 'k8s') {
CloudRunnerLogger.log(lingeringLine);
} else {
fs.appendFileSync(logFile, lingeringLine);
CloudRunnerLogger.log(lingeringLine);
}
});
}
@CliFunction(`remote-cli-post-build`, `runs a cloud runner build`)
public static async remoteClientPostBuild(): Promise<string> {
RemoteClientLogger.log(`Running POST build tasks`);
await Caching.PushToCache(
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/Library`),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
`lib-${CloudRunner.buildParameters.buildGuid}`,
);
await Caching.PushToCache(
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/build`),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.projectBuildFolderAbsolute),
`build-${CloudRunner.buildParameters.buildGuid}`,
);
if (!BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)) {
await CloudRunnerSystem.Run(
`rm -r ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
);
}
await RemoteClient.runCustomHookFiles(`after-build`);
// WIP - need to give the pod permissions to create config map
await RemoteClientLogger.handleLogManagementPostJob();
return new Promise((result) => result(``));
}
static async runCustomHookFiles(hookLifecycle: string) {
RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
const gameCiCustomHooksPath = path.join(CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`);
try {
const files = fs.readdirSync(gameCiCustomHooksPath);
for (const file of files) {
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
const fileContentsObject = YAML.parse(fileContents.toString());
if (fileContentsObject.hook === hookLifecycle) {
RemoteClientLogger.log(`Active Hook File ${file} \n \n file contents: \n ${fileContents}`);
await CloudRunnerSystem.Run(fileContentsObject.commands);
}
}
} catch (error) {
RemoteClientLogger.log(JSON.stringify(error, undefined, 4));
}
}
public static async bootstrapRepository() {
await CloudRunnerSystem.Run(
`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
);
await CloudRunnerSystem.Run(
`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForCacheKeyFull)}`,
);
await RemoteClient.cloneRepoWithoutLFSFiles();
await RemoteClient.sizeOfFolder(
'repo before lfs cache pull',
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute),
);
const lfsHashes = await LfsHashing.createLFSHashFiles();
if (fs.existsSync(CloudRunnerFolders.libraryFolderAbsolute)) {
RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`);
}
await Caching.PullFromCache(
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
`${lfsHashes.lfsGuidSum}`,
);
await RemoteClient.sizeOfFolder('repo after lfs cache pull', CloudRunnerFolders.repoPathAbsolute);
await RemoteClient.pullLatestLFS();
await RemoteClient.sizeOfFolder('repo before lfs git pull', CloudRunnerFolders.repoPathAbsolute);
await Caching.PushToCache(
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
`${lfsHashes.lfsGuidSum}`,
);
await Caching.PullFromCache(
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryCacheFolderFull),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
);
await RemoteClient.sizeOfFolder('repo after library cache pull', CloudRunnerFolders.repoPathAbsolute);
await Caching.handleCachePurging();
}
private static async sizeOfFolder(message: string, folder: string) {
if (CloudRunner.buildParameters.cloudRunnerDebug) {
CloudRunnerLogger.log(`Size of ${message}`);
await CloudRunnerSystem.Run(`du -sh ${folder}`);
}
}
private static async cloneRepoWithoutLFSFiles() {
process.chdir(`${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
if (
fs.existsSync(CloudRunnerFolders.repoPathAbsolute) &&
!fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))
) {
await CloudRunnerSystem.Run(`rm -r ${CloudRunnerFolders.repoPathAbsolute}`);
CloudRunnerLogger.log(`${CloudRunnerFolders.repoPathAbsolute} repo exists, but no git folder, cleaning up`);
}
if (
BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters) &&
fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))
) {
process.chdir(CloudRunnerFolders.repoPathAbsolute);
RemoteClientLogger.log(
`${
CloudRunnerFolders.repoPathAbsolute
} repo exists - skipping clone - retained workspace mode ${BuildParameters.shouldUseRetainedWorkspaceMode(
CloudRunner.buildParameters,
)}`,
);
await CloudRunnerSystem.Run(`git fetch && git reset --hard ${CloudRunner.buildParameters.gitSha}`);
return;
}
RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
RemoteClientLogger.log(`Cloning the repository being built:`);
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f"`);
await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process --skip"`);
try {
await CloudRunnerSystem.Run(
`git clone ${CloudRunnerFolders.targetBuildRepoUrl} ${path.basename(CloudRunnerFolders.repoPathAbsolute)}`,
);
} catch (error: any) {
throw error;
}
process.chdir(CloudRunnerFolders.repoPathAbsolute);
await CloudRunnerSystem.Run(`git lfs install`);
assert(fs.existsSync(`.git`), 'git folder exists');
RemoteClientLogger.log(`${CloudRunner.buildParameters.branch}`);
if (CloudRunner.buildParameters.gitSha !== undefined) {
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.gitSha}`);
} else {
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.branch}`);
RemoteClientLogger.log(`buildParameter Git Sha is empty`);
}
assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
RemoteClientLogger.log(`Checked out ${CloudRunner.buildParameters.branch}`);
}
static async replaceLargePackageReferencesWithSharedReferences() {
CloudRunnerLogger.log(`Use Shared Pkgs ${CloudRunner.buildParameters.useLargePackages}`);
GitHub.updateGitHubCheck(`Use Shared Pkgs ${CloudRunner.buildParameters.useLargePackages}`, ``);
if (CloudRunner.buildParameters.useLargePackages) {
const filePath = path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`);
let manifest = fs.readFileSync(filePath, 'utf8');
manifest = manifest.replace(/LargeContent/g, '../../../LargeContent');
fs.writeFileSync(filePath, manifest);
CloudRunnerLogger.log(`Package Manifest \n ${manifest}`);
GitHub.updateGitHubCheck(`Package Manifest \n ${manifest}`, ``);
}
}
private static async pullLatestLFS() {
process.chdir(CloudRunnerFolders.repoPathAbsolute);
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge -- %f"`);
await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process"`);
if (!CloudRunner.buildParameters.skipLfs) {
await CloudRunnerSystem.Run(`git lfs pull`);
RemoteClientLogger.log(`pulled latest LFS files`);
assert(fs.existsSync(CloudRunnerFolders.lfsFolderAbsolute));
}
}
static async handleRetainedWorkspace() {
RemoteClientLogger.log(
`Retained Workspace: ${BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)}`,
);
if (
BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters) &&
fs.existsSync(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)) &&
fs.existsSync(CloudRunnerFolders.ToLinuxFolder(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`)))
) {
CloudRunnerLogger.log(`Retained Workspace Already Exists!`);
process.chdir(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute));
await CloudRunnerSystem.Run(`git fetch`);
await CloudRunnerSystem.Run(`git lfs pull`);
await CloudRunnerSystem.Run(`git reset --hard "${CloudRunner.buildParameters.gitSha}"`);
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.gitSha}`);
return true;
}
return false;
}
}

View File

@@ -0,0 +1,82 @@
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import fs from 'node:fs';
import path from 'node:path';
import CloudRunner from '../cloud-runner';
import CloudRunnerOptions from '../options/cloud-runner-options';
export class RemoteClientLogger {
private static get LogFilePath() {
return path.join(`/home`, `job-log.txt`);
}
public static log(message: string) {
const finalMessage = `[Client] ${message}`;
this.appendToFile(finalMessage);
CloudRunnerLogger.log(finalMessage);
}
public static logCliError(message: string) {
CloudRunnerLogger.log(`[Client][Error] ${message}`);
}
public static logCliDiagnostic(message: string) {
CloudRunnerLogger.log(`[Client][Diagnostic] ${message}`);
}
public static logWarning(message: string) {
CloudRunnerLogger.logWarning(message);
}
public static appendToFile(message: string) {
if (CloudRunner.isCloudRunnerEnvironment) {
fs.appendFileSync(RemoteClientLogger.LogFilePath, `${message}\n`);
}
}
public static async handleLogManagementPostJob() {
if (CloudRunnerOptions.providerStrategy !== 'k8s') {
return;
}
CloudRunnerLogger.log(`Collected Logs`);
// check for log file not existing
if (!fs.existsSync(RemoteClientLogger.LogFilePath)) {
CloudRunnerLogger.log(`Log file does not exist`);
// check if CloudRunner.isCloudRunnerEnvironment is true, log
if (!CloudRunner.isCloudRunnerEnvironment) {
CloudRunnerLogger.log(`Cloud Runner is not running in a cloud environment, not collecting logs`);
}
return;
}
CloudRunnerLogger.log(`Log file exist`);
await new Promise((resolve) => setTimeout(resolve, 1));
// let hashedLogs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
//
// hashedLogs = md5(hashedLogs);
//
// for (let index = 0; index < 3; index++) {
// CloudRunnerLogger.log(`LOGHASH: ${hashedLogs}`);
// const logs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
// CloudRunnerLogger.log(`LOGS: ${Buffer.from(logs).toString('base64')}`);
// CloudRunnerLogger.log(
// `Game CI's "Cloud Runner System" will cancel the log when it has successfully received the log data to verify all logs have been received.`,
// );
//
// // wait for 15 seconds to allow the log to be sent
// await new Promise((resolve) => setTimeout(resolve, 15000));
// }
}
public static HandleLog(message: string): boolean {
if (RemoteClientLogger.value !== '') {
RemoteClientLogger.value += `\n`;
}
RemoteClientLogger.value += message;
return false;
}
static value: string = '';
}

View File

@@ -1,6 +1,6 @@
import * as core from '@actions/core';
class OrchestratorLogger {
class CloudRunnerLogger {
private static timestamp: number;
private static globalTimestamp: number;
@@ -44,4 +44,4 @@ class OrchestratorLogger {
return Date.now();
}
}
export default OrchestratorLogger;
export default CloudRunnerLogger;

View File

@@ -1,6 +1,6 @@
import BuildParameters from '../../../build-parameters';
class OrchestratorResult {
class CloudRunnerResult {
public BuildParameters: BuildParameters;
public BuildResults: string;
public BuildSucceeded: boolean;
@@ -21,4 +21,4 @@ class OrchestratorResult {
this.LibraryCacheUsed = libraryCacheUsed;
}
}
export default OrchestratorResult;
export default CloudRunnerResult;

View File

@@ -1,9 +1,9 @@
import { exec } from 'child_process';
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
export class OrchestratorSystem {
export class CloudRunnerSystem {
public static async RunAndReadLines(command: string): Promise<string[]> {
const result = await OrchestratorSystem.Run(command, false, true);
const result = await CloudRunnerSystem.Run(command, false, true);
return result
.split(`\n`)

View File

@@ -1,7 +1,7 @@
import GitHub from '../../../github';
import Orchestrator from '../../orchestrator';
import { OrchestratorStatics } from '../../options/orchestrator-statics';
import OrchestratorLogger from './orchestrator-logger';
import CloudRunner from '../../cloud-runner';
import { CloudRunnerStatics } from '../../options/cloud-runner-statics';
import CloudRunnerLogger from './cloud-runner-logger';
import * as core from '@actions/core';
export class FollowLogStreamService {
@@ -11,8 +11,8 @@ export class FollowLogStreamService {
static errors = ``;
public static DidReceiveEndOfTransmission = false;
public static handleIteration(message: string, shouldReadLogs: boolean, shouldCleanup: boolean, output: string) {
if (message.includes(`---${Orchestrator.buildParameters.logId}`)) {
OrchestratorLogger.log('End of log transmission received');
if (message.includes(`---${CloudRunner.buildParameters.logId}`)) {
CloudRunnerLogger.log('End of log transmission received');
FollowLogStreamService.DidReceiveEndOfTransmission = true;
shouldReadLogs = false;
} else if (message.includes('Rebuilding Library because the asset database could not be found!')) {
@@ -47,10 +47,10 @@ export class FollowLogStreamService {
} else if (message.toLowerCase().includes('cannot be found')) {
FollowLogStreamService.errors += `\n${message}`;
}
// Always append log lines to output so tests can assert on BuildResults
output += `${message}\n`;
OrchestratorLogger.log(`[${OrchestratorStatics.logPrefix}] ${message}`);
if (CloudRunner.buildParameters.cloudRunnerDebug) {
output += `${message}\n`;
}
CloudRunnerLogger.log(`[${CloudRunnerStatics.logPrefix}] ${message}`);
return { shouldReadLogs, shouldCleanup, output };
}

View File

@@ -1,112 +1,23 @@
import OrchestratorLogger from './orchestrator-logger';
import { CloudRunnerSystem } from './cloud-runner-system';
import fs from 'node:fs';
import CloudRunnerLogger from './cloud-runner-logger';
import BuildParameters from '../../../build-parameters';
import Orchestrator from '../../orchestrator';
import Input from '../../../input';
import {
CreateBucketCommand,
DeleteObjectCommand,
HeadBucketCommand,
ListObjectsV2Command,
PutObjectCommand,
S3,
} from '@aws-sdk/client-s3';
import { AwsClientFactory } from '../../providers/aws/aws-client-factory';
import { promisify } from 'node:util';
import { exec as execCallback } from 'node:child_process';
const exec = promisify(execCallback);
import CloudRunner from '../../cloud-runner';
export class SharedWorkspaceLocking {
private static _s3: S3;
private static get s3(): S3 {
if (!SharedWorkspaceLocking._s3) {
// Use factory so LocalStack endpoint/path-style settings are honored
SharedWorkspaceLocking._s3 = AwsClientFactory.getS3();
}
return SharedWorkspaceLocking._s3;
}
private static get useRclone() {
return Orchestrator.buildParameters.storageProvider === 'rclone';
}
private static async rclone(command: string): Promise<string> {
const { stdout } = await exec(`rclone ${command}`);
return stdout.toString();
}
private static get bucket() {
return SharedWorkspaceLocking.useRclone
? Orchestrator.buildParameters.rcloneRemote
: Orchestrator.buildParameters.awsStackName;
}
public static get workspaceBucketRoot() {
return SharedWorkspaceLocking.useRclone
? `${SharedWorkspaceLocking.bucket}/`
: `s3://${SharedWorkspaceLocking.bucket}/`;
return `s3://${CloudRunner.buildParameters.awsStackName}/`;
}
public static get workspaceRoot() {
return `${SharedWorkspaceLocking.workspaceBucketRoot}locks/`;
}
private static get workspacePrefix() {
return `locks/`;
}
private static async ensureBucketExists(): Promise<void> {
const bucket = SharedWorkspaceLocking.bucket;
if (SharedWorkspaceLocking.useRclone) {
try {
await SharedWorkspaceLocking.rclone(`lsf ${bucket}`);
} catch {
await SharedWorkspaceLocking.rclone(`mkdir ${bucket}`);
}
return;
}
try {
await SharedWorkspaceLocking.s3.send(new HeadBucketCommand({ Bucket: bucket }));
} catch {
const region = Input.region || process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || 'us-east-1';
const createParameters: any = { Bucket: bucket };
if (region && region !== 'us-east-1') {
createParameters.CreateBucketConfiguration = { LocationConstraint: region };
}
await SharedWorkspaceLocking.s3.send(new CreateBucketCommand(createParameters));
}
}
private static async listObjects(prefix: string, bucket = SharedWorkspaceLocking.bucket): Promise<string[]> {
await SharedWorkspaceLocking.ensureBucketExists();
if (prefix !== '' && !prefix.endsWith('/')) {
prefix += '/';
}
if (SharedWorkspaceLocking.useRclone) {
const path = `${bucket}/${prefix}`;
try {
const output = await SharedWorkspaceLocking.rclone(`lsjson ${path}`);
const json = JSON.parse(output) as { Name: string; IsDir: boolean }[];
return json.map((entry) => (entry.IsDir ? `${entry.Name}/` : entry.Name));
} catch {
return [];
}
}
const result = await SharedWorkspaceLocking.s3.send(
new ListObjectsV2Command({ Bucket: bucket, Prefix: prefix, Delimiter: '/' }),
);
const entries: string[] = [];
for (const p of result.CommonPrefixes || []) {
if (p.Prefix) entries.push(p.Prefix.slice(prefix.length));
}
for (const c of result.Contents || []) {
if (c.Key && c.Key !== prefix) entries.push(c.Key.slice(prefix.length));
}
return entries;
}
public static async GetAllWorkspaces(buildParametersContext: BuildParameters): Promise<string[]> {
if (!(await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParametersContext))) {
return [];
}
return (
await SharedWorkspaceLocking.listObjects(
`${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/`,
await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
)
)
.map((x) => x.replace(`/`, ``))
@@ -115,11 +26,13 @@ export class SharedWorkspaceLocking {
}
public static async DoesCacheKeyTopLevelExist(buildParametersContext: BuildParameters) {
try {
const rootLines = await SharedWorkspaceLocking.listObjects('');
const rootLines = await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceBucketRoot}`,
);
const lockFolderExists = rootLines.map((x) => x.replace(`/`, ``)).includes(`locks`);
if (lockFolderExists) {
const lines = await SharedWorkspaceLocking.listObjects(SharedWorkspaceLocking.workspacePrefix);
const lines = await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`);
return lines.map((x) => x.replace(`/`, ``)).includes(buildParametersContext.cacheKey);
} else {
@@ -131,7 +44,7 @@ export class SharedWorkspaceLocking {
}
public static NewWorkspaceName() {
return `${Orchestrator.retainedWorkspacePrefix}-${Orchestrator.buildParameters.buildGuid}`;
return `${CloudRunner.retainedWorkspacePrefix}-${CloudRunner.buildParameters.buildGuid}`;
}
public static async GetAllLocksForWorkspace(
workspace: string,
@@ -142,8 +55,8 @@ export class SharedWorkspaceLocking {
}
return (
await SharedWorkspaceLocking.listObjects(
`${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/`,
await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
)
)
.map((x) => x.replace(`/`, ``))
@@ -156,10 +69,10 @@ export class SharedWorkspaceLocking {
if (await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParametersContext)) {
const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
OrchestratorLogger.log(`run agent ${runId} is trying to access a workspace, free: ${JSON.stringify(workspaces)}`);
CloudRunnerLogger.log(`run agent ${runId} is trying to access a workspace, free: ${JSON.stringify(workspaces)}`);
for (const element of workspaces) {
const lockResult = await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext);
OrchestratorLogger.log(
CloudRunnerLogger.log(
`run agent: ${runId} try lock workspace: ${element} locking attempt result: ${lockResult}`,
);
@@ -171,12 +84,12 @@ export class SharedWorkspaceLocking {
if (await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext)) {
workspace = SharedWorkspaceLocking.NewWorkspaceName();
Orchestrator.lockedWorkspace = workspace;
CloudRunner.lockedWorkspace = workspace;
}
const createResult = await SharedWorkspaceLocking.CreateWorkspace(workspace, buildParametersContext);
const lockResult = await SharedWorkspaceLocking.LockWorkspace(workspace, runId, buildParametersContext);
OrchestratorLogger.log(
CloudRunnerLogger.log(
`run agent ${runId} didn't find a free workspace so created: ${workspace} createWorkspaceSuccess: ${createResult} Lock:${lockResult}`,
);
@@ -204,7 +117,7 @@ export class SharedWorkspaceLocking {
.sort((x) => x.timestamp);
const lockMatches = locks.filter((x) => x.name.includes(runId));
const includesRunLock = lockMatches.length > 0 && locks.indexOf(lockMatches[0]) === 0;
OrchestratorLogger.log(
CloudRunnerLogger.log(
`Checking has workspace lock, runId: ${runId}, workspace: ${workspace}, success: ${includesRunLock} \n- Num of locks created by Run Agent: ${
lockMatches.length
} Num of Locks: ${locks.length}, Time ordered index for Run Agent: ${locks.indexOf(lockMatches[0])} \n \n`,
@@ -219,7 +132,7 @@ export class SharedWorkspaceLocking {
for (const element of workspaces) {
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext);
const isBelowMax = await SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParametersContext);
OrchestratorLogger.log(`workspace ${element} locked:${isLocked} below max:${isBelowMax}`);
CloudRunnerLogger.log(`workspace ${element} locked:${isLocked} below max:${isBelowMax}`);
if (!isLocked && isBelowMax) {
result.push(element);
}
@@ -269,8 +182,8 @@ export class SharedWorkspaceLocking {
}
return (
await SharedWorkspaceLocking.listObjects(
`${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/`,
await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
)
)
.map((x) => x.replace(`/`, ``))
@@ -282,8 +195,8 @@ export class SharedWorkspaceLocking {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
throw new Error(`workspace doesn't exist ${workspace}`);
}
const files = await SharedWorkspaceLocking.listObjects(
`${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/`,
const files = await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
);
const lockFilesExist =
@@ -299,19 +212,20 @@ export class SharedWorkspaceLocking {
throw new Error(`${workspace} already exists`);
}
const timestamp = Date.now();
const key = `${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/${timestamp}_${workspace}_workspace`;
await SharedWorkspaceLocking.ensureBucketExists();
await (SharedWorkspaceLocking.useRclone
? SharedWorkspaceLocking.rclone(`touch ${SharedWorkspaceLocking.bucket}/${key}`)
: SharedWorkspaceLocking.s3.send(
new PutObjectCommand({ Bucket: SharedWorkspaceLocking.bucket, Key: key, Body: new Uint8Array(0) }),
));
const file = `${timestamp}_${workspace}_workspace`;
fs.writeFileSync(file, '');
await CloudRunnerSystem.Run(
`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
false,
true,
);
fs.rmSync(file);
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
OrchestratorLogger.log(`All workspaces ${workspaces}`);
CloudRunnerLogger.log(`All workspaces ${workspaces}`);
if (!(await SharedWorkspaceLocking.IsWorkspaceBelowMax(workspace, buildParametersContext))) {
OrchestratorLogger.log(`Workspace is above max ${workspaces} ${buildParametersContext.maxRetainedWorkspaces}`);
CloudRunnerLogger.log(`Workspace is above max ${workspaces} ${buildParametersContext.maxRetainedWorkspaces}`);
await SharedWorkspaceLocking.CleanupWorkspace(workspace, buildParametersContext);
return false;
@@ -327,24 +241,25 @@ export class SharedWorkspaceLocking {
): Promise<boolean> {
const existingWorkspace = workspace.endsWith(`_workspace`);
const ending = existingWorkspace ? workspace : `${workspace}_workspace`;
const key = `${SharedWorkspaceLocking.workspacePrefix}${
buildParametersContext.cacheKey
}/${Date.now()}_${runId}_${ending}_lock`;
await SharedWorkspaceLocking.ensureBucketExists();
await (SharedWorkspaceLocking.useRclone
? SharedWorkspaceLocking.rclone(`touch ${SharedWorkspaceLocking.bucket}/${key}`)
: SharedWorkspaceLocking.s3.send(
new PutObjectCommand({ Bucket: SharedWorkspaceLocking.bucket, Key: key, Body: new Uint8Array(0) }),
));
const file = `${Date.now()}_${runId}_${ending}_lock`;
fs.writeFileSync(file, '');
await CloudRunnerSystem.Run(
`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
false,
true,
);
fs.rmSync(file);
const hasLock = await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
if (hasLock) {
Orchestrator.lockedWorkspace = workspace;
CloudRunner.lockedWorkspace = workspace;
} else {
await (SharedWorkspaceLocking.useRclone
? SharedWorkspaceLocking.rclone(`delete ${SharedWorkspaceLocking.bucket}/${key}`)
: SharedWorkspaceLocking.s3.send(new DeleteObjectCommand({ Bucket: SharedWorkspaceLocking.bucket, Key: key })));
await CloudRunnerSystem.Run(
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
false,
true,
);
}
return hasLock;
@@ -355,47 +270,30 @@ export class SharedWorkspaceLocking {
runId: string,
buildParametersContext: BuildParameters,
): Promise<boolean> {
await SharedWorkspaceLocking.ensureBucketExists();
const files = await SharedWorkspaceLocking.GetAllLocksForWorkspace(workspace, buildParametersContext);
const file = files.find((x) => x.includes(workspace) && x.endsWith(`_lock`) && x.includes(runId));
OrchestratorLogger.log(`All Locks ${files} ${workspace} ${runId}`);
OrchestratorLogger.log(`Deleting lock ${workspace}/${file}`);
OrchestratorLogger.log(`rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`);
if (file) {
await (SharedWorkspaceLocking.useRclone
? SharedWorkspaceLocking.rclone(
`delete ${SharedWorkspaceLocking.bucket}/${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/${file}`,
)
: SharedWorkspaceLocking.s3.send(
new DeleteObjectCommand({
Bucket: SharedWorkspaceLocking.bucket,
Key: `${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/${file}`,
}),
));
}
CloudRunnerLogger.log(`All Locks ${files} ${workspace} ${runId}`);
CloudRunnerLogger.log(`Deleting lock ${workspace}/${file}`);
CloudRunnerLogger.log(`rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`);
await CloudRunnerSystem.Run(
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
false,
true,
);
return !(await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext));
}
public static async CleanupWorkspace(workspace: string, buildParametersContext: BuildParameters) {
const prefix = `${SharedWorkspaceLocking.workspacePrefix}${buildParametersContext.cacheKey}/`;
const files = await SharedWorkspaceLocking.listObjects(prefix);
for (const file of files.filter((x) => x.includes(`_${workspace}_`))) {
await (SharedWorkspaceLocking.useRclone
? SharedWorkspaceLocking.rclone(`delete ${SharedWorkspaceLocking.bucket}/${prefix}${file}`)
: SharedWorkspaceLocking.s3.send(
new DeleteObjectCommand({ Bucket: SharedWorkspaceLocking.bucket, Key: `${prefix}${file}` }),
));
}
await CloudRunnerSystem.Run(
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey} --exclude "*" --include "*_${workspace}_*"`,
false,
true,
);
}
public static async ReadLines(command: string): Promise<string[]> {
const path = command.replace('aws s3 ls', '').replace('rclone lsf', '').trim();
const withoutScheme = path.replace('s3://', '');
const [bucket, ...rest] = withoutScheme.split('/');
const prefix = rest.join('/');
return SharedWorkspaceLocking.listObjects(prefix, bucket);
return CloudRunnerSystem.RunAndReadLines(command);
}
}

View File

@@ -1,10 +1,10 @@
import BuildParameters from '../../../build-parameters';
import Input from '../../../input';
import OrchestratorOptions from '../../options/orchestrator-options';
import OrchestratorEnvironmentVariable from '../../options/orchestrator-environment-variable';
import OrchestratorOptionsReader from '../../options/orchestrator-options-reader';
import OrchestratorQueryOverride from '../../options/orchestrator-query-override';
import OrchestratorSecret from '../../options/orchestrator-secret';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerOptionsReader from '../../options/cloud-runner-options-reader';
import CloudRunnerQueryOverride from '../../options/cloud-runner-query-override';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import { CommandHookService } from '../hooks/command-hook-service';
export class TaskParameterSerializer {
@@ -19,10 +19,10 @@ export class TaskParameterSerializer {
'NAME',
'CUSTOM_JOB',
]);
public static createOrchestratorEnvironmentVariables(
public static createCloudRunnerEnvironmentVariables(
buildParameters: BuildParameters,
): OrchestratorEnvironmentVariable[] {
const result: OrchestratorEnvironmentVariable[] = this.uniqBy(
): CloudRunnerEnvironmentVariable[] {
const result: CloudRunnerEnvironmentVariable[] = this.uniqBy(
[
...[
{ name: 'BUILD_TARGET', value: buildParameters.targetPlatform },
@@ -31,11 +31,8 @@ export class TaskParameterSerializer {
],
...TaskParameterSerializer.serializeFromObject(buildParameters),
...TaskParameterSerializer.serializeInput(),
...TaskParameterSerializer.serializeOrchestratorOptions(),
...TaskParameterSerializer.serializeCloudRunnerOptions(),
...CommandHookService.getSecrets(CommandHookService.getHooks(buildParameters.commandHooks)),
// Include AWS environment variables for LocalStack compatibility
...TaskParameterSerializer.serializeAwsEnvironmentVariables(),
]
.filter(
(x) =>
@@ -50,14 +47,14 @@ export class TaskParameterSerializer {
return x;
}),
(item: OrchestratorEnvironmentVariable) => item.name,
(item: CloudRunnerEnvironmentVariable) => item.name,
);
return result;
}
// eslint-disable-next-line no-unused-vars
static uniqBy(a: OrchestratorEnvironmentVariable[], key: (parameters: OrchestratorEnvironmentVariable) => string) {
static uniqBy(a: CloudRunnerEnvironmentVariable[], key: (parameters: CloudRunnerEnvironmentVariable) => string) {
const seen: { [key: string]: boolean } = {};
return a.filter(function (item) {
@@ -90,34 +87,12 @@ export class TaskParameterSerializer {
return TaskParameterSerializer.serializeFromType(Input);
}
private static serializeOrchestratorOptions() {
return TaskParameterSerializer.serializeFromType(OrchestratorOptions);
}
private static serializeAwsEnvironmentVariables() {
const awsEnvironmentVariables = [
'AWS_ACCESS_KEY_ID',
'AWS_SECRET_ACCESS_KEY',
'AWS_DEFAULT_REGION',
'AWS_REGION',
'AWS_S3_ENDPOINT',
'AWS_ENDPOINT',
'AWS_CLOUD_FORMATION_ENDPOINT',
'AWS_ECS_ENDPOINT',
'AWS_KINESIS_ENDPOINT',
'AWS_CLOUD_WATCH_LOGS_ENDPOINT',
];
return awsEnvironmentVariables
.filter((key) => process.env[key] !== undefined)
.map((key) => ({
name: key,
value: process.env[key] || '',
}));
private static serializeCloudRunnerOptions() {
return TaskParameterSerializer.serializeFromType(CloudRunnerOptions);
}
public static ToEnvVarFormat(input: string): string {
return OrchestratorOptions.ToEnvVarFormat(input);
return CloudRunnerOptions.ToEnvVarFormat(input);
}
public static UndoEnvVarFormat(element: string): string {
@@ -153,7 +128,7 @@ export class TaskParameterSerializer {
private static serializeFromType(type: any) {
const array: any[] = [];
const input = OrchestratorOptionsReader.GetProperties();
const input = CloudRunnerOptionsReader.GetProperties();
for (const element of input) {
if (typeof type[element] !== 'function' && array.filter((x) => x.name === element).length === 0) {
array.push({
@@ -166,7 +141,7 @@ export class TaskParameterSerializer {
return array;
}
public static readDefaultSecrets(): OrchestratorSecret[] {
public static readDefaultSecrets(): CloudRunnerSecret[] {
let array = new Array();
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_SERIAL');
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_EMAIL');
@@ -179,13 +154,13 @@ export class TaskParameterSerializer {
}
private static getValue(key: string) {
return OrchestratorQueryOverride.queryOverrides !== undefined &&
OrchestratorQueryOverride.queryOverrides[key] !== undefined
? OrchestratorQueryOverride.queryOverrides[key]
return CloudRunnerQueryOverride.queryOverrides !== undefined &&
CloudRunnerQueryOverride.queryOverrides[key] !== undefined
? CloudRunnerQueryOverride.queryOverrides[key]
: process.env[key];
}
private static tryAddInput(array: OrchestratorSecret[], key: string): OrchestratorSecret[] {
private static tryAddInput(array: CloudRunnerSecret[], key: string): CloudRunnerSecret[] {
const value = TaskParameterSerializer.getValue(key);
if (value !== undefined && value !== '' && value !== 'null') {
array.push({

View File

@@ -2,26 +2,26 @@ import { BuildParameters, Input } from '../../..';
import YAML from 'yaml';
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
import path from 'node:path';
import OrchestratorOptions from '../../options/orchestrator-options';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import * as fs from 'node:fs';
import OrchestratorLogger from '../core/orchestrator-logger';
import CloudRunnerLogger from '../core/cloud-runner-logger';
import { CommandHook } from './command-hook';
// import OrchestratorLogger from './orchestrator-logger';
// import CloudRunnerLogger from './cloud-runner-logger';
export class CommandHookService {
public static ApplyHooksToCommands(commands: string, buildParameters: BuildParameters): string {
const hooks = CommandHookService.getHooks(buildParameters.commandHooks);
OrchestratorLogger.log(`Applying hooks ${hooks.length}`);
CloudRunnerLogger.log(`Applying hooks ${hooks.length}`);
return `echo "---"
echo "start orchestrator init"
${OrchestratorOptions.orchestratorDebug ? `printenv` : `#`}
echo "start of orchestrator job"
echo "start cloud runner init"
${CloudRunnerOptions.cloudRunnerDebug ? `printenv` : `#`}
echo "start of cloud runner job"
${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
${commands}
${hooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
echo "end of orchestrator job"
echo "end of cloud runner job"
echo "---${buildParameters.logId}"`;
}
@@ -51,7 +51,7 @@ echo "---${buildParameters.logId}"`;
const gameCiCustomHooksPath = path.join(process.cwd(), `game-ci`, `command-hooks`);
const files = fs.readdirSync(gameCiCustomHooksPath);
for (const file of files) {
if (!OrchestratorOptions.commandHookFiles.includes(file.replace(`.yaml`, ``))) {
if (!CloudRunnerOptions.commandHookFiles.includes(file.replace(`.yaml`, ``))) {
continue;
}
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
@@ -89,9 +89,9 @@ echo "---${buildParameters.logId}"`;
return [];
}
// if (Orchestrator.buildParameters?.orchestratorIntegrationTests) {
// if (CloudRunner.buildParameters?.cloudRunnerIntegrationTests) {
// OrchestratorLogger.log(`Parsing build hooks: ${steps}`);
// CloudRunnerLogger.log(`Parsing build hooks: ${steps}`);
// }
const isArray = hooks.replace(/\s/g, ``)[0] === `-`;

View File

@@ -1,8 +1,8 @@
import OrchestratorSecret from '../../options/orchestrator-secret';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
export class CommandHook {
public commands: string[] = new Array<string>();
public secrets: OrchestratorSecret[] = new Array<OrchestratorSecret>();
public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
public name!: string;
public hook!: string[];
public step!: string[];

View File

@@ -0,0 +1,265 @@
import YAML from 'yaml';
import CloudRunner from '../../cloud-runner';
import { CustomWorkflow } from '../../workflows/custom-workflow';
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
import path from 'node:path';
import fs from 'node:fs';
import Input from '../../../input';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import { ContainerHook as ContainerHook } from './container-hook';
import { CloudRunnerStepParameters } from '../../options/cloud-runner-step-parameters';
export class ContainerHookService {
static GetContainerHooksFromFiles(hookLifecycle: string): ContainerHook[] {
const results: ContainerHook[] = [];
try {
const gameCiCustomStepsPath = path.join(process.cwd(), `game-ci`, `container-hooks`);
const files = fs.readdirSync(gameCiCustomStepsPath);
for (const file of files) {
if (!CloudRunnerOptions.containerHookFiles.includes(file.replace(`.yaml`, ``))) {
// RemoteClientLogger.log(`Skipping CustomStepFile: ${file}`);
continue;
}
const fileContents = fs.readFileSync(path.join(gameCiCustomStepsPath, file), `utf8`);
const fileContentsObject = ContainerHookService.ParseContainerHooks(fileContents)[0];
if (fileContentsObject.hook === hookLifecycle) {
results.push(fileContentsObject);
}
}
} catch (error) {
RemoteClientLogger.log(`Failed Getting: ${hookLifecycle} \n ${JSON.stringify(error, undefined, 4)}`);
}
// RemoteClientLogger.log(`Active Steps From Files: \n ${JSON.stringify(results, undefined, 4)}`);
const builtInContainerHooks: ContainerHook[] = ContainerHookService.ParseContainerHooks(
`- name: aws-s3-upload-build
image: amazon/aws-cli
hook: after
commands: |
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 cp /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
} s3://${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}
rm /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}
secrets:
- name: awsAccessKeyId
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
- name: awsSecretAccessKey
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
- name: awsDefaultRegion
value: ${process.env.AWS_REGION || ``}
- name: aws-s3-pull-build
image: amazon/aws-cli
commands: |
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/ || true
aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/build || true
mkdir -p /data/cache/$CACHE_KEY/build/
aws s3 cp s3://${
CloudRunner.buildParameters.awsStackName
}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
} /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}
secrets:
- name: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
- name: AWS_DEFAULT_REGION
- name: BUILD_GUID_TARGET
- name: steam-deploy-client
image: steamcmd/steamcmd
commands: |
apt-get update
apt-get install -y curl tar coreutils git tree > /dev/null
curl -s https://gist.githubusercontent.com/frostebite/1d56f5505b36b403b64193b7a6e54cdc/raw/fa6639ed4ef750c4268ea319d63aa80f52712ffb/deploy-client-steam.sh | bash
secrets:
- name: STEAM_USERNAME
- name: STEAM_PASSWORD
- name: STEAM_APPID
- name: STEAM_SSFN_FILE_NAME
- name: STEAM_SSFN_FILE_CONTENTS
- name: STEAM_CONFIG_VDF_1
- name: STEAM_CONFIG_VDF_2
- name: STEAM_CONFIG_VDF_3
- name: STEAM_CONFIG_VDF_4
- name: BUILD_GUID_TARGET
- name: RELEASE_BRANCH
- name: steam-deploy-project
image: steamcmd/steamcmd
commands: |
apt-get update
apt-get install -y curl tar coreutils git tree > /dev/null
curl -s https://gist.githubusercontent.com/frostebite/969da6a41002a0e901174124b643709f/raw/02403e53fb292026cba81ddcf4ff35fc1eba111d/steam-deploy-project.sh | bash
secrets:
- name: STEAM_USERNAME
- name: STEAM_PASSWORD
- name: STEAM_APPID
- name: STEAM_SSFN_FILE_NAME
- name: STEAM_SSFN_FILE_CONTENTS
- name: STEAM_CONFIG_VDF_1
- name: STEAM_CONFIG_VDF_2
- name: STEAM_CONFIG_VDF_3
- name: STEAM_CONFIG_VDF_4
- name: BUILD_GUID_2
- name: RELEASE_BRANCH
- name: aws-s3-upload-cache
image: amazon/aws-cli
hook: after
commands: |
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://${
CloudRunner.buildParameters.awsStackName
}/cloud-runner-cache/$CACHE_KEY/lfs
rm -r /data/cache/$CACHE_KEY/lfs
aws s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://${
CloudRunner.buildParameters.awsStackName
}/cloud-runner-cache/$CACHE_KEY/Library
rm -r /data/cache/$CACHE_KEY/Library
secrets:
- name: AWS_ACCESS_KEY_ID
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
- name: AWS_SECRET_ACCESS_KEY
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
- name: AWS_DEFAULT_REGION
value: ${process.env.AWS_REGION || ``}
- name: aws-s3-pull-cache
image: amazon/aws-cli
hook: before
commands: |
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
mkdir -p /data/cache/$CACHE_KEY/Library/
mkdir -p /data/cache/$CACHE_KEY/lfs/
aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/ || true
aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/ || true
BUCKET1="${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/Library/"
aws s3 ls $BUCKET1 || true
OBJECT1="$(aws s3 ls $BUCKET1 | sort | tail -n 1 | awk '{print $4}' || '')"
aws s3 cp s3://$BUCKET1$OBJECT1 /data/cache/$CACHE_KEY/Library/ || true
BUCKET2="${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/lfs/"
aws s3 ls $BUCKET2 || true
OBJECT2="$(aws s3 ls $BUCKET2 | sort | tail -n 1 | awk '{print $4}' || '')"
aws s3 cp s3://$BUCKET2$OBJECT2 /data/cache/$CACHE_KEY/lfs/ || true
secrets:
- name: AWS_ACCESS_KEY_ID
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
- name: AWS_SECRET_ACCESS_KEY
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
- name: AWS_DEFAULT_REGION
value: ${process.env.AWS_REGION || ``}
- name: debug-cache
image: ubuntu
hook: after
commands: |
apt-get update > /dev/null
${CloudRunnerOptions.cloudRunnerDebug ? `apt-get install -y tree > /dev/null` : `#`}
${CloudRunnerOptions.cloudRunnerDebug ? `tree -L 3 /data/cache` : `#`}
secrets:
- name: awsAccessKeyId
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
- name: awsSecretAccessKey
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
- name: awsDefaultRegion
value: ${process.env.AWS_REGION || ``}`,
).filter((x) => CloudRunnerOptions.containerHookFiles.includes(x.name) && x.hook === hookLifecycle);
if (builtInContainerHooks.length > 0) {
results.push(...builtInContainerHooks);
}
return results;
}
private static ConvertYamlSecrets(object: ContainerHook) {
if (object.secrets === undefined) {
object.secrets = [];
return;
}
object.secrets = object.secrets.map((x: { [key: string]: any }) => {
return {
ParameterKey: x.name,
EnvironmentVariable: Input.ToEnvVarFormat(x.name),
ParameterValue: x.value,
};
});
}
public static ParseContainerHooks(steps: string): ContainerHook[] {
if (steps === '') {
return [];
}
const isArray = steps.replace(/\s/g, ``)[0] === `-`;
const object: ContainerHook[] = isArray ? YAML.parse(steps) : [YAML.parse(steps)];
for (const step of object) {
ContainerHookService.ConvertYamlSecrets(step);
if (step.secrets === undefined) {
step.secrets = [];
} else {
for (const secret of step.secrets) {
if (secret.ParameterValue === undefined && process.env[secret.EnvironmentVariable] !== undefined) {
if (CloudRunner.buildParameters?.cloudRunnerDebug) {
// CloudRunnerLogger.log(`Injecting custom step ${step.name} from env var ${secret.ParameterKey}`);
}
secret.ParameterValue = process.env[secret.ParameterKey] || ``;
}
}
}
if (step.image === undefined) {
step.image = `ubuntu`;
}
}
if (object === undefined) {
throw new Error(`Failed to parse ${steps}`);
}
return object;
}
static async RunPostBuildSteps(cloudRunnerStepState: CloudRunnerStepParameters) {
let output = ``;
const steps: ContainerHook[] = [
...ContainerHookService.ParseContainerHooks(CloudRunner.buildParameters.postBuildContainerHooks),
...ContainerHookService.GetContainerHooksFromFiles(`after`),
];
if (steps.length > 0) {
output += await CustomWorkflow.runContainerJob(
steps,
cloudRunnerStepState.environment,
cloudRunnerStepState.secrets,
);
}
return output;
}
static async RunPreBuildSteps(cloudRunnerStepState: CloudRunnerStepParameters) {
let output = ``;
const steps: ContainerHook[] = [
...ContainerHookService.ParseContainerHooks(CloudRunner.buildParameters.preBuildContainerHooks),
...ContainerHookService.GetContainerHooksFromFiles(`before`),
];
if (steps.length > 0) {
output += await CustomWorkflow.runContainerJob(
steps,
cloudRunnerStepState.environment,
cloudRunnerStepState.secrets,
);
}
return output;
}
}

View File

@@ -0,0 +1,9 @@
import CloudRunnerSecret from '../../options/cloud-runner-secret';
export class ContainerHook {
public commands!: string;
public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
public name!: string;
public image: string = `ubuntu`;
public hook!: string;
}

View File

@@ -1,20 +1,20 @@
import path from 'node:path';
import { OrchestratorFolders } from '../../options/orchestrator-folders';
import { OrchestratorSystem } from '../core/orchestrator-system';
import { CloudRunnerFolders } from '../../options/cloud-runner-folders';
import { CloudRunnerSystem } from '../core/cloud-runner-system';
import fs from 'node:fs';
import { Cli } from '../../../cli/cli';
import { CliFunction } from '../../../cli/cli-functions-repository';
export class LfsHashing {
public static async createLFSHashFiles() {
await OrchestratorSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`);
await OrchestratorSystem.Run(`md5sum .lfs-assets-guid > .lfs-assets-guid-sum`);
await CloudRunnerSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`);
await CloudRunnerSystem.Run(`md5sum .lfs-assets-guid > .lfs-assets-guid-sum`);
const lfsHashes = {
lfsGuid: fs
.readFileSync(`${path.join(OrchestratorFolders.repoPathAbsolute, `.lfs-assets-guid`)}`, 'utf8')
.readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid`)}`, 'utf8')
.replace(/\n/g, ``),
lfsGuidSum: fs
.readFileSync(`${path.join(OrchestratorFolders.repoPathAbsolute, `.lfs-assets-guid-sum`)}`, 'utf8')
.readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid-sum`)}`, 'utf8')
.replace(' .lfs-assets-guid', '')
.replace(/\n/g, ``),
};
@@ -24,7 +24,7 @@ export class LfsHashing {
public static async hashAllFiles(folder: string) {
const startPath = process.cwd();
process.chdir(folder);
const result = await (await OrchestratorSystem.Run(`find -type f -exec md5sum "{}" + | sort | md5sum`))
const result = await (await CloudRunnerSystem.Run(`find -type f -exec md5sum "{}" + | sort | md5sum`))
.replace(/\n/g, '')
.split(` `)[0];
process.chdir(startPath);

View File

@@ -1,9 +1,9 @@
import { BuildParameters, ImageTag } from '../..';
import Orchestrator from '../orchestrator';
import CloudRunner from '../cloud-runner';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import OrchestratorOptions from '../options/orchestrator-options';
import setups from './orchestrator-suite.test';
import CloudRunnerOptions from '../options/cloud-runner-options';
import setups from './cloud-runner-suite.test';
import { OptionValues } from 'commander';
async function CreateParameters(overrides: OptionValues | undefined) {
@@ -11,18 +11,18 @@ async function CreateParameters(overrides: OptionValues | undefined) {
return BuildParameters.create();
}
describe('Orchestrator Async Workflows', () => {
describe('Cloud Runner Async Workflows', () => {
setups();
it('Responds', () => {});
if (OrchestratorOptions.orchestratorDebug && OrchestratorOptions.providerStrategy !== `local-docker`) {
if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.providerStrategy !== `local-docker`) {
it('Async Workflows', async () => {
// Setup parameters
const buildParameter = await CreateParameters({
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.read('test-project'),
asyncOrchestrator: `true`,
asyncCloudRunner: `true`,
githubChecks: `true`,
providerStrategy: 'k8s',
buildPlatform: 'linux',
@@ -31,7 +31,7 @@ describe('Orchestrator Async Workflows', () => {
const baseImage = new ImageTag(buildParameter);
// Run the job
await Orchestrator.run(buildParameter, baseImage.toString());
await CloudRunner.run(buildParameter, baseImage.toString());
// wait for 15 seconds
await new Promise((resolve) => setTimeout(resolve, 1000 * 60 * 12));

View File

@@ -3,15 +3,15 @@ import path from 'node:path';
import BuildParameters from '../../build-parameters';
import { Cli } from '../../cli/cli';
import UnityVersioning from '../../unity-versioning';
import Orchestrator from '../orchestrator';
import { OrchestratorSystem } from '../services/core/orchestrator-system';
import CloudRunner from '../cloud-runner';
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
import { Caching } from '../remote-client/caching';
import { v4 as uuidv4 } from 'uuid';
import GitHub from '../../github';
import OrchestratorOptions from '../options/orchestrator-options';
describe('Orchestrator (Remote Client) Caching', () => {
import CloudRunnerOptions from '../options/cloud-runner-options';
describe('Cloud Runner (Remote Client) Caching', () => {
it('responds', () => {});
if (OrchestratorOptions.providerStrategy === `local-docker`) {
if (CloudRunnerOptions.providerStrategy === `local-docker`) {
it('Simple caching works', async () => {
Cli.options = {
versioning: 'None',
@@ -22,7 +22,7 @@ describe('Orchestrator (Remote Client) Caching', () => {
};
GitHub.githubInputEnabled = false;
const buildParameter = await BuildParameters.create();
Orchestrator.buildParameters = buildParameter;
CloudRunner.buildParameters = buildParameter;
// Create test folder
const testFolder = path.resolve(__dirname, Cli.options.cacheKey);
@@ -43,7 +43,7 @@ describe('Orchestrator (Remote Client) Caching', () => {
testFolder.replace(/\\/g, `/`),
`${Cli.options.cacheKey}`,
);
await OrchestratorSystem.Run(`du -h ${__dirname}`);
await CloudRunnerSystem.Run(`du -h ${__dirname}`);
// Compare validity to original hash
expect(fs.readFileSync(path.resolve(testFolder, 'test.txt'), { encoding: 'utf8' }).toString()).toContain(

View File

@@ -1,12 +1,12 @@
import { BuildParameters, Orchestrator, ImageTag, Input } from '../..';
import { BuildParameters, CloudRunner, ImageTag, Input } from '../..';
import { TaskParameterSerializer } from '../services/core/task-parameter-serializer';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import GitHub from '../../github';
import setups from './orchestrator-suite.test';
import { OrchestratorStatics } from '../options/orchestrator-statics';
import OrchestratorOptions from '../options/orchestrator-options';
import OrchestratorLogger from '../services/core/orchestrator-logger';
import setups from './cloud-runner-suite.test';
import { CloudRunnerStatics } from '../options/cloud-runner-statics';
import CloudRunnerOptions from '../options/cloud-runner-options';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
async function CreateParameters(overrides: any) {
if (overrides) {
@@ -21,14 +21,14 @@ async function CreateParameters(overrides: any) {
return results;
}
describe('Orchestrator Sync Environments', () => {
describe('Cloud Runner Sync Environments', () => {
setups();
const testSecretName = 'testSecretName';
const testSecretValue = 'testSecretValue';
it('Responds', () => {});
if (OrchestratorOptions.orchestratorDebug) {
it('All build parameters sent to orchestrator as env vars', async () => {
if (CloudRunnerOptions.cloudRunnerDebug) {
it('All build parameters sent to cloud runner as env vars', async () => {
// Setup parameters
const buildParameter = await CreateParameters({
versioning: 'None',
@@ -43,7 +43,6 @@ describe('Orchestrator Sync Environments', () => {
- name: '${testSecretName}'
value: '${testSecretValue}'
`,
orchestratorDebug: true,
});
const baseImage = new ImageTag(buildParameter);
if (baseImage.toString().includes('undefined')) {
@@ -51,49 +50,23 @@ describe('Orchestrator Sync Environments', () => {
}
// Run the job
const file = (await Orchestrator.run(buildParameter, baseImage.toString())).BuildResults;
const file = (await CloudRunner.run(buildParameter, baseImage.toString())).BuildResults;
// Assert results
// expect(file).toContain(JSON.stringify(buildParameter));
expect(file).toContain(`${Input.ToEnvVarFormat(testSecretName)}=${testSecretValue}`);
const environmentVariables = TaskParameterSerializer.createOrchestratorEnvironmentVariables(buildParameter);
const environmentVariables = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
const secrets = TaskParameterSerializer.readDefaultSecrets().map((x) => {
return {
name: x.EnvironmentVariable,
value: x.ParameterValue,
};
});
// Apply the same localhost -> host.docker.internal replacement that the Docker provider does
// This ensures the test expectations match what's actually in the output
const endpointEnvironmentNames = new Set([
'AWS_S3_ENDPOINT',
'AWS_ENDPOINT',
'AWS_CLOUD_FORMATION_ENDPOINT',
'AWS_ECS_ENDPOINT',
'AWS_KINESIS_ENDPOINT',
'AWS_CLOUD_WATCH_LOGS_ENDPOINT',
'INPUT_AWSS3ENDPOINT',
'INPUT_AWSENDPOINT',
]);
const combined = [...environmentVariables, ...secrets]
.filter((element) => element.value !== undefined && element.value !== '' && typeof element.value !== 'function')
.map((x) => {
if (typeof x.value === `string`) {
x.value = x.value.replace(/\s+/g, '');
// Apply localhost -> host.docker.internal replacement for LocalStack endpoints
// when using local-docker or aws provider (which uses Docker)
if (
endpointEnvironmentNames.has(x.name) &&
(x.value.startsWith('http://localhost') || x.value.startsWith('http://127.0.0.1')) &&
(OrchestratorOptions.providerStrategy === 'local-docker' ||
OrchestratorOptions.providerStrategy === 'aws')
) {
x.value = x.value
.replace('http://localhost', 'http://host.docker.internal')
.replace('http://127.0.0.1', 'http://host.docker.internal');
}
}
return x;
@@ -103,10 +76,10 @@ describe('Orchestrator Sync Environments', () => {
});
const newLinePurgedFile = file
.replace(/\s+/g, '')
.replace(new RegExp(`\\[${OrchestratorStatics.logPrefix}\\]`, 'g'), '');
.replace(new RegExp(`\\[${CloudRunnerStatics.logPrefix}\\]`, 'g'), '');
for (const element of combined) {
expect(newLinePurgedFile).toContain(`${element.name}`);
OrchestratorLogger.log(`Contains ${element.name}`);
CloudRunnerLogger.log(`Contains ${element.name}`);
const fullNameEqualValue = `${element.name}=${element.value}`;
expect(newLinePurgedFile).toContain(fullNameEqualValue);
}
@@ -114,11 +87,11 @@ describe('Orchestrator Sync Environments', () => {
}
});
describe('Orchestrator Environment Serializer', () => {
describe('Cloud Runner Environment Serializer', () => {
setups();
const testSecretName = 'testSecretName';
const testSecretValue = 'testSecretValue';
it('Orchestrator Parameter Serialization', async () => {
it('Cloud Runner Parameter Serialization', async () => {
// Setup parameters
const buildParameter = await CreateParameters({
versioning: 'None',
@@ -134,9 +107,9 @@ describe('Orchestrator Environment Serializer', () => {
`,
});
const result = TaskParameterSerializer.createOrchestratorEnvironmentVariables(buildParameter);
const result = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
expect(result.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
const result2 = TaskParameterSerializer.createOrchestratorEnvironmentVariables(buildParameter);
const result2 = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
expect(result2.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
});
});

View File

@@ -1,9 +1,17 @@
import Orchestrator from '../orchestrator';
import { BuildParameters } from '../..';
import CloudRunner from '../cloud-runner';
import UnityVersioning from '../../unity-versioning';
import setups from './orchestrator-suite.test';
import { Cli } from '../../cli/cli';
import setups from './cloud-runner-suite.test';
import { OptionValues } from 'commander';
import GitHub from '../../github';
import { TIMEOUT_INFINITE, createParameters } from '../../../test-utils/orchestrator-test-helpers';
describe('Orchestrator Github Checks', () => {
export const TIMEOUT_INFINITE = 1e9;
async function CreateParameters(overrides: OptionValues | undefined) {
if (overrides) Cli.options = overrides;
return BuildParameters.create();
}
describe('Cloud Runner Github Checks', () => {
setups();
it('Responds', () => {});
@@ -17,7 +25,6 @@ describe('Orchestrator Github Checks', () => {
status: 200,
data: {},
});
// eslint-disable-next-line unicorn/no-useless-undefined
jest.spyOn(GitHub as any, 'runUpdateAsyncChecksWorkflow').mockResolvedValue(undefined);
});
@@ -29,15 +36,15 @@ describe('Orchestrator Github Checks', () => {
'Check Handling Direct',
async () => {
// Setup parameters
const buildParameter = await createParameters({
const buildParameter = await CreateParameters({
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.read('test-project'),
asyncOrchestrator: `true`,
asyncCloudRunner: `true`,
githubChecks: `true`,
});
await Orchestrator.setup(buildParameter);
Orchestrator.buildParameters.githubCheckId = await GitHub.createGitHubCheck(`direct create`);
await CloudRunner.setup(buildParameter);
CloudRunner.buildParameters.githubCheckId = await GitHub.createGitHubCheck(`direct create`);
await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `direct`);
await GitHub.updateGitHubCheck(`2 ${new Date().toISOString()}`, `direct`, `success`, `completed`);
},
@@ -47,16 +54,16 @@ describe('Orchestrator Github Checks', () => {
'Check Handling Via Async Workflow',
async () => {
// Setup parameters
const buildParameter = await createParameters({
const buildParameter = await CreateParameters({
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.read('test-project'),
asyncOrchestrator: `true`,
asyncCloudRunner: `true`,
githubChecks: `true`,
});
GitHub.forceAsyncTest = true;
await Orchestrator.setup(buildParameter);
Orchestrator.buildParameters.githubCheckId = await GitHub.createGitHubCheck(`async create`);
await CloudRunner.setup(buildParameter);
CloudRunner.buildParameters.githubCheckId = await GitHub.createGitHubCheck(`async create`);
await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `async`);
await GitHub.updateGitHubCheck(`2 ${new Date().toISOString()}`, `async`, `success`, `completed`);
GitHub.forceAsyncTest = false;

View File

@@ -1,11 +1,11 @@
import Orchestrator from '../orchestrator';
import CloudRunner from '../cloud-runner';
import { BuildParameters, ImageTag } from '../..';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import OrchestratorLogger from '../services/core/orchestrator-logger';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid';
import OrchestratorOptions from '../options/orchestrator-options';
import setups from './orchestrator-suite.test';
import CloudRunnerOptions from '../options/cloud-runner-options';
import setups from './cloud-runner-suite.test';
import { ContainerHookService } from '../services/hooks/container-hook-service';
import { CommandHookService } from '../services/hooks/command-hook-service';
@@ -17,7 +17,7 @@ async function CreateParameters(overrides: any) {
return await BuildParameters.create();
}
describe('Orchestrator Custom Hooks And Steps', () => {
describe('Cloud Runner Custom Hooks And Steps', () => {
it('Responds', () => {});
setups();
it('Check parsing and reading of steps', async () => {
@@ -33,12 +33,12 @@ commands: echo "test"`;
image: 'ubuntu',
cacheKey: `test-case-${uuidv4()}`,
};
Orchestrator.setup(await CreateParameters(overrides));
CloudRunner.setup(await CreateParameters(overrides));
const stringObject = ContainerHookService.ParseContainerHooks(yamlString);
const stringObject2 = ContainerHookService.ParseContainerHooks(yamlString2);
OrchestratorLogger.log(yamlString);
OrchestratorLogger.log(JSON.stringify(stringObject, undefined, 4));
CloudRunnerLogger.log(yamlString);
CloudRunnerLogger.log(JSON.stringify(stringObject, undefined, 4));
expect(stringObject.length).toBe(1);
expect(stringObject[0].hook).toBe(`before`);
@@ -46,9 +46,9 @@ commands: echo "test"`;
expect(stringObject2[0].hook).toBe(`before`);
const getCustomStepsFromFiles = ContainerHookService.GetContainerHooksFromFiles(`before`);
OrchestratorLogger.log(JSON.stringify(getCustomStepsFromFiles, undefined, 4));
CloudRunnerLogger.log(JSON.stringify(getCustomStepsFromFiles, undefined, 4));
});
if (OrchestratorOptions.orchestratorDebug) {
if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.providerStrategy !== `k8s`) {
it('Should be 1 before and 1 after hook', async () => {
const overrides = {
versioning: 'None',
@@ -61,7 +61,7 @@ commands: echo "test"`;
commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
};
const buildParameter2 = await CreateParameters(overrides);
await Orchestrator.setup(buildParameter2);
await CloudRunner.setup(buildParameter2);
const beforeHooks = CommandHookService.GetCustomHooksFromFiles(`before`);
const afterHooks = CommandHookService.GetCustomHooksFromFiles(`after`);
expect(beforeHooks).toHaveLength(1);
@@ -79,7 +79,7 @@ commands: echo "test"`;
commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
};
const buildParameter2 = await CreateParameters(overrides);
await Orchestrator.setup(buildParameter2);
await CloudRunner.setup(buildParameter2);
const beforeSteps = ContainerHookService.GetContainerHooksFromFiles(`before`);
const afterSteps = ContainerHookService.GetContainerHooksFromFiles(`after`);
expect(beforeSteps).toHaveLength(1);
@@ -94,29 +94,21 @@ commands: echo "test"`;
cacheKey: `test-case-${uuidv4()}`,
containerHookFiles: `my-test-step-pre-build,my-test-step-post-build`,
commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
orchestratorDebug: true,
};
const buildParameter2 = await CreateParameters(overrides);
const baseImage2 = new ImageTag(buildParameter2);
const results2Object = await Orchestrator.run(buildParameter2, baseImage2.toString());
const results2Object = await CloudRunner.run(buildParameter2, baseImage2.toString());
const results2 = results2Object.BuildResults;
OrchestratorLogger.log(`run 2 succeeded`);
CloudRunnerLogger.log(`run 2 succeeded`);
const buildContainsBuildSucceeded = results2.includes('Build succeeded');
const buildContainsPreBuildHookRunMessage = results2.includes('before-build hook test!!');
const buildContainsPreBuildHookRunMessage = results2.includes('before-build hook test!');
const buildContainsPostBuildHookRunMessage = results2.includes('after-build hook test!');
const buildContainsPreBuildStepMessage = results2.includes('before-build step test!');
const buildContainsPostBuildStepMessage = results2.includes('after-build step test!');
// Skip "Build succeeded" check for local-docker and aws when using ubuntu image (Unity doesn't run)
if (
OrchestratorOptions.providerStrategy !== 'local' &&
OrchestratorOptions.providerStrategy !== 'local-docker' &&
OrchestratorOptions.providerStrategy !== 'aws'
) {
expect(buildContainsBuildSucceeded).toBeTruthy();
}
expect(buildContainsBuildSucceeded).toBeTruthy();
expect(buildContainsPreBuildHookRunMessage).toBeTruthy();
expect(buildContainsPostBuildHookRunMessage).toBeTruthy();
expect(buildContainsPreBuildStepMessage).toBeTruthy();

View File

@@ -2,7 +2,7 @@ import { BuildParameters, ImageTag } from '../..';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import GitHub from '../../github';
import setups from './orchestrator-suite.test';
import setups from './cloud-runner-suite.test';
async function CreateParameters(overrides: any) {
if (overrides) {
@@ -17,7 +17,7 @@ async function CreateParameters(overrides: any) {
return results;
}
describe('Orchestrator Image', () => {
describe('Cloud Runner Image', () => {
setups();
const testSecretName = 'testSecretName';
const testSecretValue = 'testSecretValue';

View File

@@ -1,17 +1,17 @@
import { ImageTag } from '../..';
import Orchestrator from '../orchestrator';
import CloudRunner from '../cloud-runner';
import UnityVersioning from '../../unity-versioning';
import OrchestratorOptions from '../options/orchestrator-options';
import setups from './orchestrator-suite.test';
import CloudRunnerOptions from '../options/cloud-runner-options';
import setups from './cloud-runner-suite.test';
import fs from 'node:fs';
import { CreateParameters } from './create-test-parameter';
import OrchestratorLogger from '../services/core/orchestrator-logger';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
describe('Orchestrator Local Docker Workflows', () => {
describe('Cloud Runner Local Docker Workflows', () => {
setups();
it('Responds', () => {});
if (OrchestratorOptions.providerStrategy === `local-docker`) {
if (CloudRunnerOptions.providerStrategy === `local-docker`) {
it('inspect stateful folder of workflows', async () => {
const testValue = `the state in a job exits in the expected local-docker folder`;
@@ -39,15 +39,15 @@ describe('Orchestrator Local Docker Workflows', () => {
const baseImage = new ImageTag(buildParameter);
// Run the job
await Orchestrator.run(buildParameter, baseImage.toString());
await Orchestrator.run(buildParameter2, baseImage.toString());
await CloudRunner.run(buildParameter, baseImage.toString());
await CloudRunner.run(buildParameter2, baseImage.toString());
const outputFile = fs.readFileSync(`./orchestrator-cache/test-out-state.txt`, `utf-8`);
const outputFile = fs.readFileSync(`./cloud-runner-cache/test-out-state.txt`, `utf-8`);
expect(outputFile).toMatch(testValue);
const outputFile2 = fs.readFileSync(`./orchestrator-cache/test-out-state-2.txt`, `utf-8`);
const outputFile2 = fs.readFileSync(`./cloud-runner-cache/test-out-state-2.txt`, `utf-8`);
expect(outputFile2).toMatch(testValue);
OrchestratorLogger.log(outputFile);
CloudRunnerLogger.log(outputFile);
}, 1_000_000_000);
}
});

View File

@@ -1,12 +1,12 @@
import SharedWorkspaceLocking from '../services/core/shared-workspace-locking';
import { Cli } from '../../cli/cli';
import setups from './orchestrator-suite.test';
import OrchestratorLogger from '../services/core/orchestrator-logger';
import setups from './cloud-runner-suite.test';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid';
import OrchestratorOptions from '../options/orchestrator-options';
import CloudRunnerOptions from '../options/cloud-runner-options';
import UnityVersioning from '../../unity-versioning';
import BuildParameters from '../../build-parameters';
import Orchestrator from '../orchestrator';
import CloudRunner from '../cloud-runner';
async function CreateParameters(overrides: any) {
if (overrides) {
@@ -16,10 +16,10 @@ async function CreateParameters(overrides: any) {
return await BuildParameters.create();
}
describe('Orchestrator Locking Core', () => {
describe('Cloud Runner Locking Core', () => {
setups();
it('Responds', () => {});
if (OrchestratorOptions.orchestratorDebug) {
if (CloudRunnerOptions.cloudRunnerDebug) {
it(`Create Workspace`, async () => {
const overrides: any = {
versioning: 'None',
@@ -30,7 +30,7 @@ describe('Orchestrator Locking Core', () => {
maxRetainedWorkspaces: 3,
};
const buildParameters = await CreateParameters(overrides);
Orchestrator.buildParameters = buildParameters;
CloudRunner.buildParameters = buildParameters;
const newWorkspaceName = `test-workspace-${uuidv4()}`;
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
}, 150000);
@@ -45,7 +45,7 @@ describe('Orchestrator Locking Core', () => {
};
const runId = uuidv4();
const buildParameters = await CreateParameters(overrides);
Orchestrator.buildParameters = buildParameters;
CloudRunner.buildParameters = buildParameters;
const newWorkspaceName = `test-workspace-${uuidv4()}`;
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
@@ -63,7 +63,7 @@ describe('Orchestrator Locking Core', () => {
const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4();
Orchestrator.buildParameters = buildParameters;
CloudRunner.buildParameters = buildParameters;
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.HasWorkspaceLock(newWorkspaceName, runId, buildParameters)).toBeTruthy();
@@ -97,7 +97,7 @@ describe('Orchestrator Locking Core', () => {
await new Promise((promise) => setTimeout(promise, 1500));
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParameters);
const isBelowMax = await SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParameters);
OrchestratorLogger.log(`workspace ${element} locked:${isLocked} below max:${isBelowMax}`);
CloudRunnerLogger.log(`workspace ${element} locked:${isLocked} below max:${isBelowMax}`);
const lock = files.find((x) => {
return x.endsWith(`_lock`);
});

View File

@@ -1,11 +1,11 @@
import SharedWorkspaceLocking from '../services/core/shared-workspace-locking';
import { Cli } from '../../cli/cli';
import setups from './orchestrator-suite.test';
import setups from './cloud-runner-suite.test';
import { v4 as uuidv4 } from 'uuid';
import OrchestratorOptions from '../options/orchestrator-options';
import CloudRunnerOptions from '../options/cloud-runner-options';
import UnityVersioning from '../../unity-versioning';
import BuildParameters from '../../build-parameters';
import Orchestrator from '../orchestrator';
import CloudRunner from '../cloud-runner';
async function CreateParameters(overrides: any) {
if (overrides) {
@@ -15,10 +15,10 @@ async function CreateParameters(overrides: any) {
return await BuildParameters.create();
}
describe('Orchestrator Locking Get Locked Workspace', () => {
describe('Cloud Runner Locking Get Locked Workspace', () => {
setups();
it('Responds', () => {});
if (OrchestratorOptions.orchestratorDebug) {
if (CloudRunnerOptions.cloudRunnerDebug) {
it(`Get locked workspace From No Workspace`, async () => {
const overrides: any = {
versioning: 'None',
@@ -32,7 +32,7 @@ describe('Orchestrator Locking Get Locked Workspace', () => {
const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4();
Orchestrator.buildParameters = buildParameters;
CloudRunner.buildParameters = buildParameters;
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
}, 150000);
it(`Get locked workspace from unlocked`, async () => {
@@ -48,10 +48,10 @@ describe('Orchestrator Locking Get Locked Workspace', () => {
const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4();
Orchestrator.buildParameters = buildParameters;
CloudRunner.buildParameters = buildParameters;
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(Orchestrator.lockedWorkspace).toMatch(newWorkspaceName);
expect(CloudRunner.lockedWorkspace).toMatch(newWorkspaceName);
}, 300000);
it(`Get locked workspace from locked`, async () => {
const overrides: any = {
@@ -67,7 +67,7 @@ describe('Orchestrator Locking Get Locked Workspace', () => {
const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4();
const runId2 = uuidv4();
Orchestrator.buildParameters = buildParameters;
CloudRunner.buildParameters = buildParameters;
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.HasWorkspaceLock(newWorkspaceName, runId, buildParameters)).toBeTruthy();
@@ -75,7 +75,7 @@ describe('Orchestrator Locking Get Locked Workspace', () => {
expect(await SharedWorkspaceLocking.IsWorkspaceBelowMax(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId2, buildParameters)).toBeTruthy();
expect(Orchestrator.lockedWorkspace).not.toMatch(newWorkspaceName);
expect(CloudRunner.lockedWorkspace).not.toMatch(newWorkspaceName);
}, 300000);
it(`Get locked workspace after double lock and one unlock`, async () => {
const overrides: any = {
@@ -91,7 +91,7 @@ describe('Orchestrator Locking Get Locked Workspace', () => {
const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4();
const runId2 = uuidv4();
Orchestrator.buildParameters = buildParameters;
CloudRunner.buildParameters = buildParameters;
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
@@ -102,7 +102,7 @@ describe('Orchestrator Locking Get Locked Workspace', () => {
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId2, buildParameters)).toBeTruthy();
expect(Orchestrator.lockedWorkspace).not.toContain(newWorkspaceName);
expect(CloudRunner.lockedWorkspace).not.toContain(newWorkspaceName);
}, 300000);
it(`Get locked workspace after double lock and unlock`, async () => {
const overrides: any = {
@@ -118,7 +118,7 @@ describe('Orchestrator Locking Get Locked Workspace', () => {
const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4();
const runId2 = uuidv4();
Orchestrator.buildParameters = buildParameters;
CloudRunner.buildParameters = buildParameters;
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
@@ -130,7 +130,7 @@ describe('Orchestrator Locking Get Locked Workspace', () => {
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeFalsy();
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId2, buildParameters)).toBeTruthy();
expect(Orchestrator.lockedWorkspace).toContain(newWorkspaceName);
expect(CloudRunner.lockedWorkspace).toContain(newWorkspaceName);
}, 300000);
it(`Get locked workspace from unlocked was locked`, async () => {
const overrides: any = {
@@ -145,12 +145,12 @@ describe('Orchestrator Locking Get Locked Workspace', () => {
const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4();
Orchestrator.buildParameters = buildParameters;
CloudRunner.buildParameters = buildParameters;
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(Orchestrator.lockedWorkspace).toMatch(newWorkspaceName);
expect(CloudRunner.lockedWorkspace).toMatch(newWorkspaceName);
}, 300000);
}
});

View File

@@ -0,0 +1,48 @@
import CloudRunner from '../cloud-runner';
import { BuildParameters, ImageTag } from '../..';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../options/cloud-runner-options';
import setups from './cloud-runner-suite.test';
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
import { OptionValues } from 'commander';
async function CreateParameters(overrides: OptionValues | undefined) {
if (overrides) {
Cli.options = overrides;
}
return await BuildParameters.create();
}
describe('Cloud Runner pre-built S3 steps', () => {
it('Responds', () => {});
setups();
if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.providerStrategy !== `local-docker`) {
it('Run build and prebuilt s3 cache pull, cache push and upload build', async () => {
const overrides = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
containerHookFiles: `aws-s3-pull-cache,aws-s3-upload-cache,aws-s3-upload-build`,
};
const buildParameter2 = await CreateParameters(overrides);
const baseImage2 = new ImageTag(buildParameter2);
const results2Object = await CloudRunner.run(buildParameter2, baseImage2.toString());
const results2 = results2Object.BuildResults;
CloudRunnerLogger.log(`run 2 succeeded`);
const build2ContainsBuildSucceeded = results2.includes('Build succeeded');
expect(build2ContainsBuildSucceeded).toBeTruthy();
const results = await CloudRunnerSystem.RunAndReadLines(
`aws s3 ls s3://${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/`,
);
CloudRunnerLogger.log(results.join(`,`));
}, 1_000_000_000);
}
});

View File

@@ -1,7 +1,7 @@
import { Cli } from '../../cli/cli';
import GitHub from '../../github';
describe('Orchestrator', () => {
describe('Cloud Runner', () => {
it('Responds', () => {});
});

View File

@@ -0,0 +1,92 @@
import CloudRunner from '../../cloud-runner';
import { BuildParameters, ImageTag } from '../../..';
import UnityVersioning from '../../../unity-versioning';
import { Cli } from '../../../cli/cli';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import setups from '../cloud-runner-suite.test';
import * as fs from 'node:fs';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
async function CreateParameters(overrides: any) {
if (overrides) {
Cli.options = overrides;
}
return await BuildParameters.create();
}
describe('Cloud Runner Caching', () => {
it('Responds', () => {});
setups();
if (CloudRunnerOptions.cloudRunnerDebug) {
it('Run one build it should not use cache, run subsequent build which should use cache', async () => {
const overrides = {
versioning: 'None',
image: 'ubuntu',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
containerHookFiles: `debug-cache`,
cloudRunnerBranch: `cloud-runner-develop`,
};
if (CloudRunnerOptions.providerStrategy === `k8s`) {
overrides.containerHookFiles += `,aws-s3-pull-cache,aws-s3-upload-cache`;
}
const buildParameter = await CreateParameters(overrides);
expect(buildParameter.projectPath).toEqual(overrides.projectPath);
const baseImage = new ImageTag(buildParameter);
const resultsObject = await CloudRunner.run(buildParameter, baseImage.toString());
const results = resultsObject.BuildResults;
const libraryString = 'Rebuilding Library because the asset database could not be found!';
const cachePushFail = 'Did not push source folder to cache because it was empty Library';
const buildSucceededString = 'Build succeeded';
expect(results).toContain(libraryString);
expect(results).toContain(buildSucceededString);
expect(results).not.toContain(cachePushFail);
CloudRunnerLogger.log(`run 1 succeeded`);
if (CloudRunnerOptions.providerStrategy === `local-docker`) {
await CloudRunnerSystem.Run(`tree ./cloud-runner-cache/cache`);
await CloudRunnerSystem.Run(
`cp ./cloud-runner-cache/cache/${buildParameter.cacheKey}/Library/lib-${buildParameter.buildGuid}.tar ./`,
);
await CloudRunnerSystem.Run(`mkdir results`);
await CloudRunnerSystem.Run(`tar -xf lib-${buildParameter.buildGuid}.tar -C ./results`);
await CloudRunnerSystem.Run(`tree -d ./results`);
const cacheFolderExists = fs.existsSync(`cloud-runner-cache/cache/${overrides.cacheKey}`);
expect(cacheFolderExists).toBeTruthy();
}
const buildParameter2 = await CreateParameters(overrides);
buildParameter2.cacheKey = buildParameter.cacheKey;
const baseImage2 = new ImageTag(buildParameter2);
const results2Object = await CloudRunner.run(buildParameter2, baseImage2.toString());
const results2 = results2Object.BuildResults;
CloudRunnerLogger.log(`run 2 succeeded`);
const build2ContainsCacheKey = results2.includes(buildParameter.cacheKey);
const build2ContainsBuildSucceeded = results2.includes(buildSucceededString);
const build2NotContainsZeroLibraryCacheFilesMessage = !results2.includes(
'There is 0 files/dir in the cache pulled contents for Library',
);
const build2NotContainsZeroLFSCacheFilesMessage = !results2.includes(
'There is 0 files/dir in the cache pulled contents for LFS',
);
expect(build2ContainsCacheKey).toBeTruthy();
expect(results2).toContain('Activation successful');
expect(build2ContainsBuildSucceeded).toBeTruthy();
expect(results2).toContain(buildSucceededString);
const splitResults = results2.split('Activation successful');
expect(splitResults[splitResults.length - 1]).not.toContain(libraryString);
expect(build2NotContainsZeroLibraryCacheFilesMessage).toBeTruthy();
expect(build2NotContainsZeroLFSCacheFilesMessage).toBeTruthy();
}, 1_000_000_000);
}
});

Some files were not shown because too many files have changed in this diff Show More