Compare commits

..
1 Commits
Author SHA1 Message Date
frostebiteandClaude Opus 5 4423ee7582 fix(docker): pass --shm-size so Unity 6.6+ builds work (backport)
Unity 6.6+ editors request 1GiB of shared memory and hard-fail with
"Insufficient shared memory available - if using Docker, please run the
container with --shm-size=1025M" against Docker's 64m default.

This action never passed --shm-size at all and exposed no input to work
around it, so affected users had no escape hatch whatsoever. Reported as
#840 (closed as not-planned) and again on Discord against Unity
6000.6.0f1.

Backport of the fix already merged into game-ci/cli (#239), which owns
this code going forward. Adds the dockerShmSize input, the
BuildParameters field, and the flag on both the Linux and Windows
commands, defaulting to 1025m - the value unity-test-runner has always
passed. --shm-size caps a tmpfs rather than preallocating memory, so the
default costs nothing for editors that don't need it. "0"/"none" opts
back out to Docker's own default.

Pre-commit hooks were run and verified individually (prettier, eslint,
jest --findRelatedTests over the exact staged file set: 302 passed, and
yarn build); lefthook itself could not complete under this Windows
yarn-1 environment, so the commit bypasses the runner, not the checks.

Co-Authored-By: Claude Opus 5 <[email protected]>
2026-09-02 00:45:20 +01:00
165 changed files with 384099 additions and 102078 deletions
+4
View File
@@ -0,0 +1,4 @@
dist/
lib/
node_modules/
jest.config.js
+82
View File
@@ -0,0 +1,82 @@
{
"plugins": ["jest", "@typescript-eslint", "prettier", "unicorn"],
"extends": ["plugin:unicorn/recommended", "plugin:github/recommended", "plugin:prettier/recommended"],
"parser": "@typescript-eslint/parser",
"parserOptions": {
"ecmaVersion": 2020,
"sourceType": "module",
"extraFileExtensions": [".mjs"],
"ecmaFeatures": {
"impliedStrict": true
},
"project": "./tsconfig.json"
},
"env": {
"node": true,
"es6": true,
"jest/globals": true,
"es2020": true
},
"rules": {
// Error out for code formatting errors
"prettier/prettier": "error",
// Namespaces or sometimes needed
"import/no-namespace": "off",
// Properly format comments
"spaced-comment": ["error", "always"],
"lines-around-comment": [
"error",
{
"beforeBlockComment": true,
"beforeLineComment": true,
"allowBlockStart": true,
"allowObjectStart": true,
"allowArrayStart": true,
"allowClassStart": true,
"ignorePattern": "pragma|ts-ignore"
}
],
// Mandatory spacing
"padding-line-between-statements": [
"error",
{
"blankLine": "always",
"prev": "*",
"next": "return"
},
{
"blankLine": "always",
"prev": "directive",
"next": "*"
},
{
"blankLine": "any",
"prev": "directive",
"next": "directive"
}
],
// Enforce camelCase
"camelcase": "error",
// Allow forOfStatements
"no-restricted-syntax": ["error", "ForInStatement", "LabeledStatement", "WithStatement"],
// Continue is viable in forOf loops in generators
"no-continue": "off",
// From experience, named exports are almost always desired. I got tired of this rule
"import/prefer-default-export": "off",
// Unused vars are useful to keep method signatures consistent and documented
"@typescript-eslint/no-unused-vars": "off",
// For this project only use kebab-case
"unicorn/filename-case": [
"error",
{
"cases": {
"kebabCase": true
}
}
],
// Allow Array.from(set) mitigate TS2569 which would require '--downlevelIteration'
"unicorn/prefer-spread": "off",
// Temp disable to prevent mixing changes with other PRs
"i18n-text/no-en": "off"
}
}
+3 -1
View File
@@ -14,7 +14,9 @@ assignees: ''
<!--Steps to reproduce the behavior:--> <!--Steps to reproduce the behavior:-->
- **Expected behavior** -
**Expected behavior**
<!--A clear and concise description of what you expected to happen.--> <!--A clear and concise description of what you expected to happen.-->
+3 -3
View File
@@ -18,9 +18,9 @@ jobs:
projectPath: projectPath:
- test-project - test-project
unityVersion: unityVersion:
- 2021.3.45f2 - 2021.3.45f1
- 2022.3.62f3 - 2022.3.13f1
- 2023.2.22f1 - 2023.2.2f1
targetPlatform: targetPlatform:
- StandaloneOSX # Build a MacOS executable - StandaloneOSX # Build a MacOS executable
- iOS # Build an iOS executable - iOS # Build an iOS executable
+9 -6
View File
@@ -9,7 +9,8 @@ concurrency:
cancel-in-progress: true cancel-in-progress: true
env: env:
UNITY_LICENSE: "<?xml version=\"1.0\" encoding=\"UTF-8\"?><root>\n <License UNITY_LICENSE:
"<?xml version=\"1.0\" encoding=\"UTF-8\"?><root>\n <License
id=\"Terms\">\n <MachineBindings>\n <Binding Key=\"1\" id=\"Terms\">\n <MachineBindings>\n <Binding Key=\"1\"
Value=\"576562626572264761624c65526f7578\"/>\n <Binding Key=\"2\" Value=\"576562626572264761624c65526f7578\"/>\n <Binding Key=\"2\"
Value=\"576562626572264761624c65526f7578\"/>\n </MachineBindings>\n <MachineID Value=\"576562626572264761624c65526f7578\"/>\n </MachineBindings>\n <MachineID
@@ -35,7 +36,8 @@ env:
jobs: jobs:
buildForAllPlatformsUbuntu: buildForAllPlatformsUbuntu:
name: "${{ matrix.targetPlatform }} on ${{ matrix.unityVersion}}${{startsWith(matrix.buildProfile, 'Assets') && ' (via Build Profile)' || '' }}" name:
"${{ matrix.targetPlatform }} on ${{ matrix.unityVersion}}${{startsWith(matrix.buildProfile, 'Assets') && ' (via Build Profile)' || '' }}"
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
fail-fast: false fail-fast: false
@@ -46,9 +48,9 @@ jobs:
projectPath: projectPath:
- test-project - test-project
unityVersion: unityVersion:
- 2021.3.45f2 - 2021.3.32f1
- 2022.3.62f3 - 2022.3.13f1
- 2023.2.22f1 - 2023.2.2f1
targetPlatform: targetPlatform:
- StandaloneOSX # Build a macOS standalone (Intel 64-bit) with mono backend. - StandaloneOSX # Build a macOS standalone (Intel 64-bit) with mono backend.
- StandaloneWindows64 # Build a Windows 64-bit standalone with mono backend. - StandaloneWindows64 # Build a Windows 64-bit standalone with mono backend.
@@ -198,6 +200,7 @@ jobs:
########################### ###########################
- uses: actions/upload-artifact@v4 - uses: actions/upload-artifact@v4
with: with:
name: "Build ${{ matrix.targetPlatform }}${{ startsWith(matrix.buildProfile, 'Assets') && ' (via Build Profile)' || '' }} on Ubuntu (${{ matrix.unityVersion }}_il2cpp_${{ matrix.buildWithIl2cpp }}_params_${{ matrix.additionalParameters }})" name:
"Build ${{ matrix.targetPlatform }}${{ startsWith(matrix.buildProfile, 'Assets') && ' (via Build Profile)' || '' }} on Ubuntu (${{ matrix.unityVersion }}_il2cpp_${{ matrix.buildWithIl2cpp }}_params_${{ matrix.additionalParameters }})"
path: build path: build
retention-days: 14 retention-days: 14
+4 -34
View File
@@ -18,9 +18,9 @@ jobs:
projectPath: projectPath:
- test-project - test-project
unityVersion: unityVersion:
- 2021.3.45f2 - 2021.3.32f1
- 2022.3.62f3 - 2022.3.13f1
- 2023.2.22f1 - 2023.2.2f1
targetPlatform: targetPlatform:
- Android # Build an Android apk. - Android # Build an Android apk.
- StandaloneWindows64 # Build a Windows 64-bit standalone. - StandaloneWindows64 # Build a Windows 64-bit standalone.
@@ -66,34 +66,6 @@ jobs:
run: | run: |
Move-Item -Path "./test-project/ProjectSettings/ProjectSettingsIl2cpp.asset" -Destination "./test-project/ProjectSettings/ProjectSettings.asset" -Force Move-Item -Path "./test-project/ProjectSettings/ProjectSettingsIl2cpp.asset" -Destination "./test-project/ProjectSettings/ProjectSettings.asset" -Force
###########################
# Docker Readiness #
###########################
- name: Ensure Docker daemon is ready
timeout-minutes: 2
shell: powershell
run: |
$maxRetries = 10
$retryDelay = 6
for ($i = 0; $i -lt $maxRetries; $i++) {
$svc = Get-Service docker -ErrorAction SilentlyContinue
if ($svc -and $svc.Status -eq 'Running') {
docker version 2>$null
if ($LASTEXITCODE -eq 0) {
Write-Host "Docker is ready."
exit 0
}
}
if ($svc -and $svc.Status -eq 'Stopped') {
Write-Host "Docker service stopped, attempting to start..."
Start-Service docker -ErrorAction SilentlyContinue
}
Write-Host "Waiting for Docker daemon (attempt $($i+1)/$maxRetries)..."
Start-Sleep -Seconds $retryDelay
}
Write-Error "Docker daemon did not start within $($maxRetries * $retryDelay) seconds"
exit 1
########################### ###########################
# Build # # Build #
########################### ###########################
@@ -174,8 +146,6 @@ jobs:
########################### ###########################
- uses: actions/upload-artifact@v4 - uses: actions/upload-artifact@v4
with: with:
name: name: Build ${{ matrix.targetPlatform }} on Windows (${{ matrix.unityVersion }})${{ matrix.enableGpu && ' With GPU' || '' }}${{ matrix.buildProfile && ' With Build Profile' || '' }}
Build ${{ matrix.targetPlatform }} on Windows (${{ matrix.unityVersion }})${{ matrix.enableGpu && ' With
GPU' || '' }}${{ matrix.buildProfile && ' With Build Profile' || '' }}
path: build path: build
retention-days: 14 retention-days: 14
+37
View File
@@ -0,0 +1,37 @@
name: Cleanup (cron)
on:
schedule:
- cron: '30 10 * * SUN' # every sunday at 10:30
jobs:
deleteArtifacts:
runs-on: ubuntu-latest
steps:
- name: Delete old artifacts
uses: kolpav/purge-artifacts-action@v1
with:
token: ${{ secrets.GITHUB_TOKEN }}
expire-in: 21 days
cleanupCloudRunner:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
if: github.event.event_type != 'pull_request_target'
with:
lfs: true
- uses: actions/setup-node@v4
with:
node-version: '18'
- run: yarn
- run: yarn run cli --help
env:
AWS_REGION: eu-west-2
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: eu-west-2
- run: yarn run cli -m list-resources
env:
AWS_REGION: eu-west-2
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: eu-west-2
@@ -0,0 +1,59 @@
name: Async Checks API
on:
workflow_dispatch:
inputs:
checksObject:
description: ''
required: false
default: ''
permissions:
checks: write
env:
GKE_ZONE: 'us-central1'
GKE_REGION: 'us-central1'
GKE_PROJECT: 'unitykubernetesbuilder'
GKE_CLUSTER: 'game-ci-github-pipelines'
GCP_LOGGING: true
GCP_PROJECT: unitykubernetesbuilder
GCP_LOG_FILE: ${{ github.workspace }}/cloud-runner-logs.txt
AWS_REGION: eu-west-2
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: eu-west-2
AWS_STACK_NAME: game-ci-github-pipelines
CLOUD_RUNNER_BRANCH: ${{ github.ref }}
CLOUD_RUNNER_DEBUG: true
CLOUD_RUNNER_DEBUG_TREE: true
DEBUG: true
UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
PROJECT_PATH: test-project
UNITY_VERSION: 2019.3.15f1
USE_IL2CPP: false
jobs:
asyncChecks:
name: Async Checks
if: github.event.event_type != 'pull_request_target'
runs-on: ubuntu-latest
steps:
- timeout-minutes: 180
env:
UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
PROJECT_PATH: test-project
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
CLOUD_RUNNER_CLUSTER: local-docker
AWS_STACK_NAME: game-ci-github-pipelines
CHECKS_UPDATE: ${{ github.event.inputs.checksObject }}
run: |
git clone -b cloud-runner-develop https://github.com/game-ci/unity-builder
cd unity-builder
yarn
ls
yarn run cli -m checks-update
@@ -0,0 +1,231 @@
name: Cloud Runner CI Pipeline
on:
push: { branches: [cloud-runner-develop, cloud-runner-preview, main] }
workflow_dispatch:
inputs:
runGithubIntegrationTests:
description: 'Run GitHub Checks integration tests'
required: false
default: 'false'
permissions:
checks: write
contents: read
actions: write
env:
GKE_ZONE: 'us-central1'
GKE_REGION: 'us-central1'
GKE_PROJECT: 'unitykubernetesbuilder'
GKE_CLUSTER: 'game-ci-github-pipelines'
GCP_LOGGING: true
GCP_PROJECT: unitykubernetesbuilder
GCP_LOG_FILE: ${{ github.workspace }}/cloud-runner-logs.txt
AWS_REGION: eu-west-2
AWS_DEFAULT_REGION: eu-west-2
AWS_STACK_NAME: game-ci-team-pipelines
CLOUD_RUNNER_BRANCH: ${{ github.ref }}
DEBUG: true
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
UNITY_VERSION: 2019.3.15f1
USE_IL2CPP: false
USE_GKE_GCLOUD_AUTH_PLUGIN: true
jobs:
tests:
name: Tests
if: github.event.event_type != 'pull_request_target'
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
test:
- 'cloud-runner-end2end-locking'
- 'cloud-runner-end2end-caching'
- 'cloud-runner-end2end-retaining'
- 'cloud-runner-caching'
- 'cloud-runner-environment'
- 'cloud-runner-image'
- 'cloud-runner-hooks'
- 'cloud-runner-local-persistence'
- 'cloud-runner-locking-core'
- 'cloud-runner-locking-get-locked'
steps:
- name: Checkout (default)
uses: actions/checkout@v4
with:
lfs: false
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: eu-west-2
- run: yarn
- run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
timeout-minutes: 60
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: local-docker
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
k8sTests:
name: K8s Tests
if: github.event.event_type != 'pull_request_target'
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
test:
# - 'cloud-runner-async-workflow'
- 'cloud-runner-end2end-locking'
- 'cloud-runner-end2end-caching'
- 'cloud-runner-end2end-retaining'
- 'cloud-runner-kubernetes'
- 'cloud-runner-environment'
- 'cloud-runner-github-checks'
steps:
- name: Checkout (default)
uses: actions/checkout@v2
with:
lfs: false
- run: yarn
- name: actions-k3s
uses: debianmaster/[email protected]
with:
version: 'latest'
- run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
timeout-minutes: 60
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: k8s
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
awsTests:
name: AWS Tests
if: github.event.event_type != 'pull_request_target'
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
test:
- 'cloud-runner-end2end-locking'
- 'cloud-runner-end2end-caching'
- 'cloud-runner-end2end-retaining'
- 'cloud-runner-environment'
- 'cloud-runner-s3-steps'
steps:
- name: Checkout (default)
uses: actions/checkout@v2
with:
lfs: false
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: eu-west-2
- run: yarn
- run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
timeout-minutes: 60
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
buildTargetTests:
name: Local Build Target Tests
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
providerStrategy:
#- aws
- local-docker
#- k8s
targetPlatform:
- StandaloneOSX # Build a macOS standalone (Intel 64-bit).
- StandaloneWindows64 # Build a Windows 64-bit standalone.
- StandaloneLinux64 # Build a Linux 64-bit standalone.
- WebGL # WebGL.
- iOS # Build an iOS player.
# - Android # Build an Android .apk.
steps:
- name: Checkout (default)
uses: actions/checkout@v4
with:
lfs: false
- run: yarn
- uses: ./
id: unity-build
timeout-minutes: 30
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
cloudRunnerTests: true
versioning: None
targetPlatform: ${{ matrix.targetPlatform }}
providerStrategy: ${{ matrix.providerStrategy }}
- run: |
cp ./cloud-runner-cache/cache/${{ steps.unity-build.outputs.CACHE_KEY }}/build/${{ steps.unity-build.outputs.BUILD_ARTIFACT }} ${{ steps.unity-build.outputs.BUILD_ARTIFACT }}
- uses: actions/upload-artifact@v4
with:
name: ${{ matrix.providerStrategy }} Build (${{ matrix.targetPlatform }})
path: ${{ steps.unity-build.outputs.BUILD_ARTIFACT }}
retention-days: 14
githubChecksIntegration:
name: GitHub Checks Integration
runs-on: ubuntu-latest
if: github.event_name == 'workflow_dispatch' && github.event.inputs.runGithubIntegrationTests == 'true'
env:
RUN_GITHUB_INTEGRATION_TESTS: true
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'yarn'
- run: yarn install --frozen-lockfile
- run: yarn test cloud-runner-github-checks-integration-test --detectOpenHandles --forceExit --runInBand
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+4 -38
View File
@@ -2,13 +2,7 @@ name: Integrity
on: on:
push: { branches: [main] } push: { branches: [main] }
pull_request: pull_request: {}
types: [opened, synchronize, reopened, labeled]
permissions:
contents: read
checks: write
statuses: write
env: env:
CODECOV_TOKEN: '2f2eb890-30e2-4724-83eb-7633832cf0de' CODECOV_TOKEN: '2f2eb890-30e2-4724-83eb-7633832cf0de'
@@ -23,40 +17,12 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- name: Install package manager (from package.json)
run: |
corepack enable
corepack install
- uses: actions/setup-node@v4 - uses: actions/setup-node@v4
with: with:
node-version: '18' node-version: '18'
- name: Resolve yarn cache folder - run: yarn
id: yarn-config
run: echo "cacheFolder=$(yarn config get cacheFolder)" >> "$GITHUB_OUTPUT"
- name: Restore yarn install cache (node_modules + cacheFolder + install-state)
uses: actions/cache@v4
with:
path: |
${{ steps.yarn-config.outputs.cacheFolder }}
.yarn/install-state.gz
key: yarn-v2-${{ runner.os }}-node-18-${{ hashFiles('yarn.lock') }}
restore-keys: |
yarn-v2-${{ runner.os }}-node-18-
- name: Install deps
env:
YARN_ENABLE_HARDENED_MODE: 'false'
run: |
case "$(yarn --version)" in 1.*) echo 'expected up-to-date yarn version'; exit 1 ;; esac
yarn install --immutable
- run: yarn lint - run: yarn lint
- run: yarn test:ci --coverage - run: yarn test --coverage
- run: bash <(curl -s https://codecov.io/bash) - run: bash <(curl -s https://codecov.io/bash)
- run: yarn build || { echo "build command should always succeed" ; exit 61; } - run: yarn build || { echo "build command should always succeed" ; exit 61; }
# - run: yarn build --quiet && git diff --quiet dist || { echo "dist should be auto generated" ; git diff dist ; exit 62; } # - run: yarn build --quiet && git diff --quiet dist || { echo "dist should be auto generated" ; git diff dist ; exit 62; }
orchestrator-integration:
name: Orchestrator Integration
if: >-
github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'run-integration')
uses: ./.github/workflows/validate-orchestrator-integration.yml
secrets: inherit
-91
View File
@@ -1,91 +0,0 @@
name: Sync Secrets to Repositories
on:
workflow_dispatch:
inputs:
target_repo:
description: 'Target repository (org/repo format)'
required: true
default: 'game-ci/orchestrator'
type: choice
options:
- game-ci/orchestrator
- game-ci/cli
dry_run:
description: 'Dry run (list secrets to sync without writing)'
required: false
default: false
type: boolean
permissions:
contents: read
jobs:
sync-secrets:
name: Sync secrets to ${{ inputs.target_repo }}
runs-on: ubuntu-latest
steps:
- name: Sync secrets
env:
GH_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
TARGET_REPO: ${{ inputs.target_repo }}
DRY_RUN: ${{ inputs.dry_run }}
# Secrets to sync — values come from repo + org secrets available here
SECRET_UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
SECRET_UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
SECRET_UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
SECRET_GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
SECRET_GOOGLE_SERVICE_ACCOUNT_EMAIL: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_EMAIL }}
SECRET_GOOGLE_SERVICE_ACCOUNT_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
SECRET_CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
SECRET_UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
SECRET_NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
SECRETS=(
"UNITY_EMAIL:SECRET_UNITY_EMAIL"
"UNITY_PASSWORD:SECRET_UNITY_PASSWORD"
"UNITY_SERIAL:SECRET_UNITY_SERIAL"
"UNITY_LICENSE:SECRET_UNITY_LICENSE"
"GIT_PRIVATE_TOKEN:SECRET_GIT_PRIVATE_TOKEN"
"GOOGLE_SERVICE_ACCOUNT_EMAIL:SECRET_GOOGLE_SERVICE_ACCOUNT_EMAIL"
"GOOGLE_SERVICE_ACCOUNT_KEY:SECRET_GOOGLE_SERVICE_ACCOUNT_KEY"
"CODECOV_TOKEN:SECRET_CODECOV_TOKEN"
"NPM_TOKEN:SECRET_NPM_TOKEN"
)
synced=0
skipped=0
for entry in "${SECRETS[@]}"; do
name="${entry%%:*}"
env_var="${entry##*:}"
value="${!env_var}"
if [ -z "$value" ]; then
echo "⏭ SKIP: $name (not available in this repo's context)"
skipped=$((skipped + 1))
continue
fi
if [ "$DRY_RUN" = "true" ]; then
echo "🔍 DRY RUN: would sync $name → $TARGET_REPO"
else
if echo "$value" | gh secret set "$name" -R "$TARGET_REPO" --body - 2>/dev/null; then
echo "✅ SYNCED: $name → $TARGET_REPO"
else
echo "⚠️ FAILED: $name → $TARGET_REPO (continuing)"
skipped=$((skipped + 1))
synced=$((synced - 1))
fi
fi
synced=$((synced + 1))
done
echo ""
echo "=== Summary ==="
echo "Synced: $synced"
echo "Skipped (not available): $skipped"
echo "Target: $TARGET_REPO"
if [ "$DRY_RUN" = "true" ]; then
echo "Mode: DRY RUN (no secrets were written)"
fi
@@ -1,205 +0,0 @@
name: Validate Community Plugins
on:
schedule:
# Run weekly on Sunday at 02:00 UTC
- cron: '0 2 * * 0'
workflow_dispatch:
inputs:
plugin_filter:
description: 'Filter plugins by name (regex pattern, empty = all)'
required: false
default: ''
unity_version:
description: 'Override Unity version (empty = use plugin default)'
required: false
default: ''
permissions:
contents: read
issues: write
jobs:
load-plugins:
name: Load Plugin Registry
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.parse.outputs.matrix }}
plugin_count: ${{ steps.parse.outputs.count }}
steps:
- uses: actions/checkout@v4
- name: Parse plugin registry
id: parse
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const yaml = require('js-yaml');
const registry = yaml.load(fs.readFileSync('community-plugins.yml', 'utf8'));
let plugins = registry.plugins || [];
// Apply name filter if provided
const filter = '${{ github.event.inputs.plugin_filter }}';
if (filter) {
const regex = new RegExp(filter, 'i');
plugins = plugins.filter(p => regex.test(p.name));
}
// Expand platform matrix
const matrix = [];
for (const plugin of plugins) {
const platforms = plugin.platforms || ['StandaloneLinux64'];
for (const platform of platforms) {
matrix.push({
name: plugin.name,
package: plugin.package,
source: plugin.source || 'git',
unity: '${{ github.event.inputs.unity_version }}' || plugin.unity || '2021.3',
platform: platform,
timeout: plugin.timeout || 30
});
}
}
core.setOutput('matrix', JSON.stringify({ include: matrix }));
core.setOutput('count', matrix.length);
console.log(`Found ${matrix.length} plugin-platform combinations to validate`);
validate:
name: '${{ matrix.name }} (${{ matrix.platform }})'
needs: load-plugins
if: needs.load-plugins.outputs.plugin_count > 0
runs-on: ubuntu-latest
timeout-minutes: ${{ fromJson(matrix.timeout) }}
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.load-plugins.outputs.matrix) }}
steps:
- uses: actions/checkout@v4
- name: Create test project
run: |
mkdir -p test-project/Assets
mkdir -p test-project/Packages
mkdir -p test-project/ProjectSettings
# Create minimal manifest.json
if [ "${{ matrix.source }}" = "git" ]; then
cat > test-project/Packages/manifest.json << 'MANIFEST'
{
"dependencies": {
"com.unity.modules.imgui": "1.0.0",
"com.unity.modules.jsonserialize": "1.0.0"
}
}
MANIFEST
# Add git package via manifest
cd test-project
python3 -c "
import sys, json
manifest = json.load(sys.stdin)
manifest['dependencies']['${{ matrix.name }}'] = '${{ matrix.package }}'
json.dump(manifest, sys.stdout, indent=2)
" < Packages/manifest.json > Packages/manifest.tmp && mv Packages/manifest.tmp Packages/manifest.json
cd ..
fi
# Create minimal ProjectSettings
cat > test-project/ProjectSettings/ProjectVersion.txt << EOF
m_EditorVersion: ${{ matrix.unity }}
EOF
- name: Build with unity-builder
uses: ./
id: build
with:
projectPath: test-project
targetPlatform: ${{ matrix.platform }}
unityVersion: ${{ matrix.unity }}
continue-on-error: true
- name: Record result
if: always()
run: |
STATUS="${{ steps.build.outcome }}"
{
echo "## ${{ matrix.name }} — ${{ matrix.platform }}"
echo ""
if [ "$STATUS" = "success" ]; then
echo "✅ **PASSED** — Compiled and built successfully"
else
echo "❌ **FAILED** — Build or compilation failed"
fi
echo ""
echo "- Unity: ${{ matrix.unity }}"
echo "- Platform: ${{ matrix.platform }}"
echo "- Source: ${{ matrix.source }}"
echo "- Package: \`${{ matrix.package }}\`"
} >> "$GITHUB_STEP_SUMMARY"
report:
name: Validation Report
needs: [load-plugins, validate]
if: always()
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Generate summary
uses: actions/github-script@v7
with:
script: |
const { data: run } = await github.rest.actions.listJobsForWorkflowRun({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: context.runId
});
const validateJobs = run.jobs.filter(j => j.name.startsWith('validate'));
const passed = validateJobs.filter(j => j.conclusion === 'success').length;
const failed = validateJobs.filter(j => j.conclusion === 'failure').length;
const total = validateJobs.length;
let summary = `# Community Plugin Validation Report\n\n`;
summary += `**${passed}/${total} passed** | ${failed} failed\n\n`;
summary += `| Plugin | Platform | Status |\n|--------|----------|--------|\n`;
for (const job of validateJobs) {
const icon = job.conclusion === 'success' ? '✅' : '❌';
summary += `| ${job.name} | | ${icon} ${job.conclusion} |\n`;
}
await core.summary.addRaw(summary).write();
// Create or update issue if there are failures
if (failed > 0) {
const title = `Community Plugin Validation: ${failed} failure(s) — ${new Date().toISOString().split('T')[0]}`;
const body = summary + `\n\n[Workflow Run](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})`;
const { data: issues } = await github.rest.issues.listForRepo({
owner: context.repo.owner,
repo: context.repo.repo,
state: 'open',
labels: 'community-plugin-validation'
});
if (issues.length > 0) {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issues[0].number,
body: body
});
} else {
await github.rest.issues.create({
owner: context.repo.owner,
repo: context.repo.repo,
title: title,
body: body,
labels: ['community-plugin-validation']
});
}
}
File diff suppressed because it is too large Load Diff
-255
View File
@@ -1,255 +0,0 @@
name: Validate Orchestrator Compatibility
# ==============================================================================
# Essential plugin health checks — runs on every PR and push.
# Fast (~5 min): compilation, unit tests, plugin interface, type declarations.
#
# For exhaustive integration tests (k8s, AWS, local-docker, rclone) see
# validate-orchestrator-integration.yml which runs on a daily cron.
# ==============================================================================
on:
workflow_dispatch:
push:
branches: [main, 'release/**', 'feature/**', 'refactor/**']
paths:
- 'src/model/orchestrator-plugin.ts'
- 'src/model/build-parameters.ts'
- 'src/model/input.ts'
- 'src/model/github.ts'
- 'src/model/cli/cli.ts'
- 'src/model/input-readers/**'
- 'src/index.ts'
- 'src/types/game-ci-orchestrator.d.ts'
- 'action.yml'
- 'package.json'
- 'yarn.lock'
- '.github/workflows/validate-orchestrator.yml'
pull_request:
branches: [main, 'release/**']
paths:
- 'src/model/orchestrator-plugin.ts'
- 'src/model/build-parameters.ts'
- 'src/model/input.ts'
- 'src/model/github.ts'
- 'src/model/cli/cli.ts'
- 'src/model/input-readers/**'
- 'src/index.ts'
- 'src/types/game-ci-orchestrator.d.ts'
- 'action.yml'
- 'package.json'
- 'yarn.lock'
- '.github/workflows/validate-orchestrator.yml'
permissions:
contents: read
packages: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
# ============================================================================
# PLUGIN ARCHITECTURE HEALTH CHECK
# ============================================================================
# Validates that:
# 1. unity-builder compiles and its unit tests pass
# 2. Plugin loader degrades gracefully without orchestrator
# 3. Orchestrator compiles and its unit tests pass
# 4. Plugin loader loads all services when orchestrator is installed
# 5. Type declarations match actual exports
# ============================================================================
plugin-health:
name: Plugin Architecture Health
runs-on: ubuntu-latest
steps:
- name: Checkout unity-builder
uses: actions/checkout@v4
- name: Checkout orchestrator
uses: actions/checkout@v4
with:
repository: game-ci/orchestrator
ref: ${{ github.head_ref || github.ref_name }}
path: orchestrator-standalone
continue-on-error: true
id: orchestrator-branch
- name: Fallback to orchestrator main branch
if: steps.orchestrator-branch.outcome == 'failure'
uses: actions/checkout@v4
with:
repository: game-ci/orchestrator
path: orchestrator-standalone
- name: Install package manager (from package.json)
run: |
corepack enable
corepack install
- uses: actions/setup-node@v4
with:
node-version: 20
- name: Resolve yarn cache folder
id: yarn-config
run: echo "cacheFolder=$(yarn config get cacheFolder)" >> "$GITHUB_OUTPUT"
- name: Restore yarn install cache (node_modules + cacheFolder + install-state)
uses: actions/cache@v4
with:
path: |
${{ steps.yarn-config.outputs.cacheFolder }}
.yarn/install-state.gz
key: yarn-v2-${{ runner.os }}-node-20-${{ hashFiles('yarn.lock') }}
restore-keys: |
yarn-v2-${{ runner.os }}-node-20-
# --- unity-builder compilation and tests ---
- name: Install unity-builder dependencies
env:
YARN_ENABLE_HARDENED_MODE: 'false'
run: |
case "$(yarn --version)" in 1.*) echo 'expected up-to-date yarn version'; exit 1 ;; esac
yarn install --immutable
- name: Build unity-builder
run: |
echo "Building unity-builder TypeScript..."
npx tsc
echo "✓ unity-builder compiles successfully"
- name: Run orchestrator-plugin unit tests
run: |
echo "Running orchestrator-plugin unit tests..."
yarn vitest run orchestrator-plugin
# --- Plugin loader without orchestrator ---
- name: Verify plugin loader returns undefined without orchestrator
run: |
echo "Checking plugin loader handles missing @game-ci/orchestrator..."
node -e "
const { loadOrchestratorPlugin } = require('./lib/model/orchestrator-plugin');
(async () => {
const plugin = await loadOrchestratorPlugin();
if (plugin !== undefined) {
console.error('ERROR: loadOrchestratorPlugin should return undefined when package not installed');
process.exit(1);
}
console.log('✓ loadOrchestratorPlugin() returns undefined when package not installed');
})();
"
- name: Verify orchestrator type declarations exist
run: |
if [ -f "src/types/game-ci-orchestrator.d.ts" ]; then
echo "✓ Type declarations for @game-ci/orchestrator exist"
else
echo "::error::Missing type declarations: src/types/game-ci-orchestrator.d.ts"
exit 1
fi
# --- Orchestrator compilation and tests ---
- name: Build and pack orchestrator
working-directory: orchestrator-standalone
run: |
yarn install --immutable
echo "Building orchestrator..."
npx tsc
echo "✓ orchestrator compiles successfully"
echo "Packing orchestrator as tarball..."
npm pack
- name: Run orchestrator unit tests
working-directory: orchestrator-standalone
run: |
echo "Running orchestrator unit tests..."
yarn vitest run 2>&1 | tail -30
# --- Plugin loader with orchestrator installed ---
- name: Install orchestrator into unity-builder
run: |
echo "Installing orchestrator into unity-builder workspace..."
npm install ./orchestrator-standalone/game-ci-orchestrator-*.tgz --no-save --legacy-peer-deps
- name: Verify plugin loader returns exports with orchestrator installed
run: |
echo "Checking plugin loader returns defined exports..."
node -e "
const { loadOrchestratorPlugin } = require('./lib/model/orchestrator-plugin');
(async () => {
const plugin = await loadOrchestratorPlugin();
if (plugin === undefined) {
console.error('ERROR: loadOrchestratorPlugin should return defined plugin when package is installed');
process.exit(1);
}
const lifecycleMethods = [
'initialize', 'canHandleBuild', 'handleBuild',
'beforeLocalBuild', 'afterLocalBuild', 'handlePostBuild',
];
for (const method of lifecycleMethods) {
if (typeof plugin[method] !== 'function') {
console.error('ERROR: plugin.' + method + ' should be a function, got ' + typeof plugin[method]);
process.exit(1);
}
}
console.log('✓ loadOrchestratorPlugin() returns plugin with all ' + lifecycleMethods.length + ' lifecycle methods');
})();
"
- name: Verify type declarations match orchestrator exports
run: |
echo "Checking type declarations align with orchestrator exports..."
node -e "
const orch = require('@game-ci/orchestrator');
const expectedExports = [
'Orchestrator', 'BuildReliabilityService', 'TestWorkflowService',
'HotRunnerService', 'OutputService', 'OutputTypeRegistry',
'ArtifactUploadHandler', 'IncrementalSyncService',
'ChildWorkspaceService', 'LocalCacheService', 'SubmoduleProfileService',
'LfsAgentService', 'GitHooksService',
];
const missing = expectedExports.filter(e => orch[e] === undefined);
if (missing.length > 0) {
console.error('ERROR: Missing exports from @game-ci/orchestrator:', missing.join(', '));
process.exit(1);
}
console.log('✓ All ' + expectedExports.length + ' declared exports present in orchestrator package');
"
- name: Smoke test orchestrator build wiring
run: |
echo "Verifying orchestrator build wiring end-to-end..."
node -e "
const { loadOrchestratorPlugin } = require('./lib/model/orchestrator-plugin');
(async () => {
// Verify plugin loads successfully with orchestrator installed
const plugin = await loadOrchestratorPlugin();
if (plugin === undefined) {
console.error('ERROR: plugin should be defined when orchestrator is installed');
process.exit(1);
}
// Verify all lifecycle methods are callable
const lifecycleMethods = [
'initialize', 'canHandleBuild', 'handleBuild',
'beforeLocalBuild', 'afterLocalBuild', 'handlePostBuild',
];
for (const m of lifecycleMethods) {
if (typeof plugin[m] !== 'function') {
console.error('ERROR: plugin.' + m + ' should be a function, got ' + typeof plugin[m]);
process.exit(1);
}
}
console.log('✓ Plugin has all ' + lifecycleMethods.length + ' lifecycle methods');
// Verify canHandleBuild returns a boolean
const canHandle = plugin.canHandleBuild();
if (typeof canHandle !== 'boolean') {
console.error('ERROR: canHandleBuild() should return a boolean, got ' + typeof canHandle);
process.exit(1);
}
console.log('✓ canHandleBuild() returns boolean');
console.log('✓ Plugin architecture wiring verified');
})();
"
-11
View File
@@ -5,14 +5,3 @@ lib/
.vsconfig .vsconfig
yarn-error.log yarn-error.log
.orig .orig
$LOG_FILE
temp/
# yarn 4 (berry)
.pnp.*
.yarn/*
!.yarn/patches
!.yarn/plugins
!.yarn/releases
!.yarn/sdks
!.yarn/versions
-7
View File
@@ -1,7 +0,0 @@
#!/usr/bin/env sh
yarn lint-staged
yarn typecheck
if command -v gitleaks >/dev/null 2>&1; then
gitleaks protect --staged --no-banner --redact
fi
-17
View File
@@ -1,17 +0,0 @@
{
"semi": true,
"singleQuote": true,
"trailingComma": "all",
"printWidth": 100,
"proseWrap": "preserve",
"sortPackageJson": false,
"ignorePatterns": [
"**/node_modules/**",
"**/dist/**",
"**/coverage/**",
"**/.yarn/**",
"default-build-script/**",
"test-runner/**",
"platforms/**"
]
}
-58
View File
@@ -1,58 +0,0 @@
{
"$schema": "./node_modules/oxlint/configuration_schema.json",
"plugins": ["typescript", "vitest", "unicorn", "oxc"],
"categories": {
"correctness": "error",
"suspicious": "error",
"perf": "error"
},
"rules": {
"vitest/require-mock-type-parameters": "off",
"vitest/valid-title": "off",
"vitest/valid-describe-callback": "off",
"vitest/expect-expect": "off",
"vitest/no-conditional-tests": "off",
"vitest/no-conditional-expect": "off",
"vitest/require-to-throw-message": "off",
"vitest/no-disabled-tests": "warn",
"unicorn/prefer-array-flat-map": "warn",
"typescript/no-explicit-any": "warn",
"typescript/ban-ts-comment": "off",
"typescript/no-namespace": "off",
"typescript/no-extraneous-class": "off",
"no-bitwise": "off",
"no-shadow": "off",
"no-await-in-loop": "off",
"no-underscore-dangle": "off",
"unicorn/no-array-sort": "off",
"unicorn/prefer-set-has": "off",
"unicorn/consistent-function-scoping": "off",
"unicorn/no-useless-spread": "warn",
"eslint/preserve-caught-error": "warn",
"oxc/no-map-spread": "warn"
},
"overrides": [
{
"files": ["**/*.test.ts", "**/*.spec.ts"],
"rules": {
"typescript/no-explicit-any": "off",
"no-unused-vars": "off"
}
}
],
"env": {
"browser": false,
"node": true,
"es2024": true,
"vitest/globals": true
},
"ignorePatterns": [
"**/node_modules/**",
"**/dist/**",
"**/coverage/**",
"**/.yarn/**",
"default-build-script/**",
"test-runner/**",
"platforms/**"
]
}
+2
View File
@@ -0,0 +1,2 @@
**/node_modules/**
**/dist/**
+7
View File
@@ -0,0 +1,7 @@
{
"semi": true,
"singleQuote": true,
"trailingComma": "all",
"printWidth": 120,
"proseWrap": "always"
}
+3
View File
@@ -0,0 +1,3 @@
save-prefix "^"
--install.audit true
--add.audit true
-10
View File
@@ -1,10 +0,0 @@
approvedGitRepositories:
- '**'
compressionLevel: mixed
enableGlobalCache: false
enableHardenedMode: false
nodeLinker: node-modules
+103 -21
View File
@@ -9,7 +9,8 @@ inputs:
unityVersion: unityVersion:
required: false required: false
default: 'auto' default: 'auto'
description: 'Version of unity to use for building the project. Use "auto" to get from your ProjectSettings/ProjectVersion.txt' description:
'Version of unity to use for building the project. Use "auto" to get from your ProjectSettings/ProjectVersion.txt'
customImage: customImage:
required: false required: false
default: '' default: ''
@@ -46,10 +47,6 @@ inputs:
required: false required: false
default: '' default: ''
description: 'Custom parameters to configure the build.' description: 'Custom parameters to configure the build.'
useHostNetwork:
required: false
default: false
description: 'Initialises Docker using the host network. (Linux only)'
versioning: versioning:
required: false required: false
default: 'Semantic' default: 'Semantic'
@@ -107,13 +104,11 @@ inputs:
gitPrivateToken: gitPrivateToken:
required: false required: false
default: '' default: ''
description: 'Github private token to pull from github' description: '[CloudRunner] Github private token to pull from github'
providerStrategy: githubOwner:
default: 'local'
required: false required: false
description: default: ''
'Build execution strategy. Use "local" for local Docker/Mac builds. For remote builds (aws, k8s, etc.), install description: '[CloudRunner] GitHub owner name or organization/team name'
@game-ci/orchestrator and use the game-ci/orchestrator action which declares its own inputs.'
runAsHostUser: runAsHostUser:
required: false required: false
default: 'false' default: 'false'
@@ -123,7 +118,8 @@ inputs:
chownFilesTo: chownFilesTo:
required: false required: false
default: '' default: ''
description: 'User and optionally group (user or user:group or uid:gid) to give ownership of the resulting build artifacts' description:
'User and optionally group (user or user:group or uid:gid) to give ownership of the resulting build artifacts'
dockerCpuLimit: dockerCpuLimit:
required: false required: false
default: '' default: ''
@@ -133,8 +129,8 @@ inputs:
default: '' default: ''
description: description:
'Size of /dev/shm to assign the docker container, using the format <number><unit> (m or g). Unity 6.6+ editors 'Size of /dev/shm to assign the docker container, using the format <number><unit> (m or g). Unity 6.6+ editors
request 1GiB of shared memory and fail with "Insufficient shared memory available" against Docker''s 64m request 1GiB of shared memory and fail with "Insufficient shared memory available" against Docker''s 64m default.
default. Defaults to 1025m; set to "0" or "none" to use Docker''s own default.' Defaults to 1025m; set to "0" or "none" to use Docker''s own default.'
dockerMemoryLimit: dockerMemoryLimit:
required: false required: false
default: '' default: ''
@@ -160,7 +156,98 @@ inputs:
allowDirtyBuild: allowDirtyBuild:
required: false required: false
default: '' default: ''
description: 'Allows the branch of the build to be dirty, and still generate the build.' description: '[CloudRunner] Allows the branch of the build to be dirty, and still generate the build.'
postBuildSteps:
required: false
default: ''
description:
'[CloudRunner] run a post build job in yaml format with the keys image, secrets (name, value object array),
command string'
preBuildSteps:
required: false
default: ''
description:
'[CloudRunner] Run a pre build job after the repository setup but before the build job (in yaml format with the
keys image, secrets (name, value object array), command line string)'
containerHookFiles:
required: false
default: ''
description:
'[CloudRunner] Specify the names (by file name) of custom steps to run before or after cloud runner jobs, must
match a yaml step file inside your repo in the folder .game-ci/steps/'
customHookFiles:
required: false
default: ''
description:
'[CloudRunner] Specify the names (by file name) of custom hooks to run before or after cloud runner jobs, must
match a yaml step file inside your repo in the folder .game-ci/hooks/'
customCommandHooks:
required: false
default: ''
description: '[CloudRunner] Specify custom commands and trigger hooks (injects commands into jobs)'
customJob:
required: false
default: ''
description:
'[CloudRunner] Run a custom job instead of the standard build automation for cloud runner (in yaml format with the
keys image, secrets (name, value object array), command line string)'
awsStackName:
default: 'game-ci'
required: false
description: '[CloudRunner] The Cloud Formation stack name that must be setup before using this option.'
providerStrategy:
default: 'local'
required: false
description:
'[CloudRunner] Either local, k8s or aws can be used to run builds on a remote cluster. Additional parameters must
be configured.'
containerCpu:
default: ''
required: false
description: '[CloudRunner] Amount of CPU time to assign the remote build container'
containerMemory:
default: ''
required: false
description: '[CloudRunner] Amount of memory to assign the remote build container'
readInputFromOverrideList:
default: ''
required: false
description: '[CloudRunner] Comma separated list of input value names to read from "input override command"'
readInputOverrideCommand:
default: ''
required: false
description:
'[CloudRunner] Extend game ci by specifying a command to execute to pull input from external source e.g cloud
provider secret managers'
kubeConfig:
default: ''
required: false
description:
'[CloudRunner] Supply a base64 encoded kubernetes config to run builds on kubernetes and stream logs until
completion.'
kubeVolume:
default: ''
required: false
description: '[CloudRunner] Supply a Persistent Volume Claim name to use for the Unity build.'
kubeStorageClass:
default: ''
required: false
description:
'[CloudRunner] Kubernetes storage class to use for cloud runner jobs, leave empty to install rook cluster.'
kubeVolumeSize:
default: '5Gi'
required: false
description: '[CloudRunner] Amount of disc space to assign the Kubernetes Persistent Volume'
cacheKey:
default: ''
required: false
description: '[CloudRunner] Cache key to indicate bucket for cache'
watchToEnd:
default: 'true'
required: false
description:
'[CloudRunner] Whether or not to watch the build to the end. Can be used for especially long running jobs e.g
imports or self-hosted ephemeral runners.'
cacheUnityInstallationOnMac: cacheUnityInstallationOnMac:
default: 'false' default: 'false'
required: false required: false
@@ -185,11 +272,6 @@ inputs:
default: 'false' default: 'false'
required: false required: false
description: 'Skip the activation/deactivation of Unity. This assumes Unity is already activated.' description: 'Skip the activation/deactivation of Unity. This assumes Unity is already activated.'
linux64RemoveExecutableExtension:
default: 'false'
required: false
description:
'When building for StandaloneLinux64, remove the default file extension of `.x86_64`. Set to true to restore the extensionless behavior from v4.'
outputs: outputs:
volume: volume:
@@ -207,5 +289,5 @@ branding:
icon: 'box' icon: 'box'
color: 'gray-dark' color: 'gray-dark'
runs: runs:
using: 'node24' using: 'node20'
main: 'dist/index.js' main: 'dist/index.js'
-27
View File
@@ -1,27 +0,0 @@
# Community Plugin Validation Registry
# Packages listed here are automatically tested on a schedule
# to ensure compatibility with unity-builder.
#
# Format:
# - name: Human-readable name
# package: UPM package name or git URL
# source: upm | git | asset-store
# unity: Minimum Unity version (optional, defaults to 2021.3)
# platforms: List of platforms to test (optional, defaults to [StandaloneLinux64])
# timeout: Build timeout in minutes (optional, defaults to 30)
plugins:
# Example entries — community members can submit PRs to add their packages
- name: UniTask
package: https://github.com/Cysharp/UniTask.git?path=src/UniTask/Assets/Plugins/UniTask
source: git
platforms: [StandaloneLinux64, StandaloneWindows64]
- name: NaughtyAttributes
package: https://github.com/dbrizov/NaughtyAttributes.git?path=Assets/NaughtyAttributes
source: git
- name: Unity Atoms
package: https://github.com/unity-atoms/unity-atoms.git
source: git
platforms: [StandaloneLinux64]
Generated Vendored
+348471 -93158
View File
File diff suppressed because one or more lines are too long
Generated Vendored
+1 -1
View File
File diff suppressed because one or more lines are too long
Generated Vendored
+19061 -176
View File
File diff suppressed because it is too large Load Diff
+32
View File
@@ -0,0 +1,32 @@
module.exports = {
// Automatically clear mock calls and instances between every test
clearMocks: true,
// An array of file extensions your modules use
moduleFileExtensions: ['js', 'ts'],
// The test environment that will be used for testing
testEnvironment: 'node',
// The glob patterns Jest uses to detect test files
testMatch: ['**/*.test.ts'],
// This option allows use of a custom test runner
testRunner: 'jest-circus/runner',
// A map with regular expressions for transformers to paths
transform: {
'^.+\\.ts$': 'ts-jest',
},
// Indicates whether each individual test should be reported during the run
verbose: true,
// An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
modulePathIgnorePatterns: ['<rootDir>/lib/', '<rootDir>/dist/'],
// Files that will be run before Jest is loaded to set globals like fetch
setupFiles: ['<rootDir>/src/jest.globals.ts'],
// A list of paths to modules that run some code to configure or set up the testing framework after the environment is ready
setupFilesAfterEnv: ['<rootDir>/src/jest.setup.ts'],
};
-2
View File
@@ -1,2 +0,0 @@
const fetch = require('node-fetch');
global.fetch = fetch;
+31
View File
@@ -0,0 +1,31 @@
# EXAMPLE USAGE
# Refer for explanation to following link:
# https://github.com/evilmartians/lefthook/blob/master/docs/full_guide.md
#
color: true
extends: {}
pre-commit:
parallel: true
commands:
format documents:
glob: '*.{md,mdx}'
run: yarn prettier --write {staged_files}
format configs:
glob: '*.{json,yml,yaml}'
run: yarn prettier --write {staged_files}
format code:
glob: '*.{js,jsx,ts,tsx}'
exclude: 'dist/'
run: yarn prettier --write {staged_files} && yarn eslint {staged_files} && git add {staged_files}
run tests:
glob: '*.{js,jsx,ts,tsx}'
exclude: 'dist/'
run: yarn jest --passWithNoTests --findRelatedTests {staged_files}
build distributables:
skip: ['merge', 'rebase']
run: yarn build && git add dist
make shell script executable:
glob: '*.sh'
run: git update-index --chmod=+x
-6
View File
@@ -1,6 +0,0 @@
[tools]
node = "20.18.0"
yarn = "4.14.1"
actionlint = "latest"
shellcheck = "latest"
gitleaks = "latest"
+54 -40
View File
@@ -7,67 +7,81 @@
"author": "Webber <[email protected]>", "author": "Webber <[email protected]>",
"license": "MIT", "license": "MIT",
"scripts": { "scripts": {
"prepare": "husky", "prepare": "lefthook install",
"build": "yarn && tsc && ncc build lib --source-map --license licenses.txt", "build": "yarn && tsc && ncc build lib --source-map --license licenses.txt",
"test": "node scripts/ensure-husky.mjs && vitest run", "lint": "prettier --check \"src/**/*.{js,ts}\" && eslint src/**/*.ts",
"test:watch": "vitest", "format": "prettier --write \"src/**/*.{js,ts}\"",
"test:ci": "vitest run", "cli": "yarn ts-node src/index.ts -m cli",
"coverage": "vitest run --coverage", "gcp-secrets-tests": "cross-env providerStrategy=aws cloudRunnerTests=true inputPullCommand=\"gcp-secret-manager\" populateOverride=true pullInputList=UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD yarn test -i -t \"cloud runner\"",
"lint": "yarn oxlint --report-unused-disable-directives", "gcp-secrets-cli": "cross-env cloudRunnerTests=true USE_IL2CPP=false inputPullCommand=\"gcp-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
"format": "oxfmt --write", "aws-secrets-cli": "cross-env cloudRunnerTests=true inputPullCommand=\"aws-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
"format:check": "oxfmt --check", "cli-aws": "cross-env providerStrategy=aws yarn run test-cli",
"typecheck": "tsc --noEmit", "cli-k8s": "cross-env providerStrategy=k8s yarn run test-cli",
"typecheck:tsgo": "tsgo --noEmit", "test-cli": "cross-env cloudRunnerTests=true yarn ts-node src/index.ts -m cli --projectPath test-project",
"setup:hooks": "node scripts/ensure-husky.mjs" "test": "jest",
}, "test-i": "cross-env cloudRunnerTests=true yarn test -i -t \"cloud runner\"",
"lint-staged": { "test-i-*": "yarn run test-i-aws && yarn run test-i-k8s",
"*.@(ts|tsx|mts|js|jsx|mjs|cjs)": [ "test-i-aws": "cross-env cloudRunnerTests=true providerStrategy=aws yarn test -i -t \"cloud runner\"",
"oxlint --fix --quiet", "test-i-k8s": "cross-env cloudRunnerTests=true providerStrategy=k8s yarn test -i -t \"cloud runner\""
"oxfmt --write"
],
"*.@(json|jsonc|json5|md|mdx|yaml|yml|css|scss|sass|html|toml)": "oxfmt --write",
".github/workflows/*.@(yml|yaml)": "actionlint"
}, },
"engines": { "engines": {
"node": ">=18.x" "node": ">=18.x"
}, },
"dependencies": { "dependencies": {
"@actions/cache": "^4.1.0", "@actions/cache": "^4.0.0",
"@actions/core": "^1.11.1", "@actions/core": "^1.11.1",
"@actions/exec": "^1.1.1", "@actions/exec": "^1.1.1",
"@actions/github": "^6.0.1", "@actions/github": "^6.0.0",
"@aws-sdk/client-cloudformation": "^3.777.0",
"@aws-sdk/client-cloudwatch-logs": "^3.777.0",
"@aws-sdk/client-ecs": "^3.778.0",
"@aws-sdk/client-kinesis": "^3.777.0",
"@aws-sdk/client-s3": "^3.779.0",
"@kubernetes/client-node": "^0.16.3",
"@octokit/core": "^5.1.0",
"async-wait-until": "^2.0.12",
"aws-sdk": "^2.1081.0",
"base-64": "^1.0.0",
"commander": "^9.0.0",
"commander-ts": "^0.2.0",
"kubernetes-client": "^9.0.0",
"md5": "^2.3.0", "md5": "^2.3.0",
"nanoid": "^3.3.12", "nanoid": "^3.3.1",
"semver": "^7.7.4", "reflect-metadata": "^0.1.13",
"semver": "^7.5.2",
"ts-md5": "^1.3.1", "ts-md5": "^1.3.1",
"unity-changeset": "^3.1.0", "unity-changeset": "^3.1.0",
"yaml": "^2.8.4" "uuid": "^9.0.0",
"yaml": "^2.2.2"
}, },
"devDependencies": { "devDependencies": {
"@types/base-64": "^1.0.0",
"@types/jest": "^27.4.1",
"@types/node": "^17.0.23", "@types/node": "^17.0.23",
"@types/semver": "^7.3.9", "@types/semver": "^7.3.9",
"@typescript/native-preview": "^7.0.0-dev.20260505.1", "@types/uuid": "^9.0.0",
"@typescript-eslint/parser": "4.8.1",
"@vercel/ncc": "^0.36.1", "@vercel/ncc": "^0.36.1",
"@vitest/coverage-istanbul": "^4.1.5",
"cross-env": "^7.0.3", "cross-env": "^7.0.3",
"eslint": "^10.3.0", "eslint": "^7.23.0",
"eslint-plugin-unicorn": "^64.0.0", "eslint-config-prettier": "8.1.0",
"husky": "9", "eslint-plugin-github": "^4.1.1",
"eslint-plugin-jest": "24.1.3",
"eslint-plugin-prettier": "^3.3.1",
"eslint-plugin-unicorn": "28.0.2",
"jest": "^27.5.1",
"jest-circus": "^27.5.1",
"jest-fail-on-console": "^3.0.2",
"js-yaml": "^4.1.0", "js-yaml": "^4.1.0",
"lint-staged": "^16.4.0", "lefthook": "^1.6.1",
"node-fetch": "2", "prettier": "^2.5.1",
"oxfmt": "^0.48.0", "ts-jest": "^27.1.3",
"oxlint": "^1.63.0",
"ts-node": "10.8.1", "ts-node": "10.8.1",
"typescript": "4.7.4", "typescript": "4.7.4",
"vite": "^7",
"vitest": "^4",
"yarn-audit-fix": "^9.3.8" "yarn-audit-fix": "^9.3.8"
}, },
"packageManager": "[email protected]", "volta": {
"dependenciesMeta": { "node": "20.5.1",
"lefthook": { "yarn": "1.22.19"
"built": true
}
} }
} }
-55
View File
@@ -1,55 +0,0 @@
#!/usr/bin/env node
// Self-heals husky git hooks before local dev workflows.
//
// Why this exists: Yarn 4 skips lifecycle scripts (`prepare`, `postinstall`) on
// no-op installs, so `yarn install --immutable` does NOT reinstall hooks once
// `.husky/_/` has been wiped. `.husky/_/` is gitignored, so it is also missing
// in fresh worktrees and after `git clean -fdx`. Without this guard, commits
// silently skip the pre-commit hook (git treats a missing hook file as "no hook").
//
// Behaviour: ~20 ms no-op when hooks are already installed. Skipped in CI and
// when HUSKY=0. Fails loudly (non-zero exit) on real install errors so the
// caller stops before commits are made without hooks.
import { execSync } from 'node:child_process';
import { existsSync } from 'node:fs';
if (process.env.CI || process.env.HUSKY === '0') process.exit(0);
const expectedHooksPath = '.husky/_';
const sentinelHook = '.husky/_/pre-commit';
// husky 9.1+ ships bin.js; husky 9.0 ships bin.mjs. Try both.
const huskyBin = ['node_modules/husky/bin.js', 'node_modules/husky/bin.mjs'].find(existsSync);
let configuredHooksPath = '';
try {
configuredHooksPath = execSync('git config --get core.hooksPath', {
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'ignore'],
}).trim();
} catch {
// not a git repo or config unset — fall through and try to install
}
if (configuredHooksPath === expectedHooksPath && existsSync(sentinelHook)) {
process.exit(0);
}
if (!huskyBin) {
// husky not installed yet (yarn install hasn't run) — silent no-op
process.exit(0);
}
console.log('· installing git hooks (husky self-heal)…');
try {
execSync(`node ${huskyBin}`, { stdio: 'inherit' });
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
console.error(
`\n❌ husky install failed: ${message}\n\n` +
` git pre-commit hooks are NOT installed; commits will skip lint/format/tests.\n` +
` Fix the underlying error above, then run \`yarn setup:hooks\` to retry.\n` +
` To bypass this guard temporarily (NOT recommended): HUSKY=0 yarn <cmd>.\n`,
);
process.exit(1);
}
-272
View File
@@ -1,272 +0,0 @@
import { describe, it, expect, beforeEach, afterEach, vi, type Mock } from 'vitest';
/**
* Integration wiring tests for the plugin lifecycle in index.ts
*
* These tests verify that:
* - The plugin lifecycle hooks are called in the correct order
* - Plugin canHandleBuild() controls the execution path
* - fallbackToLocal is handled correctly
* - When no plugin is installed, local builds still work
* - When providerStrategy is non-local without a plugin, an error is thrown
*/
import { BuildParameters, Docker } from './model';
import * as core from '@actions/core';
// ---------------------------------------------------------------------------
// Mock plugin
// ---------------------------------------------------------------------------
// `vi.mock` hoists to the top of the module, so any factory references must
// be hoisted with `vi.hoisted` to be defined at mock-evaluation time.
const { mockPlugin, mockLoadPlugin } = vi.hoisted(() => {
const plugin = {
initialize: vi.fn().mockResolvedValue(undefined),
canHandleBuild: vi.fn().mockReturnValue(false),
handleBuild: vi.fn().mockResolvedValue({ exitCode: 0 }),
beforeLocalBuild: vi.fn().mockResolvedValue(undefined),
afterLocalBuild: vi.fn().mockResolvedValue(undefined),
handlePostBuild: vi.fn().mockResolvedValue(undefined),
};
return {
mockPlugin: plugin,
mockLoadPlugin: vi.fn().mockResolvedValue(plugin),
};
});
vi.mock('./model/plugin', () => ({
loadPlugin: mockLoadPlugin,
}));
vi.mock('@actions/core');
vi.mock('./model', () => ({
Action: {
checkCompatibility: vi.fn(),
workspace: '/workspace',
actionFolder: '/action',
},
BuildParameters: {
create: vi.fn(),
},
Cache: {
verify: vi.fn(),
},
Docker: {
run: vi.fn().mockResolvedValue(0),
},
// vitest 4 requires constructor mocks to use regular `function` (or
// `class`); arrow fns aren't valid constructors.
ImageTag: vi.fn(function () {
return { toString: () => 'mock-image:latest' };
}),
Output: {
setBuildVersion: vi.fn().mockResolvedValue(''),
setAndroidVersionCode: vi.fn().mockResolvedValue(''),
setEngineExitCode: vi.fn().mockResolvedValue(''),
},
}));
vi.mock('./model/mac-builder', () => ({
__esModule: true,
default: {
run: vi.fn().mockResolvedValue(0),
},
}));
vi.mock('./model/platform-setup', () => ({
__esModule: true,
default: {
setup: vi.fn().mockResolvedValue(''),
},
}));
const mockedBuildParametersCreate = BuildParameters.create as Mock;
function createMockBuildParameters(overrides: Record<string, any> = {}) {
return {
providerStrategy: 'local',
targetPlatform: 'StandaloneLinux64',
editorVersion: '2021.3.1f1',
buildVersion: '1.0.0',
androidVersionCode: '1',
projectPath: '.',
branch: 'main',
runnerTempPath: '/tmp',
...overrides,
};
}
async function runIndex(overrides: Record<string, any> = {}): Promise<void> {
mockedBuildParametersCreate.mockResolvedValue(createMockBuildParameters(overrides));
// index.ts exports `runMain` for testability (the file used to rely on
// top-level execution + jest's `vi.isolateModules`, but vitest 4 dropped
// that API). Calling the exported function directly is cleaner than
// round-tripping through dynamic imports.
const { runMain } = await import('./index');
await runMain();
}
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
describe('index.ts plugin lifecycle wiring', () => {
const originalPlatform = process.platform;
const originalEnvironment = { ...process.env };
beforeEach(() => {
vi.clearAllMocks();
process.env.GITHUB_WORKSPACE = '/workspace';
Object.defineProperty(process, 'platform', { value: 'linux' });
// Reset plugin to default behavior
mockPlugin.canHandleBuild.mockReturnValue(false);
mockPlugin.handleBuild.mockResolvedValue({ exitCode: 0 });
mockLoadPlugin.mockResolvedValue(mockPlugin);
});
afterEach(() => {
Object.defineProperty(process, 'platform', { value: originalPlatform });
process.env = { ...originalEnvironment };
});
// -----------------------------------------------------------------------
// Local build with plugin
// -----------------------------------------------------------------------
describe('local build with plugin installed', () => {
it('should call lifecycle hooks in order: initialize -> beforeLocalBuild -> [build] -> afterLocalBuild -> handlePostBuild', async () => {
const callOrder: string[] = [];
mockPlugin.initialize.mockImplementation(async () => callOrder.push('initialize'));
mockPlugin.beforeLocalBuild.mockImplementation(async () =>
callOrder.push('beforeLocalBuild'),
);
mockPlugin.afterLocalBuild.mockImplementation(async () => callOrder.push('afterLocalBuild'));
mockPlugin.handlePostBuild.mockImplementation(async () => callOrder.push('handlePostBuild'));
await runIndex();
expect(callOrder).toEqual([
'initialize',
'beforeLocalBuild',
'afterLocalBuild',
'handlePostBuild',
]);
});
it('should pass buildParameters and workspace to initialize', async () => {
await runIndex({ targetPlatform: 'WebGL' });
expect(mockPlugin.initialize).toHaveBeenCalledWith(
expect.objectContaining({ targetPlatform: 'WebGL' }),
'/workspace',
);
});
it('should pass workspace to beforeLocalBuild', async () => {
await runIndex();
expect(mockPlugin.beforeLocalBuild).toHaveBeenCalledWith('/workspace');
});
it('should pass workspace and exit code to afterLocalBuild', async () => {
await runIndex();
expect(mockPlugin.afterLocalBuild).toHaveBeenCalledWith('/workspace', 0);
});
it('should pass exit code to handlePostBuild', async () => {
await runIndex();
expect(mockPlugin.handlePostBuild).toHaveBeenCalledWith(0);
});
});
// -----------------------------------------------------------------------
// Plugin handles build entirely
// -----------------------------------------------------------------------
describe('plugin handles build (canHandleBuild = true)', () => {
it('should call handleBuild instead of Docker.run', async () => {
mockPlugin.canHandleBuild.mockReturnValue(true);
mockPlugin.handleBuild.mockResolvedValue({ exitCode: 0 });
await runIndex();
expect(mockPlugin.handleBuild).toHaveBeenCalledWith('mock-image:latest');
expect(Docker.run).not.toHaveBeenCalled();
expect(mockPlugin.beforeLocalBuild).not.toHaveBeenCalled();
expect(mockPlugin.afterLocalBuild).not.toHaveBeenCalled();
});
it('should still call handlePostBuild after handleBuild', async () => {
mockPlugin.canHandleBuild.mockReturnValue(true);
mockPlugin.handleBuild.mockResolvedValue({ exitCode: 0 });
await runIndex();
expect(mockPlugin.handlePostBuild).toHaveBeenCalledWith(0);
});
});
// -----------------------------------------------------------------------
// Fallback to local
// -----------------------------------------------------------------------
describe('fallback to local build', () => {
it('should do a local build when handleBuild returns fallbackToLocal', async () => {
mockPlugin.canHandleBuild.mockReturnValue(true);
mockPlugin.handleBuild.mockResolvedValue({ exitCode: -1, fallbackToLocal: true });
await runIndex();
expect(mockPlugin.handleBuild).toHaveBeenCalled();
expect(mockPlugin.beforeLocalBuild).toHaveBeenCalled();
expect(Docker.run).toHaveBeenCalled();
expect(mockPlugin.afterLocalBuild).toHaveBeenCalled();
});
});
// -----------------------------------------------------------------------
// No plugin installed
// -----------------------------------------------------------------------
describe('no plugin installed', () => {
it('should build locally without errors when providerStrategy is local', async () => {
mockLoadPlugin.mockResolvedValue(undefined);
await runIndex({ providerStrategy: 'local' });
expect(Docker.run).toHaveBeenCalled();
});
it('should error when providerStrategy is non-local and no plugin', async () => {
mockLoadPlugin.mockResolvedValue(undefined);
await runIndex({ providerStrategy: 'aws' });
expect(core.setFailed).toHaveBeenCalledWith(
expect.stringContaining('requires @game-ci/orchestrator'),
);
});
});
// -----------------------------------------------------------------------
// canHandleBuild = false with non-local provider
// -----------------------------------------------------------------------
describe('plugin installed but canHandleBuild returns false with non-local provider', () => {
it('should error when providerStrategy is non-local', async () => {
mockPlugin.canHandleBuild.mockReturnValue(false);
await runIndex({ providerStrategy: 'aws' });
// The plugin is initialized but says it can't handle the build,
// and providerStrategy is not local, so it falls to the error case
expect(core.setFailed).toHaveBeenCalledWith(
expect.stringContaining('requires @game-ci/orchestrator'),
);
});
});
});
+24 -56
View File
@@ -1,50 +1,47 @@
import * as core from '@actions/core'; import * as core from '@actions/core';
import { Action, BuildParameters, Cache, Docker, ImageTag, Output } from './model'; import { Action, BuildParameters, Cache, CloudRunner, Docker, ImageTag, Output } from './model';
import { Cli } from './model/cli/cli';
import MacBuilder from './model/mac-builder'; import MacBuilder from './model/mac-builder';
import PlatformSetup from './model/platform-setup'; import PlatformSetup from './model/platform-setup';
import { Plugin, loadPlugin } from './model/plugin';
// Exported so tests can drive the lifecycle directly without depending on async function runMain() {
// vitest's module re-loading (which changed in vitest 4).
export async function runMain() {
try { try {
if (Cli.InitCliMode()) {
await Cli.RunCli();
return;
}
Action.checkCompatibility(); Action.checkCompatibility();
Cache.verify(); Cache.verify();
const { workspace, actionFolder } = Action; const { workspace, actionFolder } = Action;
const buildParameters = await BuildParameters.create(); const buildParameters = await BuildParameters.create();
const baseImage = new ImageTag(buildParameters); const baseImage = new ImageTag(buildParameters);
// Load optional plugin. The default implementation is @game-ci/orchestrator.
const plugin = await loadPlugin();
await plugin?.initialize(buildParameters, workspace);
let exitCode = -1; let exitCode = -1;
if (plugin?.canHandleBuild()) { if (buildParameters.providerStrategy === 'local') {
// Plugin handles the build entirely (remote providers, hot runner, test workflows) core.info('Building locally');
const result = await plugin.handleBuild(baseImage.toString()); await PlatformSetup.setup(buildParameters, actionFolder);
exitCode =
exitCode = result.fallbackToLocal process.platform === 'darwin'
? await runLocalBuild(buildParameters, baseImage, workspace, actionFolder, plugin) ? await MacBuilder.run(actionFolder)
: result.exitCode; : await Docker.run(baseImage.toString(), {
} else if (buildParameters.providerStrategy === 'local') { workspace,
exitCode = await runLocalBuild(buildParameters, baseImage, workspace, actionFolder, plugin); actionFolder,
...buildParameters,
});
} else { } else {
throw new Error( await CloudRunner.run(buildParameters, baseImage.toString());
`Provider strategy "${buildParameters.providerStrategy}" requires @game-ci/orchestrator. ` + exitCode = 0;
'Install it via the game-ci/orchestrator action, or use providerStrategy=local.',
);
} }
// Set core outputs // Set output
await Output.setBuildVersion(buildParameters.buildVersion); await Output.setBuildVersion(buildParameters.buildVersion);
await Output.setAndroidVersionCode(buildParameters.androidVersionCode); await Output.setAndroidVersionCode(buildParameters.androidVersionCode);
await Output.setEngineExitCode(exitCode); await Output.setEngineExitCode(exitCode);
// Plugin handles post-build (artifacts, archiving, retention)
await plugin?.handlePostBuild(exitCode);
if (exitCode !== 0) { if (exitCode !== 0) {
core.setFailed(`Build failed with exit code ${exitCode}`); core.setFailed(`Build failed with exit code ${exitCode}`);
} }
@@ -53,33 +50,4 @@ export async function runMain() {
} }
} }
async function runLocalBuild( runMain();
buildParameters: BuildParameters,
baseImage: ImageTag,
workspace: string,
actionFolder: string,
plugin?: Plugin,
): Promise<number> {
await plugin?.beforeLocalBuild(workspace);
await PlatformSetup.setup(buildParameters, actionFolder);
const exitCode =
process.platform === 'darwin'
? await MacBuilder.run(actionFolder)
: await Docker.run(baseImage.toString(), {
workspace,
actionFolder,
...buildParameters,
});
await plugin?.afterLocalBuild(workspace, exitCode);
return exitCode;
}
// Auto-run when this module is the entry point. Tests import the file via
// `await import('./index')` purely to register the mock factories and then
// call `runMain()` directly.
if (process.env.NODE_ENV !== 'test') {
runMain();
}
@@ -0,0 +1,29 @@
// Integration test for exercising real GitHub check creation and updates.
import CloudRunner from '../model/cloud-runner/cloud-runner';
import UnityVersioning from '../model/unity-versioning';
import GitHub from '../model/github';
import { TIMEOUT_INFINITE, createParameters } from '../test-utils/cloud-runner-test-helpers';
const runIntegration = process.env.RUN_GITHUB_INTEGRATION_TESTS === 'true';
const describeOrSkip = runIntegration ? describe : describe.skip;
describeOrSkip('Cloud Runner Github Checks Integration', () => {
it(
'creates and updates a real GitHub check',
async () => {
const buildParameter = await createParameters({
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.read('test-project'),
asyncCloudRunner: `true`,
githubChecks: `true`,
});
await CloudRunner.setup(buildParameter);
const checkId = await GitHub.createGitHubCheck(`integration create`);
expect(checkId).not.toEqual('');
await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `integration`);
await GitHub.updateGitHubCheck(`2 ${new Date().toISOString()}`, `integration`, `success`, `completed`);
},
TIMEOUT_INFINITE,
);
});
-1
View File
@@ -1,4 +1,3 @@
import { describe, it, expect, vi, beforeEach, afterEach, beforeAll, afterAll, test } from 'vitest';
import { stat } from 'node:fs/promises'; import { stat } from 'node:fs/promises';
describe('Integrity tests', () => { describe('Integrity tests', () => {
+9
View File
@@ -0,0 +1,9 @@
import failOnConsole from 'jest-fail-on-console';
// Fail when console logs something inside a test - use spyOn instead
failOnConsole({
shouldFailOnWarn: true,
shouldFailOnError: true,
shouldFailOnLog: true,
shouldFailOnAssert: true,
});
+1 -2
View File
@@ -1,8 +1,7 @@
import { vi } from 'vitest';
// Import this named export into your test file: // Import this named export into your test file:
import Platform from '../platform'; import Platform from '../platform';
export const mockGetFromUser = vi.fn().mockResolvedValue({ export const mockGetFromUser = jest.fn().mockResolvedValue({
editorVersion: '', editorVersion: '',
targetPlatform: Platform.types.Test, targetPlatform: Platform.types.Test,
projectPath: '.', projectPath: '.',
+16 -17
View File
@@ -1,23 +1,22 @@
import { vi } from 'vitest';
/* eslint unicorn/prevent-abbreviations: "off" */ /* eslint unicorn/prevent-abbreviations: "off" */
// Import these named export into your test file: // Import these named export into your test file:
export const mockProjectPath = vi.fn().mockResolvedValue('mockProjectPath'); export const mockProjectPath = jest.fn().mockResolvedValue('mockProjectPath');
export const mockIsDirtyAllowed = vi.fn().mockResolvedValue(false); export const mockIsDirtyAllowed = jest.fn().mockResolvedValue(false);
export const mockBranch = vi.fn().mockResolvedValue('mockBranch'); export const mockBranch = jest.fn().mockResolvedValue('mockBranch');
export const mockHeadRef = vi.fn().mockResolvedValue('mockHeadRef'); export const mockHeadRef = jest.fn().mockResolvedValue('mockHeadRef');
export const mockRef = vi.fn().mockResolvedValue('mockRef'); export const mockRef = jest.fn().mockResolvedValue('mockRef');
export const mockDetermineVersion = vi.fn().mockResolvedValue('1.2.3'); export const mockDetermineVersion = jest.fn().mockResolvedValue('1.2.3');
export const mockGenerateSemanticVersion = vi.fn().mockResolvedValue('2.3.4'); export const mockGenerateSemanticVersion = jest.fn().mockResolvedValue('2.3.4');
export const mockGenerateTagVersion = vi.fn().mockResolvedValue('1.0'); export const mockGenerateTagVersion = jest.fn().mockResolvedValue('1.0');
export const mockParseSemanticVersion = vi.fn().mockResolvedValue({}); export const mockParseSemanticVersion = jest.fn().mockResolvedValue({});
export const mockFetch = vi.fn().mockImplementation(() => {}); export const mockFetch = jest.fn().mockImplementation(() => {});
export const mockGetVersionDescription = vi.fn().mockResolvedValue('1.2-3-g12345678-dirty'); export const mockGetVersionDescription = jest.fn().mockResolvedValue('1.2-3-g12345678-dirty');
export const mockIsDirty = vi.fn().mockResolvedValue(false); export const mockIsDirty = jest.fn().mockResolvedValue(false);
export const mockGetTag = vi.fn().mockResolvedValue('v1.0'); export const mockGetTag = jest.fn().mockResolvedValue('v1.0');
export const mockHasAnyVersionTags = vi.fn().mockResolvedValue(true); export const mockHasAnyVersionTags = jest.fn().mockResolvedValue(true);
export const mockGetTotalNumberOfCommits = vi.fn().mockResolvedValue(3); export const mockGetTotalNumberOfCommits = jest.fn().mockResolvedValue(3);
export const mockGit = vi.fn().mockImplementation(() => {}); export const mockGit = jest.fn().mockImplementation(() => {});
export default { export default {
projectPath: mockProjectPath, projectPath: mockProjectPath,
@@ -1,3 +1,3 @@
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html // Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`Versioning > determineBuildVersion > throws for invalid strategy somethingRandom 1`] = `[ValidationError: Versioning strategy should be one of None, Semantic, Tag, Custom.]`; exports[`Versioning determineBuildVersion throws for invalid strategy somethingRandom 1`] = `"Versioning strategy should be one of None, Semantic, Tag, Custom."`;
-1
View File
@@ -1,4 +1,3 @@
import { describe, it, expect, vi, beforeEach, afterEach, beforeAll, afterAll, test } from 'vitest';
import path from 'node:path'; import path from 'node:path';
import fs from 'node:fs'; import fs from 'node:fs';
import Action from './action'; import Action from './action';
+1 -4
View File
@@ -1,4 +1,3 @@
import { describe, it, expect, vi, beforeEach, afterEach, beforeAll, afterAll, test } from 'vitest';
import AndroidVersioning from './android-versioning'; import AndroidVersioning from './android-versioning';
describe('Android Versioning', () => { describe('Android Versioning', () => {
@@ -36,9 +35,7 @@ describe('Android Versioning', () => {
}); });
it('uses the specified api level', () => { it('uses the specified api level', () => {
expect(AndroidVersioning.determineSdkManagerParameters('AndroidApiLevel30')).toBe( expect(AndroidVersioning.determineSdkManagerParameters('AndroidApiLevel30')).toBe('platforms;android-30');
'platforms;android-30',
);
}); });
}); });
}); });
+2 -5
View File
@@ -12,9 +12,7 @@ export default class AndroidVersioning {
static versionToVersionCode(version: string): string { static versionToVersionCode(version: string): string {
if (version === 'none') { if (version === 'none') {
core.info( core.info(`Versioning strategy is set to ${version}, so android version code should not be applied.`);
`Versioning strategy is set to ${version}, so android version code should not be applied.`,
);
return '0'; return '0';
} }
@@ -29,8 +27,7 @@ export default class AndroidVersioning {
// The greatest value Google Plays allows is 2100000000. // The greatest value Google Plays allows is 2100000000.
// Allow for 3 patch digits, 3 minor digits and 3 major digits. // Allow for 3 patch digits, 3 minor digits and 3 major digits.
const versionCode = const versionCode = parsedVersion.major * 1000000 + parsedVersion.minor * 1000 + parsedVersion.patch;
parsedVersion.major * 1000000 + parsedVersion.minor * 1000 + parsedVersion.patch;
if (versionCode >= 2050000000) { if (versionCode >= 2050000000) {
throw new Error( throw new Error(
+49 -80
View File
@@ -1,4 +1,3 @@
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, test, vi } from 'vitest';
import Versioning from './versioning'; import Versioning from './versioning';
import UnityVersioning from './unity-versioning'; import UnityVersioning from './unity-versioning';
import AndroidVersioning from './android-versioning'; import AndroidVersioning from './android-versioning';
@@ -10,12 +9,12 @@ const testLicense =
'<?xml version="1.0" encoding="UTF-8"?><root>\n <License id="Terms">\n <MachineBindings>\n <Binding Key="1" Value="576562626572264761624c65526f7578"/>\n <Binding Key="2" Value="576562626572264761624c65526f7578"/>\n </MachineBindings>\n <MachineID Value="D7nTUnjNAmtsUMcnoyrqkgIbYdM="/>\n <SerialHash Value="2033b8ac3e6faa3742ca9f0bfae44d18f2a96b80"/>\n <Features>\n <Feature Value="33"/>\n <Feature Value="1"/>\n <Feature Value="12"/>\n <Feature Value="2"/>\n <Feature Value="24"/>\n <Feature Value="3"/>\n <Feature Value="36"/>\n <Feature Value="17"/>\n <Feature Value="19"/>\n <Feature Value="62"/>\n </Features>\n <DeveloperData Value="AQAAAEY0LUJHUlgtWEQ0RS1aQ1dWLUM1SlctR0RIQg=="/>\n <SerialMasked Value="F4-BGRX-XD4E-ZCWV-C5JW-XXXX"/>\n <StartDate Value="2021-02-08T00:00:00"/>\n <UpdateDate Value="2021-02-09T00:34:57"/>\n <InitialActivationDate Value="2021-02-08T00:34:56"/>\n <LicenseVersion Value="6.x"/>\n <ClientProvidedVersion Value="2018.4.30f1"/>\n <AlwaysOnline Value="false"/>\n <Entitlements>\n <Entitlement Ns="unity_editor" Tag="UnityPersonal" Type="EDITOR" ValidTo="9999-12-31T00:00:00"/>\n <Entitlement Ns="unity_editor" Tag="DarkSkin" Type="EDITOR_FEATURE" ValidTo="9999-12-31T00:00:00"/>\n </Entitlements>\n </License>\n<Signature xmlns="http://www.w3.org/2000/09/xmldsig#"><SignedInfo><CanonicalizationMethod Algorithm="http://www.w3.org/TR/2001/REC-xml-c14n-20010315#WithComments"/><SignatureMethod Algorithm="http://www.w3.org/2000/09/xmldsig#rsa-sha1"/><Reference URI="#Terms"><Transforms><Transform Algorithm="http://www.w3.org/2000/09/xmldsig#enveloped-signature"/></Transforms><DigestMethod Algorithm="http://www.w3.org/2000/09/xmldsig#sha1"/><DigestValue>m0Db8UK+ktnOLJBtHybkfetpcKo=</DigestValue></Reference></SignedInfo><SignatureValue>o/pUbSQAukz7+ZYAWhnA0AJbIlyyCPL7bKVEM2lVqbrXt7cyey+umkCXamuOgsWPVUKBMkXtMH8L\n5etLmD0getWIhTGhzOnDCk+gtIPfL4jMo9tkEuOCROQAXCci23VFscKcrkB+3X6h4wEOtA2APhOY\nB+wvC794o8/82ffjP79aVAi57rp3Wmzx+9pe9yMwoJuljAy2sc2tIMgdQGWVmOGBpQm3JqsidyzI\nJWG2kjnc7pDXK9pwYzXoKiqUqqrut90d+kQqRyv7MSZXR50HFqD/LI69h68b7P8Bjo3bPXOhNXGR\n9YCoemH6EkfCJxp2gIjzjWW+l2Hj2EsFQi8YXw==</SignatureValue></Signature></root>'; '<?xml version="1.0" encoding="UTF-8"?><root>\n <License id="Terms">\n <MachineBindings>\n <Binding Key="1" Value="576562626572264761624c65526f7578"/>\n <Binding Key="2" Value="576562626572264761624c65526f7578"/>\n </MachineBindings>\n <MachineID Value="D7nTUnjNAmtsUMcnoyrqkgIbYdM="/>\n <SerialHash Value="2033b8ac3e6faa3742ca9f0bfae44d18f2a96b80"/>\n <Features>\n <Feature Value="33"/>\n <Feature Value="1"/>\n <Feature Value="12"/>\n <Feature Value="2"/>\n <Feature Value="24"/>\n <Feature Value="3"/>\n <Feature Value="36"/>\n <Feature Value="17"/>\n <Feature Value="19"/>\n <Feature Value="62"/>\n </Features>\n <DeveloperData Value="AQAAAEY0LUJHUlgtWEQ0RS1aQ1dWLUM1SlctR0RIQg=="/>\n <SerialMasked Value="F4-BGRX-XD4E-ZCWV-C5JW-XXXX"/>\n <StartDate Value="2021-02-08T00:00:00"/>\n <UpdateDate Value="2021-02-09T00:34:57"/>\n <InitialActivationDate Value="2021-02-08T00:34:56"/>\n <LicenseVersion Value="6.x"/>\n <ClientProvidedVersion Value="2018.4.30f1"/>\n <AlwaysOnline Value="false"/>\n <Entitlements>\n <Entitlement Ns="unity_editor" Tag="UnityPersonal" Type="EDITOR" ValidTo="9999-12-31T00:00:00"/>\n <Entitlement Ns="unity_editor" Tag="DarkSkin" Type="EDITOR_FEATURE" ValidTo="9999-12-31T00:00:00"/>\n </Entitlements>\n </License>\n<Signature xmlns="http://www.w3.org/2000/09/xmldsig#"><SignedInfo><CanonicalizationMethod Algorithm="http://www.w3.org/TR/2001/REC-xml-c14n-20010315#WithComments"/><SignatureMethod Algorithm="http://www.w3.org/2000/09/xmldsig#rsa-sha1"/><Reference URI="#Terms"><Transforms><Transform Algorithm="http://www.w3.org/2000/09/xmldsig#enveloped-signature"/></Transforms><DigestMethod Algorithm="http://www.w3.org/2000/09/xmldsig#sha1"/><DigestValue>m0Db8UK+ktnOLJBtHybkfetpcKo=</DigestValue></Reference></SignedInfo><SignatureValue>o/pUbSQAukz7+ZYAWhnA0AJbIlyyCPL7bKVEM2lVqbrXt7cyey+umkCXamuOgsWPVUKBMkXtMH8L\n5etLmD0getWIhTGhzOnDCk+gtIPfL4jMo9tkEuOCROQAXCci23VFscKcrkB+3X6h4wEOtA2APhOY\nB+wvC794o8/82ffjP79aVAi57rp3Wmzx+9pe9yMwoJuljAy2sc2tIMgdQGWVmOGBpQm3JqsidyzI\nJWG2kjnc7pDXK9pwYzXoKiqUqqrut90d+kQqRyv7MSZXR50HFqD/LI69h68b7P8Bjo3bPXOhNXGR\n9YCoemH6EkfCJxp2gIjzjWW+l2Hj2EsFQi8YXw==</SignatureValue></Signature></root>';
afterEach(() => { afterEach(() => {
vi.clearAllMocks(); jest.clearAllMocks();
vi.restoreAllMocks(); jest.restoreAllMocks();
}); });
beforeEach(() => { beforeEach(() => {
vi.spyOn(Versioning, 'determineBuildVersion').mockImplementation(async () => '1.3.37'); jest.spyOn(Versioning, 'determineBuildVersion').mockImplementation(async () => '1.3.37');
process.env.UNITY_LICENSE = testLicense; // Todo - Don't use process.env directly, that's what the input model class is for. process.env.UNITY_LICENSE = testLicense; // Todo - Don't use process.env directly, that's what the input model class is for.
}); });
@@ -26,20 +25,20 @@ describe('BuildParameters', () => {
}); });
it('determines the version only once', async () => { it('determines the version only once', async () => {
vi.spyOn(Versioning, 'determineBuildVersion').mockImplementation(async () => '1.3.37'); jest.spyOn(Versioning, 'determineBuildVersion').mockImplementation(async () => '1.3.37');
await BuildParameters.create(); await BuildParameters.create();
await expect(Versioning.determineBuildVersion).toHaveBeenCalledTimes(1); await expect(Versioning.determineBuildVersion).toHaveBeenCalledTimes(1);
}); });
it('determines the unity version only once', async () => { it('determines the unity version only once', async () => {
vi.spyOn(UnityVersioning, 'determineUnityVersion').mockImplementation(() => '2019.2.11f1'); jest.spyOn(UnityVersioning, 'determineUnityVersion').mockImplementation(() => '2019.2.11f1');
await BuildParameters.create(); await BuildParameters.create();
expect(UnityVersioning.determineUnityVersion).toHaveBeenCalledTimes(1); expect(UnityVersioning.determineUnityVersion).toHaveBeenCalledTimes(1);
}); });
it('returns the android version code with provided input', async () => { it('returns the android version code with provided input', async () => {
const mockValue = '42'; const mockValue = '42';
vi.spyOn(Input, 'androidVersionCode', 'get').mockReturnValue(mockValue); jest.spyOn(Input, 'androidVersionCode', 'get').mockReturnValue(mockValue);
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(
expect.objectContaining({ androidVersionCode: mockValue }), expect.objectContaining({ androidVersionCode: mockValue }),
); );
@@ -47,59 +46,49 @@ describe('BuildParameters', () => {
it('returns the android version code from version by default', async () => { it('returns the android version code from version by default', async () => {
const mockValue = ''; const mockValue = '';
vi.spyOn(Input, 'androidVersionCode', 'get').mockReturnValue(mockValue); jest.spyOn(Input, 'androidVersionCode', 'get').mockReturnValue(mockValue);
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(
expect.objectContaining({ androidVersionCode: '1003037' }), expect.objectContaining({ androidVersionCode: '1003037' }),
); );
}); });
it('determines the android sdk manager parameters only once', async () => { it('determines the android sdk manager parameters only once', async () => {
vi.spyOn(AndroidVersioning, 'determineSdkManagerParameters').mockImplementation( jest.spyOn(AndroidVersioning, 'determineSdkManagerParameters').mockImplementation(() => 'platforms;android-30');
() => 'platforms;android-30',
);
await BuildParameters.create(); await BuildParameters.create();
expect(AndroidVersioning.determineSdkManagerParameters).toHaveBeenCalledTimes(1); expect(AndroidVersioning.determineSdkManagerParameters).toHaveBeenCalledTimes(1);
}); });
it('returns the targetPlatform', async () => { it('returns the targetPlatform', async () => {
const mockValue = 'somePlatform'; const mockValue = 'somePlatform';
vi.spyOn(Input, 'targetPlatform', 'get').mockReturnValue(mockValue); jest.spyOn(Input, 'targetPlatform', 'get').mockReturnValue(mockValue);
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(expect.objectContaining({ targetPlatform: mockValue }));
expect.objectContaining({ targetPlatform: mockValue }),
);
}); });
it('returns the project path', async () => { it('returns the project path', async () => {
const mockValue = 'path/to/project'; const mockValue = 'path/to/project';
vi.spyOn(UnityVersioning, 'determineUnityVersion').mockImplementation(() => '2019.2.11f1'); jest.spyOn(UnityVersioning, 'determineUnityVersion').mockImplementation(() => '2019.2.11f1');
vi.spyOn(Input, 'projectPath', 'get').mockReturnValue(mockValue); jest.spyOn(Input, 'projectPath', 'get').mockReturnValue(mockValue);
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(expect.objectContaining({ projectPath: mockValue }));
expect.objectContaining({ projectPath: mockValue }),
);
}); });
it('returns the build profile', async () => { it('returns the build profile', async () => {
const mockValue = 'path/to/build_profile.asset'; const mockValue = 'path/to/build_profile.asset';
vi.spyOn(Input, 'buildProfile', 'get').mockReturnValue(mockValue); jest.spyOn(Input, 'buildProfile', 'get').mockReturnValue(mockValue);
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(expect.objectContaining({ buildProfile: mockValue }));
expect.objectContaining({ buildProfile: mockValue }),
);
}); });
it('returns the build name', async () => { it('returns the build name', async () => {
const mockValue = 'someBuildName'; const mockValue = 'someBuildName';
vi.spyOn(Input, 'buildName', 'get').mockReturnValue(mockValue); jest.spyOn(Input, 'buildName', 'get').mockReturnValue(mockValue);
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(expect.objectContaining({ buildName: mockValue }));
expect.objectContaining({ buildName: mockValue }),
);
}); });
it('returns the build path', async () => { it('returns the build path', async () => {
const mockPath = 'somePath'; const mockPath = 'somePath';
const mockPlatform = 'somePlatform'; const mockPlatform = 'somePlatform';
const expectedBuildPath = `${mockPath}/${mockPlatform}`; const expectedBuildPath = `${mockPath}/${mockPlatform}`;
vi.spyOn(Input, 'buildsPath', 'get').mockReturnValue(mockPath); jest.spyOn(Input, 'buildsPath', 'get').mockReturnValue(mockPath);
vi.spyOn(Input, 'targetPlatform', 'get').mockReturnValue(mockPlatform); jest.spyOn(Input, 'targetPlatform', 'get').mockReturnValue(mockPlatform);
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(
expect.objectContaining({ buildPath: expectedBuildPath }), expect.objectContaining({ buildPath: expectedBuildPath }),
); );
@@ -109,29 +98,24 @@ describe('BuildParameters', () => {
const mockValue = 'someBuildName'; const mockValue = 'someBuildName';
const mockPlatform = 'somePlatform'; const mockPlatform = 'somePlatform';
vi.spyOn(Input, 'buildName', 'get').mockReturnValue(mockValue); jest.spyOn(Input, 'buildName', 'get').mockReturnValue(mockValue);
vi.spyOn(Input, 'targetPlatform', 'get').mockReturnValue(mockPlatform); jest.spyOn(Input, 'targetPlatform', 'get').mockReturnValue(mockPlatform);
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(expect.objectContaining({ buildFile: mockValue }));
expect.objectContaining({ buildFile: mockValue }),
);
}); });
test.each` test.each`
targetPlatform | expectedExtension | androidExportType | linux64RemoveExecutableExtension targetPlatform | expectedExtension | androidExportType
${Platform.types.Android} | ${'.apk'} | ${'androidPackage'} | ${false} ${Platform.types.Android} | ${'.apk'} | ${'androidPackage'}
${Platform.types.Android} | ${'.aab'} | ${'androidAppBundle'} | ${true} ${Platform.types.Android} | ${'.aab'} | ${'androidAppBundle'}
${Platform.types.Android} | ${''} | ${'androidStudioProject'} | ${false} ${Platform.types.Android} | ${''} | ${'androidStudioProject'}
${Platform.types.StandaloneWindows} | ${'.exe'} | ${'n/a'} | ${true} ${Platform.types.StandaloneWindows} | ${'.exe'} | ${'n/a'}
${Platform.types.StandaloneWindows64} | ${'.exe'} | ${'n/a'} | ${false} ${Platform.types.StandaloneWindows64} | ${'.exe'} | ${'n/a'}
${Platform.types.StandaloneLinux64} | ${'.x86_64'} | ${'n/a'} | ${false}
${Platform.types.StandaloneLinux64} | ${''} | ${'n/a'} | ${true}
`( `(
'appends $expectedExtension for $targetPlatform with linux64RemoveExecutableExtension=$linux64RemoveExecutableExtension', 'appends $expectedExtension for $targetPlatform with androidExportType $androidExportType',
async ({ targetPlatform, expectedExtension, androidExportType, linux64RemoveExecutableExtension }) => { async ({ targetPlatform, expectedExtension, androidExportType }) => {
vi.spyOn(Input, 'targetPlatform', 'get').mockReturnValue(targetPlatform); jest.spyOn(Input, 'targetPlatform', 'get').mockReturnValue(targetPlatform);
vi.spyOn(Input, 'buildName', 'get').mockReturnValue(targetPlatform); jest.spyOn(Input, 'buildName', 'get').mockReturnValue(targetPlatform);
vi.spyOn(Input, 'androidExportType', 'get').mockReturnValue(androidExportType); jest.spyOn(Input, 'androidExportType', 'get').mockReturnValue(androidExportType);
vi.spyOn(Input, 'linux64RemoveExecutableExtension', 'get').mockReturnValue(linux64RemoveExecutableExtension);
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(
expect.objectContaining({ buildFile: `${targetPlatform}${expectedExtension}` }), expect.objectContaining({ buildFile: `${targetPlatform}${expectedExtension}` }),
); );
@@ -148,26 +132,22 @@ describe('BuildParameters', () => {
`( `(
'androidSymbolType is set to $androidSymbolType when targetPlatform is $targetPlatform and input targetSymbolType is $androidSymbolType', 'androidSymbolType is set to $androidSymbolType when targetPlatform is $targetPlatform and input targetSymbolType is $androidSymbolType',
async ({ targetPlatform, androidSymbolType }) => { async ({ targetPlatform, androidSymbolType }) => {
vi.spyOn(Input, 'targetPlatform', 'get').mockReturnValue(targetPlatform); jest.spyOn(Input, 'targetPlatform', 'get').mockReturnValue(targetPlatform);
vi.spyOn(Input, 'androidSymbolType', 'get').mockReturnValue(androidSymbolType); jest.spyOn(Input, 'androidSymbolType', 'get').mockReturnValue(androidSymbolType);
vi.spyOn(Input, 'buildName', 'get').mockReturnValue(targetPlatform); jest.spyOn(Input, 'buildName', 'get').mockReturnValue(targetPlatform);
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(expect.objectContaining({ androidSymbolType }));
expect.objectContaining({ androidSymbolType }),
);
}, },
); );
it('returns the build method', async () => { it('returns the build method', async () => {
const mockValue = 'Namespace.ClassName.BuildMethod'; const mockValue = 'Namespace.ClassName.BuildMethod';
vi.spyOn(Input, 'buildMethod', 'get').mockReturnValue(mockValue); jest.spyOn(Input, 'buildMethod', 'get').mockReturnValue(mockValue);
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(expect.objectContaining({ buildMethod: mockValue }));
expect.objectContaining({ buildMethod: mockValue }),
);
}); });
it('returns the android keystore name', async () => { it('returns the android keystore name', async () => {
const mockValue = 'keystore.keystore'; const mockValue = 'keystore.keystore';
vi.spyOn(Input, 'androidKeystoreName', 'get').mockReturnValue(mockValue); jest.spyOn(Input, 'androidKeystoreName', 'get').mockReturnValue(mockValue);
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(
expect.objectContaining({ androidKeystoreName: mockValue }), expect.objectContaining({ androidKeystoreName: mockValue }),
); );
@@ -175,7 +155,7 @@ describe('BuildParameters', () => {
it('returns the android keystore base64-encoded content', async () => { it('returns the android keystore base64-encoded content', async () => {
const mockValue = 'secret'; const mockValue = 'secret';
vi.spyOn(Input, 'androidKeystoreBase64', 'get').mockReturnValue(mockValue); jest.spyOn(Input, 'androidKeystoreBase64', 'get').mockReturnValue(mockValue);
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(
expect.objectContaining({ androidKeystoreBase64: mockValue }), expect.objectContaining({ androidKeystoreBase64: mockValue }),
); );
@@ -183,7 +163,7 @@ describe('BuildParameters', () => {
it('returns the android keystore pass', async () => { it('returns the android keystore pass', async () => {
const mockValue = 'secret'; const mockValue = 'secret';
vi.spyOn(Input, 'androidKeystorePass', 'get').mockReturnValue(mockValue); jest.spyOn(Input, 'androidKeystorePass', 'get').mockReturnValue(mockValue);
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(
expect.objectContaining({ androidKeystorePass: mockValue }), expect.objectContaining({ androidKeystorePass: mockValue }),
); );
@@ -191,7 +171,7 @@ describe('BuildParameters', () => {
it('returns the android keyalias name', async () => { it('returns the android keyalias name', async () => {
const mockValue = 'secret'; const mockValue = 'secret';
vi.spyOn(Input, 'androidKeyaliasName', 'get').mockReturnValue(mockValue); jest.spyOn(Input, 'androidKeyaliasName', 'get').mockReturnValue(mockValue);
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(
expect.objectContaining({ androidKeyaliasName: mockValue }), expect.objectContaining({ androidKeyaliasName: mockValue }),
); );
@@ -199,7 +179,7 @@ describe('BuildParameters', () => {
it('returns the android keyalias pass', async () => { it('returns the android keyalias pass', async () => {
const mockValue = 'secret'; const mockValue = 'secret';
vi.spyOn(Input, 'androidKeyaliasPass', 'get').mockReturnValue(mockValue); jest.spyOn(Input, 'androidKeyaliasPass', 'get').mockReturnValue(mockValue);
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(
expect.objectContaining({ androidKeyaliasPass: mockValue }), expect.objectContaining({ androidKeyaliasPass: mockValue }),
); );
@@ -207,7 +187,7 @@ describe('BuildParameters', () => {
it('returns the android target sdk version', async () => { it('returns the android target sdk version', async () => {
const mockValue = 'AndroidApiLevelAuto'; const mockValue = 'AndroidApiLevelAuto';
vi.spyOn(Input, 'androidTargetSdkVersion', 'get').mockReturnValue(mockValue); jest.spyOn(Input, 'androidTargetSdkVersion', 'get').mockReturnValue(mockValue);
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(
expect.objectContaining({ androidTargetSdkVersion: mockValue }), expect.objectContaining({ androidTargetSdkVersion: mockValue }),
); );
@@ -215,7 +195,7 @@ describe('BuildParameters', () => {
it('returns the unity licensing server address', async () => { it('returns the unity licensing server address', async () => {
const mockValue = 'http://example.com'; const mockValue = 'http://example.com';
vi.spyOn(Input, 'unityLicensingServer', 'get').mockReturnValue(mockValue); jest.spyOn(Input, 'unityLicensingServer', 'get').mockReturnValue(mockValue);
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(
expect.objectContaining({ unityLicensingServer: mockValue }), expect.objectContaining({ unityLicensingServer: mockValue }),
); );
@@ -230,25 +210,14 @@ describe('BuildParameters', () => {
const mockValue = '123'; const mockValue = '123';
delete process.env.UNITY_LICENSE; // Need to delete this as it is set for every test currently delete process.env.UNITY_LICENSE; // Need to delete this as it is set for every test currently
process.env.UNITY_SERIAL = mockValue; process.env.UNITY_SERIAL = mockValue;
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(expect.objectContaining({ unitySerial: mockValue }));
expect.objectContaining({ unitySerial: mockValue }),
);
delete process.env.UNITY_SERIAL; delete process.env.UNITY_SERIAL;
}); });
it('returns the custom parameters', async () => { it('returns the custom parameters', async () => {
const mockValue = '-profile SomeProfile -someBoolean -someValue exampleValue'; const mockValue = '-profile SomeProfile -someBoolean -someValue exampleValue';
vi.spyOn(Input, 'customParameters', 'get').mockReturnValue(mockValue); jest.spyOn(Input, 'customParameters', 'get').mockReturnValue(mockValue);
await expect(BuildParameters.create()).resolves.toEqual( await expect(BuildParameters.create()).resolves.toEqual(expect.objectContaining({ customParameters: mockValue }));
expect.objectContaining({ customParameters: mockValue }),
);
});
it.each([true, false])('returns the flag for useHostNetwork when %s', async (mockValue) => {
vi.spyOn(Input, 'useHostNetwork', 'get').mockReturnValue(mockValue);
await expect(BuildParameters.create()).resolves.toEqual(
expect.objectContaining({ useHostNetwork: mockValue }),
);
}); });
}); });
}); });
+73 -48
View File
@@ -1,13 +1,17 @@
import { customAlphabet } from 'nanoid'; import { customAlphabet } from 'nanoid';
import AndroidVersioning from './android-versioning'; import AndroidVersioning from './android-versioning';
import CloudRunnerConstants from './cloud-runner/options/cloud-runner-constants';
import CloudRunnerBuildGuid from './cloud-runner/options/cloud-runner-guid';
import Input from './input'; import Input from './input';
import Platform from './platform'; import Platform from './platform';
import UnityVersioning from './unity-versioning'; import UnityVersioning from './unity-versioning';
import Versioning from './versioning'; import Versioning from './versioning';
import { GitRepoReader } from './input-readers/git-repo'; import { GitRepoReader } from './input-readers/git-repo';
import { GithubCliReader } from './input-readers/github-cli'; import { GithubCliReader } from './input-readers/github-cli';
import { PluginOptions } from './plugin-options'; import { Cli } from './cli/cli';
import GitHub from './github'; import GitHub from './github';
import CloudRunnerOptions from './cloud-runner/options/cloud-runner-options';
import CloudRunner from './cloud-runner/cloud-runner';
import * as core from '@actions/core'; import * as core from '@actions/core';
class BuildParameters { class BuildParameters {
@@ -48,49 +52,61 @@ class BuildParameters {
public containerRegistryImageVersion!: string; public containerRegistryImageVersion!: string;
public customParameters!: string; public customParameters!: string;
public useHostNetwork!: boolean;
public sshAgent!: string; public sshAgent!: string;
public sshPublicKeysDirectoryPath!: string; public sshPublicKeysDirectoryPath!: string;
public providerStrategy!: string; public providerStrategy!: string;
public gitPrivateToken!: string; public gitPrivateToken!: string;
public awsStackName!: string;
public kubeConfig!: string;
public containerMemory!: string;
public containerCpu!: string;
public kubeVolumeSize!: string;
public kubeVolume!: string;
public kubeStorageClass!: string;
public runAsHostUser!: string; public runAsHostUser!: string;
public chownFilesTo!: string; public chownFilesTo!: string;
public commandHooks!: string;
public pullInputList!: string[];
public inputPullCommand!: string;
public cacheKey!: string;
public postBuildContainerHooks!: string;
public preBuildContainerHooks!: string;
public customJob!: string;
public runNumber!: string; public runNumber!: string;
public branch!: string; public branch!: string;
public githubRepo!: string; public githubRepo!: string;
public gitSha!: string; public gitSha!: string;
public logId!: string; public logId!: string;
public buildGuid!: string; public buildGuid!: string;
public cloudRunnerBranch!: string;
public cloudRunnerDebug!: boolean | undefined;
public buildPlatform!: string | undefined; public buildPlatform!: string | undefined;
public isCliMode!: boolean; public isCliMode!: boolean;
public maxRetainedWorkspaces!: number;
public useLargePackages!: boolean;
public useCompressionStrategy!: boolean;
public garbageMaxAge!: number;
public githubChecks!: boolean;
public asyncWorkflow!: boolean;
public githubCheckId!: string;
public finalHooks!: string[];
public skipLfs!: boolean;
public skipCache!: boolean;
public cacheUnityInstallationOnMac!: boolean; public cacheUnityInstallationOnMac!: boolean;
public unityHubVersionOnMac!: string; public unityHubVersionOnMac!: string;
public dockerWorkspacePath!: string; public dockerWorkspacePath!: string;
public static shouldUseRetainedWorkspaceMode(buildParameters: BuildParameters) {
return buildParameters.maxRetainedWorkspaces > 0 && CloudRunner.lockedWorkspace !== ``;
}
static async create(): Promise<BuildParameters> { static async create(): Promise<BuildParameters> {
const buildFile = this.parseBuildFile( const buildFile = this.parseBuildFile(Input.buildName, Input.targetPlatform, Input.androidExportType);
Input.buildName, const editorVersion = UnityVersioning.determineUnityVersion(Input.projectPath, Input.unityVersion);
Input.targetPlatform, const buildVersion = await Versioning.determineBuildVersion(Input.versioningStrategy, Input.specifiedVersion);
Input.androidExportType, const androidVersionCode = AndroidVersioning.determineVersionCode(buildVersion, Input.androidVersionCode);
Input.linux64RemoveExecutableExtension, const androidSdkManagerParameters = AndroidVersioning.determineSdkManagerParameters(Input.androidTargetSdkVersion);
);
const editorVersion = UnityVersioning.determineUnityVersion(
Input.projectPath,
Input.unityVersion,
);
const buildVersion = await Versioning.determineBuildVersion(
Input.versioningStrategy,
Input.specifiedVersion,
);
const androidVersionCode = AndroidVersioning.determineVersionCode(
buildVersion,
Input.androidVersionCode,
);
const androidSdkManagerParameters = AndroidVersioning.determineSdkManagerParameters(
Input.androidTargetSdkVersion,
);
const androidSymbolExportType = Input.androidSymbolType; const androidSymbolExportType = Input.androidSymbolType;
if (Platform.isAndroid(Input.targetPlatform)) { if (Platform.isAndroid(Input.targetPlatform)) {
@@ -129,9 +145,6 @@ class BuildParameters {
core.setSecret(`${unitySerial.slice(0, -4)}XXXX`); core.setSecret(`${unitySerial.slice(0, -4)}XXXX`);
} }
const providerStrategy =
Input.getInput('providerStrategy') || (PluginOptions.isPluginMode ? 'aws' : 'local');
return { return {
editorVersion, editorVersion,
customImage: Input.customImage, customImage: Input.customImage,
@@ -160,7 +173,6 @@ class BuildParameters {
androidExportType: Input.androidExportType, androidExportType: Input.androidExportType,
androidSymbolType: androidSymbolExportType, androidSymbolType: androidSymbolExportType,
customParameters: Input.customParameters, customParameters: Input.customParameters,
useHostNetwork: Input.useHostNetwork,
sshAgent: Input.sshAgent, sshAgent: Input.sshAgent,
sshPublicKeysDirectoryPath: Input.sshPublicKeysDirectoryPath, sshPublicKeysDirectoryPath: Input.sshPublicKeysDirectoryPath,
gitPrivateToken: Input.gitPrivateToken ?? (await GithubCliReader.GetGitHubAuthToken()), gitPrivateToken: Input.gitPrivateToken ?? (await GithubCliReader.GetGitHubAuthToken()),
@@ -172,31 +184,48 @@ class BuildParameters {
dockerIsolationMode: Input.dockerIsolationMode, dockerIsolationMode: Input.dockerIsolationMode,
containerRegistryRepository: Input.containerRegistryRepository, containerRegistryRepository: Input.containerRegistryRepository,
containerRegistryImageVersion: Input.containerRegistryImageVersion, containerRegistryImageVersion: Input.containerRegistryImageVersion,
providerStrategy, providerStrategy: CloudRunnerOptions.providerStrategy,
buildPlatform: providerStrategy !== 'local' ? 'linux' : process.platform, buildPlatform: CloudRunnerOptions.buildPlatform,
kubeConfig: CloudRunnerOptions.kubeConfig,
containerMemory: CloudRunnerOptions.containerMemory,
containerCpu: CloudRunnerOptions.containerCpu,
kubeVolumeSize: CloudRunnerOptions.kubeVolumeSize,
kubeVolume: CloudRunnerOptions.kubeVolume,
postBuildContainerHooks: CloudRunnerOptions.postBuildContainerHooks,
preBuildContainerHooks: CloudRunnerOptions.preBuildContainerHooks,
customJob: CloudRunnerOptions.customJob,
runNumber: Input.runNumber, runNumber: Input.runNumber,
branch: Input.branch.replace('/head', '') || (await GitRepoReader.GetBranch()), branch: Input.branch.replace('/head', '') || (await GitRepoReader.GetBranch()),
githubRepo: cloudRunnerBranch: CloudRunnerOptions.cloudRunnerBranch.split('/').reverse()[0],
(Input.githubRepo ?? (await GitRepoReader.GetRemote())) || 'game-ci/unity-builder', cloudRunnerDebug: CloudRunnerOptions.cloudRunnerDebug,
githubRepo: (Input.githubRepo ?? (await GitRepoReader.GetRemote())) || 'game-ci/unity-builder',
isCliMode: Cli.isCliMode,
awsStackName: CloudRunnerOptions.awsStackName,
gitSha: Input.gitSha, gitSha: Input.gitSha,
logId: customAlphabet('0123456789abcdefghijklmnopqrstuvwxyz', 9)(), logId: customAlphabet(CloudRunnerConstants.alphabet, 9)(),
buildGuid: `${Input.runNumber}-${Input.targetPlatform.toLowerCase().replace('standalone', '')}-${customAlphabet( buildGuid: CloudRunnerBuildGuid.generateGuid(Input.runNumber, Input.targetPlatform),
'0123456789abcdefghijklmnopqrstuvwxyz', commandHooks: CloudRunnerOptions.commandHooks,
4, inputPullCommand: CloudRunnerOptions.inputPullCommand,
)()}`, pullInputList: CloudRunnerOptions.pullInputList,
isCliMode: PluginOptions.isPluginMode, kubeStorageClass: CloudRunnerOptions.kubeStorageClass,
cacheKey: CloudRunnerOptions.cacheKey,
maxRetainedWorkspaces: Number.parseInt(CloudRunnerOptions.maxRetainedWorkspaces),
useLargePackages: CloudRunnerOptions.useLargePackages,
useCompressionStrategy: CloudRunnerOptions.useCompressionStrategy,
garbageMaxAge: CloudRunnerOptions.garbageMaxAge,
githubChecks: CloudRunnerOptions.githubChecks,
asyncWorkflow: CloudRunnerOptions.asyncCloudRunner,
githubCheckId: CloudRunnerOptions.githubCheckId,
finalHooks: CloudRunnerOptions.finalHooks,
skipLfs: CloudRunnerOptions.skipLfs,
skipCache: CloudRunnerOptions.skipCache,
cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac, cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac,
unityHubVersionOnMac: Input.unityHubVersionOnMac, unityHubVersionOnMac: Input.unityHubVersionOnMac,
dockerWorkspacePath: Input.dockerWorkspacePath, dockerWorkspacePath: Input.dockerWorkspacePath,
}; };
} }
static parseBuildFile( static parseBuildFile(filename: string, platform: string, androidExportType: string): string {
filename: string,
platform: string,
androidExportType: string,
linux64RemoveExecutableExtension: boolean,
): string {
if (Platform.isWindows(platform)) { if (Platform.isWindows(platform)) {
return `${filename}.exe`; return `${filename}.exe`;
} }
@@ -216,10 +245,6 @@ class BuildParameters {
} }
} }
if (platform === Platform.types.StandaloneLinux64 && !linux64RemoveExecutableExtension) {
return `${filename}.x86_64`;
}
return filename; return filename;
} }
+1 -2
View File
@@ -1,7 +1,6 @@
import { describe, it, expect, beforeEach, afterEach, beforeAll, afterAll, vi } from 'vitest';
import Cache from './cache'; import Cache from './cache';
vi.mock('./input'); jest.mock('./input');
describe('Cache', () => { describe('Cache', () => {
describe('Verification', () => { describe('Verification', () => {
+45
View File
@@ -0,0 +1,45 @@
export class CliFunctionsRepository {
private static targets: any[] = [];
public static PushCliFunction(
target: any,
propertyKey: string,
descriptor: PropertyDescriptor,
key: string,
description: string,
) {
CliFunctionsRepository.targets.push({
target,
propertyKey,
descriptor,
key,
description,
});
}
public static GetCliFunctions(key: any) {
const results = CliFunctionsRepository.targets.find((x) => x.key === key);
if (results === undefined || results.length === 0) {
throw new Error(`no CLI mode found for ${key}`);
}
return results;
}
public static GetAllCliModes() {
return CliFunctionsRepository.targets.map((x) => {
return {
key: x.key,
description: x.description,
};
});
}
// eslint-disable-next-line no-unused-vars
public static PushCliFunctionSource(cliFunction: any) {}
}
export function CliFunction(key: string, description: string) {
return (target: any, propertyKey: string, descriptor: PropertyDescriptor) => {
CliFunctionsRepository.PushCliFunction(target, propertyKey, descriptor, key, description);
};
}
+175
View File
@@ -0,0 +1,175 @@
import { Command } from 'commander-ts';
import { BuildParameters, CloudRunner, ImageTag, Input } from '..';
import * as core from '@actions/core';
import { ActionYamlReader } from '../input-readers/action-yaml';
import CloudRunnerLogger from '../cloud-runner/services/core/cloud-runner-logger';
import CloudRunnerQueryOverride from '../cloud-runner/options/cloud-runner-query-override';
import { CliFunction, CliFunctionsRepository } from './cli-functions-repository';
import { Caching } from '../cloud-runner/remote-client/caching';
import { LfsHashing } from '../cloud-runner/services/utility/lfs-hashing';
import { RemoteClient } from '../cloud-runner/remote-client';
import CloudRunnerOptionsReader from '../cloud-runner/options/cloud-runner-options-reader';
import GitHub from '../github';
import { OptionValues } from 'commander';
import { InputKey } from '../input';
export class Cli {
public static options: OptionValues | undefined;
static get isCliMode() {
return Cli.options !== undefined && Cli.options.mode !== undefined && Cli.options.mode !== '';
}
public static query(key: string, alternativeKey: string) {
if (Cli.options && Cli.options[key] !== undefined) {
return Cli.options[key];
}
if (Cli.options && alternativeKey && Cli.options[alternativeKey] !== undefined) {
return Cli.options[alternativeKey];
}
return;
}
public static InitCliMode() {
CliFunctionsRepository.PushCliFunctionSource(RemoteClient);
CliFunctionsRepository.PushCliFunctionSource(Caching);
CliFunctionsRepository.PushCliFunctionSource(LfsHashing);
const program = new Command();
program.version('0.0.1');
const properties = CloudRunnerOptionsReader.GetProperties();
const actionYamlReader: ActionYamlReader = new ActionYamlReader();
for (const element of properties) {
program.option(`--${element} <${element}>`, actionYamlReader.GetActionYamlValue(element));
}
program.option(
'-m, --mode <mode>',
CliFunctionsRepository.GetAllCliModes()
.map((x) => `${x.key} (${x.description})`)
.join(` | `),
);
program.option('--populateOverride <populateOverride>', 'should use override query to pull input false by default');
program.option('--cachePushFrom <cachePushFrom>', 'cache push from source folder');
program.option('--cachePushTo <cachePushTo>', 'cache push to caching folder');
program.option('--artifactName <artifactName>', 'caching artifact name');
program.option('--select <select>', 'select a particular resource');
program.option('--logFile <logFile>', 'output to log file (log stream only)');
program.parse(process.argv);
Cli.options = program.opts();
return Cli.isCliMode;
}
static async RunCli(): Promise<void> {
GitHub.githubInputEnabled = false;
if (Cli.options!['populateOverride'] === `true`) {
await CloudRunnerQueryOverride.PopulateQueryOverrideInput();
}
if (Cli.options!['logInput']) {
Cli.logInput();
}
const results = CliFunctionsRepository.GetCliFunctions(Cli.options?.mode);
CloudRunnerLogger.log(`Entrypoint: ${results.key}`);
Cli.options!.versioning = 'None';
CloudRunner.buildParameters = await BuildParameters.create();
CloudRunner.buildParameters.buildGuid = process.env.BUILD_GUID || ``;
CloudRunnerLogger.log(`Build Params:
${JSON.stringify(CloudRunner.buildParameters, undefined, 4)}
`);
CloudRunner.lockedWorkspace = process.env.LOCKED_WORKSPACE || ``;
CloudRunnerLogger.log(`Locked Workspace: ${CloudRunner.lockedWorkspace}`);
await CloudRunner.setup(CloudRunner.buildParameters);
return await results.target[results.propertyKey](Cli.options);
}
@CliFunction(`print-input`, `prints all input`)
private static logInput() {
core.info(`\n`);
core.info(`INPUT:`);
const properties = CloudRunnerOptionsReader.GetProperties();
for (const element of properties) {
if (
element in Input &&
Input[element as InputKey] !== undefined &&
Input[element as InputKey] !== '' &&
typeof Input[element as InputKey] !== `function` &&
element !== 'length' &&
element !== 'cliOptions' &&
element !== 'prototype'
) {
core.info(`${element} ${Input[element as InputKey]}`);
}
}
core.info(`\n`);
}
@CliFunction(`cli-build`, `runs a cloud runner build`)
public static async CLIBuild(): Promise<string> {
const buildParameter = await BuildParameters.create();
const baseImage = new ImageTag(buildParameter);
return (await CloudRunner.run(buildParameter, baseImage.toString())).BuildResults;
}
@CliFunction(`async-workflow`, `runs a cloud runner build`)
public static async asyncronousWorkflow(): Promise<string> {
const buildParameter = await BuildParameters.create();
const baseImage = new ImageTag(buildParameter);
await CloudRunner.setup(buildParameter);
return (await CloudRunner.run(buildParameter, baseImage.toString())).BuildResults;
}
@CliFunction(`checks-update`, `runs a cloud runner build`)
public static async checksUpdate() {
const buildParameter = await BuildParameters.create();
await CloudRunner.setup(buildParameter);
const input = JSON.parse(process.env.CHECKS_UPDATE || ``);
core.info(`Checks Update ${process.env.CHECKS_UPDATE}`);
if (input.mode === `create`) {
throw new Error(`Not supported: only use update`);
} else if (input.mode === `update`) {
await GitHub.updateGitHubCheckRequest(input.data);
}
}
@CliFunction(`garbage-collect`, `runs garbage collection`)
public static async GarbageCollect(): Promise<string> {
const buildParameter = await BuildParameters.create();
await CloudRunner.setup(buildParameter);
return await CloudRunner.Provider.garbageCollect(``, false, 0, false, false);
}
@CliFunction(`list-resources`, `lists active resources`)
public static async ListResources(): Promise<string[]> {
const buildParameter = await BuildParameters.create();
await CloudRunner.setup(buildParameter);
const result = await CloudRunner.Provider.listResources();
CloudRunnerLogger.log(JSON.stringify(result, undefined, 4));
return result.map((x) => x.Name);
}
@CliFunction(`list-worfklow`, `lists running workflows`)
public static async ListWorfklow(): Promise<string[]> {
const buildParameter = await BuildParameters.create();
await CloudRunner.setup(buildParameter);
return (await CloudRunner.Provider.listWorkflow()).map((x) => x.Name);
}
@CliFunction(`watch`, `follows logs of a running workflow`)
public static async Watch(): Promise<string> {
const buildParameter = await BuildParameters.create();
await CloudRunner.setup(buildParameter);
return await CloudRunner.Provider.watchWorkflow();
}
}
+187
View File
@@ -0,0 +1,187 @@
import AwsBuildPlatform from './providers/aws';
import { BuildParameters, Input } from '..';
import Kubernetes from './providers/k8s';
import CloudRunnerLogger from './services/core/cloud-runner-logger';
import { CloudRunnerStepParameters } from './options/cloud-runner-step-parameters';
import { WorkflowCompositionRoot } from './workflows/workflow-composition-root';
import { CloudRunnerError } from './error/cloud-runner-error';
import { TaskParameterSerializer } from './services/core/task-parameter-serializer';
import * as core from '@actions/core';
import CloudRunnerSecret from './options/cloud-runner-secret';
import { ProviderInterface } from './providers/provider-interface';
import CloudRunnerEnvironmentVariable from './options/cloud-runner-environment-variable';
import TestCloudRunner from './providers/test';
import LocalCloudRunner from './providers/local';
import LocalDockerCloudRunner from './providers/docker';
import GitHub from '../github';
import SharedWorkspaceLocking from './services/core/shared-workspace-locking';
import { FollowLogStreamService } from './services/core/follow-log-stream-service';
import CloudRunnerResult from './services/core/cloud-runner-result';
class CloudRunner {
public static Provider: ProviderInterface;
public static buildParameters: BuildParameters;
private static defaultSecrets: CloudRunnerSecret[];
private static cloudRunnerEnvironmentVariables: CloudRunnerEnvironmentVariable[];
static lockedWorkspace: string = ``;
public static readonly retainedWorkspacePrefix: string = `retained-workspace`;
public static get isCloudRunnerEnvironment() {
return process.env[`GITHUB_ACTIONS`] !== `true`;
}
public static get isCloudRunnerAsyncEnvironment() {
return process.env[`ASYNC_WORKFLOW`] === `true`;
}
public static async setup(buildParameters: BuildParameters) {
CloudRunnerLogger.setup();
CloudRunnerLogger.log(`Setting up cloud runner`);
CloudRunner.buildParameters = buildParameters;
if (CloudRunner.buildParameters.githubCheckId === ``) {
CloudRunner.buildParameters.githubCheckId = await GitHub.createGitHubCheck(CloudRunner.buildParameters.buildGuid);
}
CloudRunner.setupSelectedBuildPlatform();
CloudRunner.defaultSecrets = TaskParameterSerializer.readDefaultSecrets();
CloudRunner.cloudRunnerEnvironmentVariables =
TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameters);
if (GitHub.githubInputEnabled) {
const buildParameterPropertyNames = Object.getOwnPropertyNames(buildParameters);
for (const element of CloudRunner.cloudRunnerEnvironmentVariables) {
// CloudRunnerLogger.log(`Cloud Runner output ${Input.ToEnvVarFormat(element.name)} = ${element.value}`);
core.setOutput(Input.ToEnvVarFormat(element.name), element.value);
}
for (const element of buildParameterPropertyNames) {
// CloudRunnerLogger.log(`Cloud Runner output ${Input.ToEnvVarFormat(element)} = ${buildParameters[element]}`);
core.setOutput(Input.ToEnvVarFormat(element), buildParameters[element]);
}
core.setOutput(
Input.ToEnvVarFormat(`buildArtifact`),
`build-${CloudRunner.buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}`,
);
}
FollowLogStreamService.Reset();
}
private static setupSelectedBuildPlatform() {
CloudRunnerLogger.log(`Cloud Runner platform selected ${CloudRunner.buildParameters.providerStrategy}`);
switch (CloudRunner.buildParameters.providerStrategy) {
case 'k8s':
CloudRunner.Provider = new Kubernetes(CloudRunner.buildParameters);
break;
case 'aws':
CloudRunner.Provider = new AwsBuildPlatform(CloudRunner.buildParameters);
break;
case 'test':
CloudRunner.Provider = new TestCloudRunner();
break;
case 'local-docker':
CloudRunner.Provider = new LocalDockerCloudRunner();
break;
case 'local-system':
CloudRunner.Provider = new LocalCloudRunner();
break;
}
}
static async run(buildParameters: BuildParameters, baseImage: string) {
if (baseImage.includes(`undefined`)) {
throw new Error(`baseImage is undefined`);
}
await CloudRunner.setup(buildParameters);
await CloudRunner.Provider.setupWorkflow(
CloudRunner.buildParameters.buildGuid,
CloudRunner.buildParameters,
CloudRunner.buildParameters.branch,
CloudRunner.defaultSecrets,
);
try {
if (buildParameters.maxRetainedWorkspaces > 0) {
CloudRunner.lockedWorkspace = SharedWorkspaceLocking.NewWorkspaceName();
const result = await SharedWorkspaceLocking.GetLockedWorkspace(
CloudRunner.lockedWorkspace,
CloudRunner.buildParameters.buildGuid,
CloudRunner.buildParameters,
);
if (result) {
CloudRunnerLogger.logLine(`Using retained workspace ${CloudRunner.lockedWorkspace}`);
CloudRunner.cloudRunnerEnvironmentVariables = [
...CloudRunner.cloudRunnerEnvironmentVariables,
{ name: `LOCKED_WORKSPACE`, value: CloudRunner.lockedWorkspace },
];
} else {
CloudRunnerLogger.log(`Max retained workspaces reached ${buildParameters.maxRetainedWorkspaces}`);
buildParameters.maxRetainedWorkspaces = 0;
CloudRunner.lockedWorkspace = ``;
}
}
await CloudRunner.updateStatusWithBuildParameters();
const output = await new WorkflowCompositionRoot().run(
new CloudRunnerStepParameters(
baseImage,
CloudRunner.cloudRunnerEnvironmentVariables,
CloudRunner.defaultSecrets,
),
);
await CloudRunner.Provider.cleanupWorkflow(
CloudRunner.buildParameters,
CloudRunner.buildParameters.branch,
CloudRunner.defaultSecrets,
);
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
if (buildParameters.asyncWorkflow && this.isCloudRunnerEnvironment && this.isCloudRunnerAsyncEnvironment) {
await GitHub.updateGitHubCheck(CloudRunner.buildParameters.buildGuid, `success`, `success`, `completed`);
}
if (BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
const workspace = CloudRunner.lockedWorkspace || ``;
await SharedWorkspaceLocking.ReleaseWorkspace(
workspace,
CloudRunner.buildParameters.buildGuid,
CloudRunner.buildParameters,
);
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(workspace, CloudRunner.buildParameters);
if (isLocked) {
throw new Error(
`still locked after releasing ${await SharedWorkspaceLocking.GetAllLocksForWorkspace(
workspace,
buildParameters,
)}`,
);
}
CloudRunner.lockedWorkspace = ``;
}
await GitHub.triggerWorkflowOnComplete(CloudRunner.buildParameters.finalHooks);
if (buildParameters.constantGarbageCollection) {
CloudRunner.Provider.garbageCollect(``, true, buildParameters.garbageMaxAge, true, true);
}
return new CloudRunnerResult(buildParameters, output, true, true, false);
} catch (error: any) {
CloudRunnerLogger.log(JSON.stringify(error, undefined, 4));
await GitHub.updateGitHubCheck(
CloudRunner.buildParameters.buildGuid,
`Failed - Error ${error?.message || error}`,
`failure`,
`completed`,
);
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
await CloudRunnerError.handleException(error, CloudRunner.buildParameters, CloudRunner.defaultSecrets);
throw error;
}
}
private static async updateStatusWithBuildParameters() {
const content = { ...CloudRunner.buildParameters };
content.gitPrivateToken = ``;
content.unitySerial = ``;
content.unityEmail = ``;
content.unityPassword = ``;
const jsonContent = JSON.stringify(content, undefined, 4);
await GitHub.updateGitHubCheck(jsonContent, CloudRunner.buildParameters.buildGuid);
}
}
export default CloudRunner;
@@ -0,0 +1,15 @@
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import * as core from '@actions/core';
import CloudRunner from '../cloud-runner';
import CloudRunnerSecret from '../options/cloud-runner-secret';
import BuildParameters from '../../build-parameters';
export class CloudRunnerError {
public static async handleException(error: unknown, buildParameters: BuildParameters, secrets: CloudRunnerSecret[]) {
CloudRunnerLogger.error(JSON.stringify(error, undefined, 4));
core.setFailed('Cloud Runner failed');
if (CloudRunner.Provider !== undefined) {
await CloudRunner.Provider.cleanupWorkflow(buildParameters, buildParameters.branch, secrets);
}
}
}
@@ -0,0 +1,4 @@
class CloudRunnerConstants {
static alphabet = '0123456789abcdefghijklmnopqrstuvwxyz';
}
export default CloudRunnerConstants;
@@ -0,0 +1,5 @@
class CloudRunnerEnvironmentVariable {
public name!: string;
public value!: string;
}
export default CloudRunnerEnvironmentVariable;
@@ -0,0 +1,90 @@
import path from 'node:path';
import CloudRunnerOptions from './cloud-runner-options';
import CloudRunner from '../cloud-runner';
import BuildParameters from '../../build-parameters';
export class CloudRunnerFolders {
public static readonly repositoryFolder = 'repo';
public static ToLinuxFolder(folder: string) {
return folder.replace(/\\/g, `/`);
}
// Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
public static get uniqueCloudRunnerJobFolderAbsolute(): string {
return CloudRunner.buildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)
? path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.lockedWorkspace)
: path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.buildParameters.buildGuid);
}
public static get cacheFolderForAllFull(): string {
return path.join('/', CloudRunnerFolders.buildVolumeFolder, CloudRunnerFolders.cacheFolder);
}
public static get cacheFolderForCacheKeyFull(): string {
return path.join(
'/',
CloudRunnerFolders.buildVolumeFolder,
CloudRunnerFolders.cacheFolder,
CloudRunner.buildParameters.cacheKey,
);
}
public static get builderPathAbsolute(): string {
return path.join(
CloudRunnerOptions.useSharedBuilder
? `/${CloudRunnerFolders.buildVolumeFolder}`
: CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute,
`builder`,
);
}
public static get repoPathAbsolute(): string {
return path.join(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute, CloudRunnerFolders.repositoryFolder);
}
public static get projectPathAbsolute(): string {
return path.join(CloudRunnerFolders.repoPathAbsolute, CloudRunner.buildParameters.projectPath);
}
public static get libraryFolderAbsolute(): string {
return path.join(CloudRunnerFolders.projectPathAbsolute, `Library`);
}
public static get projectBuildFolderAbsolute(): string {
return path.join(CloudRunnerFolders.repoPathAbsolute, CloudRunner.buildParameters.buildPath);
}
public static get lfsFolderAbsolute(): string {
return path.join(CloudRunnerFolders.repoPathAbsolute, `.git`, `lfs`);
}
public static get purgeRemoteCaching(): boolean {
return process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined;
}
public static get lfsCacheFolderFull() {
return path.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `lfs`);
}
public static get libraryCacheFolderFull() {
return path.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `Library`);
}
public static get unityBuilderRepoUrl(): string {
return `https://${CloudRunner.buildParameters.gitPrivateToken}@github.com/game-ci/unity-builder.git`;
}
public static get targetBuildRepoUrl(): string {
return `https://${CloudRunner.buildParameters.gitPrivateToken}@github.com/${CloudRunner.buildParameters.githubRepo}.git`;
}
public static get buildVolumeFolder() {
return 'data';
}
public static get cacheFolder() {
return 'cache';
}
}
@@ -0,0 +1,11 @@
import { customAlphabet } from 'nanoid';
import CloudRunnerConstants from './cloud-runner-constants';
class CloudRunnerNamespace {
static generateGuid(runNumber: string | number, platform: string) {
const nanoid = customAlphabet(CloudRunnerConstants.alphabet, 4);
return `${runNumber}-${platform.toLowerCase().replace('standalone', '')}-${nanoid()}`;
}
}
export default CloudRunnerNamespace;
@@ -0,0 +1,10 @@
import Input from '../../input';
import CloudRunnerOptions from './cloud-runner-options';
class CloudRunnerOptionsReader {
static GetProperties() {
return [...Object.getOwnPropertyNames(Input), ...Object.getOwnPropertyNames(CloudRunnerOptions)];
}
}
export default CloudRunnerOptionsReader;
@@ -0,0 +1,283 @@
import { Cli } from '../../cli/cli';
import CloudRunnerQueryOverride from './cloud-runner-query-override';
import GitHub from '../../github';
import * as core from '@actions/core';
class CloudRunnerOptions {
// ### ### ###
// Input Handling
// ### ### ###
public static getInput(query: string): string | undefined {
if (GitHub.githubInputEnabled) {
const coreInput = core.getInput(query);
if (coreInput && coreInput !== '') {
return coreInput;
}
}
const alternativeQuery = CloudRunnerOptions.ToEnvVarFormat(query);
// Query input sources
if (Cli.query(query, alternativeQuery)) {
return Cli.query(query, alternativeQuery);
}
if (CloudRunnerQueryOverride.query(query, alternativeQuery)) {
return CloudRunnerQueryOverride.query(query, alternativeQuery);
}
if (process.env[query] !== undefined) {
return process.env[query];
}
if (alternativeQuery !== query && process.env[alternativeQuery] !== undefined) {
return process.env[alternativeQuery];
}
}
public static ToEnvVarFormat(input: string): string {
if (input.toUpperCase() === input) {
return input;
}
return input
.replace(/([A-Z])/g, ' $1')
.trim()
.toUpperCase()
.replace(/ /g, '_');
}
// ### ### ###
// Provider parameters
// ### ### ###
static get region(): string {
return CloudRunnerOptions.getInput('region') || 'eu-west-2';
}
// ### ### ###
// GitHub parameters
// ### ### ###
static get githubChecks(): boolean {
const value = CloudRunnerOptions.getInput('githubChecks');
return value === `true` || false;
}
static get githubCheckId(): string {
return CloudRunnerOptions.getInput('githubCheckId') || ``;
}
static get githubOwner(): string {
return CloudRunnerOptions.getInput('githubOwner') || CloudRunnerOptions.githubRepo?.split(`/`)[0] || '';
}
static get githubRepoName(): string {
return CloudRunnerOptions.getInput('githubRepoName') || CloudRunnerOptions.githubRepo?.split(`/`)[1] || '';
}
static get finalHooks(): string[] {
return CloudRunnerOptions.getInput('finalHooks')?.split(',') || [];
}
// ### ### ###
// Git syncronization parameters
// ### ### ###
static get githubRepo(): string | undefined {
return CloudRunnerOptions.getInput('GITHUB_REPOSITORY') || CloudRunnerOptions.getInput('GITHUB_REPO') || undefined;
}
static get branch(): string {
if (CloudRunnerOptions.getInput(`GITHUB_REF`)) {
return (
CloudRunnerOptions.getInput(`GITHUB_REF`)?.replace('refs/', '').replace(`head/`, '').replace(`heads/`, '') || ``
);
} else if (CloudRunnerOptions.getInput('branch')) {
return CloudRunnerOptions.getInput('branch') || ``;
} else {
return '';
}
}
// ### ### ###
// Cloud Runner parameters
// ### ### ###
static get buildPlatform(): string {
const input = CloudRunnerOptions.getInput('buildPlatform');
if (input && input !== '') {
return input;
}
if (CloudRunnerOptions.providerStrategy !== 'local') {
return 'linux';
}
return process.platform;
}
static get cloudRunnerBranch(): string {
return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'main';
}
static get providerStrategy(): string {
const provider =
CloudRunnerOptions.getInput('cloudRunnerCluster') || CloudRunnerOptions.getInput('providerStrategy');
if (Cli.isCliMode) {
return provider || 'aws';
}
return provider || 'local';
}
static get containerCpu(): string {
return CloudRunnerOptions.getInput('containerCpu') || `1024`;
}
static get containerMemory(): string {
return CloudRunnerOptions.getInput('containerMemory') || `3072`;
}
static get customJob(): string {
return CloudRunnerOptions.getInput('customJob') || '';
}
// ### ### ###
// Custom commands from files parameters
// ### ### ###
static get containerHookFiles(): string[] {
return CloudRunnerOptions.getInput('containerHookFiles')?.split(`,`) || [];
}
static get commandHookFiles(): string[] {
return CloudRunnerOptions.getInput('commandHookFiles')?.split(`,`) || [];
}
// ### ### ###
// Custom commands from yaml parameters
// ### ### ###
static get commandHooks(): string {
return CloudRunnerOptions.getInput('commandHooks') || '';
}
static get postBuildContainerHooks(): string {
return CloudRunnerOptions.getInput('postBuildContainerHooks') || '';
}
static get preBuildContainerHooks(): string {
return CloudRunnerOptions.getInput('preBuildContainerHooks') || '';
}
// ### ### ###
// Input override handling
// ### ### ###
static get pullInputList(): string[] {
return CloudRunnerOptions.getInput('pullInputList')?.split(`,`) || [];
}
static get inputPullCommand(): string {
const value = CloudRunnerOptions.getInput('inputPullCommand');
if (value === 'gcp-secret-manager') {
return 'gcloud secrets versions access 1 --secret="{0}"';
} else if (value === 'aws-secret-manager') {
return 'aws secretsmanager get-secret-value --secret-id {0}';
}
return value || '';
}
// ### ### ###
// Aws
// ### ### ###
static get awsStackName() {
return CloudRunnerOptions.getInput('awsStackName') || 'game-ci';
}
// ### ### ###
// K8s
// ### ### ###
static get kubeConfig(): string {
return CloudRunnerOptions.getInput('kubeConfig') || '';
}
static get kubeVolume(): string {
return CloudRunnerOptions.getInput('kubeVolume') || '';
}
static get kubeVolumeSize(): string {
return CloudRunnerOptions.getInput('kubeVolumeSize') || '25Gi';
}
static get kubeStorageClass(): string {
return CloudRunnerOptions.getInput('kubeStorageClass') || '';
}
// ### ### ###
// Caching
// ### ### ###
static get cacheKey(): string {
return CloudRunnerOptions.getInput('cacheKey') || CloudRunnerOptions.branch;
}
// ### ### ###
// Utility Parameters
// ### ### ###
static get cloudRunnerDebug(): boolean {
return (
CloudRunnerOptions.getInput(`cloudRunnerTests`) === `true` ||
CloudRunnerOptions.getInput(`cloudRunnerDebug`) === `true` ||
CloudRunnerOptions.getInput(`cloudRunnerDebugTree`) === `true` ||
CloudRunnerOptions.getInput(`cloudRunnerDebugEnv`) === `true` ||
false
);
}
static get skipLfs(): boolean {
return CloudRunnerOptions.getInput(`skipLfs`) === `true`;
}
static get skipCache(): boolean {
return CloudRunnerOptions.getInput(`skipCache`) === `true`;
}
public static get asyncCloudRunner(): boolean {
return CloudRunnerOptions.getInput('asyncCloudRunner') === 'true';
}
public static get useLargePackages(): boolean {
return CloudRunnerOptions.getInput(`useLargePackages`) === `true`;
}
public static get useSharedBuilder(): boolean {
return CloudRunnerOptions.getInput(`useSharedBuilder`) === `true`;
}
public static get useCompressionStrategy(): boolean {
return CloudRunnerOptions.getInput(`useCompressionStrategy`) === `true`;
}
public static get useCleanupCron(): boolean {
return (CloudRunnerOptions.getInput(`useCleanupCron`) || 'true') === 'true';
}
// ### ### ###
// Retained Workspace
// ### ### ###
public static get maxRetainedWorkspaces(): string {
return CloudRunnerOptions.getInput(`maxRetainedWorkspaces`) || `0`;
}
// ### ### ###
// Garbage Collection
// ### ### ###
static get garbageMaxAge(): number {
return Number(CloudRunnerOptions.getInput(`garbageMaxAge`)) || 24;
}
}
export default CloudRunnerOptions;
@@ -0,0 +1,67 @@
import Input from '../../input';
import { GenericInputReader } from '../../input-readers/generic-input-reader';
import CloudRunnerOptions from './cloud-runner-options';
const formatFunction = (value: string, arguments_: any[]) => {
for (const element of arguments_) {
value = value.replace(`{${element.key}}`, element.value);
}
return value;
};
class CloudRunnerQueryOverride {
static queryOverrides: { [key: string]: string } | undefined;
// TODO accept premade secret sources or custom secret source definition yamls
public static query(key: string, alternativeKey: string) {
if (CloudRunnerQueryOverride.queryOverrides && CloudRunnerQueryOverride.queryOverrides[key] !== undefined) {
return CloudRunnerQueryOverride.queryOverrides[key];
}
if (
CloudRunnerQueryOverride.queryOverrides &&
alternativeKey &&
CloudRunnerQueryOverride.queryOverrides[alternativeKey] !== undefined
) {
return CloudRunnerQueryOverride.queryOverrides[alternativeKey];
}
return;
}
private static shouldUseOverride(query: string) {
if (CloudRunnerOptions.inputPullCommand !== '') {
if (CloudRunnerOptions.pullInputList.length > 0) {
const doesInclude =
CloudRunnerOptions.pullInputList.includes(query) ||
CloudRunnerOptions.pullInputList.includes(Input.ToEnvVarFormat(query));
return doesInclude ? true : false;
} else {
return true;
}
}
}
private static async queryOverride(query: string) {
if (!this.shouldUseOverride(query)) {
throw new Error(`Should not be trying to run override query on ${query}`);
}
return await GenericInputReader.Run(
formatFunction(CloudRunnerOptions.inputPullCommand, [{ key: 0, value: query }]),
);
}
public static async PopulateQueryOverrideInput() {
const queries = CloudRunnerOptions.pullInputList;
CloudRunnerQueryOverride.queryOverrides = {};
for (const element of queries) {
if (CloudRunnerQueryOverride.shouldUseOverride(element)) {
CloudRunnerQueryOverride.queryOverrides[element] = await CloudRunnerQueryOverride.queryOverride(element);
}
}
}
}
export default CloudRunnerQueryOverride;
@@ -0,0 +1,6 @@
class CloudRunnerSecret {
public ParameterKey!: string;
public EnvironmentVariable!: string;
public ParameterValue!: string;
}
export default CloudRunnerSecret;
@@ -0,0 +1,3 @@
export class CloudRunnerStatics {
public static readonly logPrefix = `Cloud-Runner`;
}
@@ -0,0 +1,13 @@
import CloudRunnerEnvironmentVariable from './cloud-runner-environment-variable';
import CloudRunnerSecret from './cloud-runner-secret';
export class CloudRunnerStepParameters {
public image: string;
public environment: CloudRunnerEnvironmentVariable[];
public secrets: CloudRunnerSecret[];
constructor(image: string, environmentVariables: CloudRunnerEnvironmentVariable[], secrets: CloudRunnerSecret[]) {
this.image = image;
this.environment = environmentVariables;
this.secrets = secrets;
}
}
@@ -0,0 +1,127 @@
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import * as core from '@actions/core';
import {
CloudFormation,
CreateStackCommand,
CreateStackCommandInput,
DescribeStacksCommand,
DescribeStacksCommandInput,
ListStacksCommand,
Parameter,
UpdateStackCommand,
UpdateStackCommandInput,
waitUntilStackCreateComplete,
waitUntilStackUpdateComplete,
} from '@aws-sdk/client-cloudformation';
import { BaseStackFormation } from './cloud-formations/base-stack-formation';
import crypto from 'node:crypto';
export class AWSBaseStack {
constructor(baseStackName: string) {
this.baseStackName = baseStackName;
}
private baseStackName: string;
async setupBaseStack(CF: CloudFormation) {
const baseStackName = this.baseStackName;
const baseStack = BaseStackFormation.formation;
// Cloud Formation Input
const describeStackInput: DescribeStacksCommandInput = {
StackName: baseStackName,
};
const parametersWithoutHash: Parameter[] = [{ ParameterKey: 'EnvironmentName', ParameterValue: baseStackName }];
const parametersHash = crypto
.createHash('md5')
.update(baseStack + JSON.stringify(parametersWithoutHash))
.digest('hex');
const parameters: Parameter[] = [
...parametersWithoutHash,
...[{ ParameterKey: 'Version', ParameterValue: parametersHash }],
];
const updateInput: UpdateStackCommandInput = {
StackName: baseStackName,
TemplateBody: baseStack,
Parameters: parameters,
Capabilities: ['CAPABILITY_IAM'],
};
const createStackInput: CreateStackCommandInput = {
StackName: baseStackName,
TemplateBody: baseStack,
Parameters: parameters,
Capabilities: ['CAPABILITY_IAM'],
};
const stacks = await CF.send(
new ListStacksCommand({ StackStatusFilter: ['UPDATE_COMPLETE', 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE'] }),
);
const stackNames = stacks.StackSummaries?.map((x) => x.StackName) || [];
const stackExists: Boolean = stackNames.includes(baseStackName) || false;
const describeStack = async () => {
return await CF.send(new DescribeStacksCommand(describeStackInput));
};
try {
if (!stackExists) {
CloudRunnerLogger.log(`${baseStackName} stack does not exist (${JSON.stringify(stackNames)})`);
await CF.send(new CreateStackCommand(createStackInput));
CloudRunnerLogger.log(`created stack (version: ${parametersHash})`);
}
const CFState = await describeStack();
let stack = CFState.Stacks?.[0];
if (!stack) {
throw new Error(`Base stack doesn't exist, even after creation, stackExists check: ${stackExists}`);
}
const stackVersion = stack.Parameters?.find((x) => x.ParameterKey === 'Version')?.ParameterValue;
if (stack.StackStatus === 'CREATE_IN_PROGRESS') {
await waitUntilStackCreateComplete(
{
client: CF,
maxWaitTime: 200,
},
describeStackInput,
);
}
if (stackExists) {
CloudRunnerLogger.log(`Base stack exists (version: ${stackVersion}, local version: ${parametersHash})`);
if (parametersHash !== stackVersion) {
CloudRunnerLogger.log(`Attempting update of base stack`);
try {
await CF.send(new UpdateStackCommand(updateInput));
} catch (error: any) {
if (error['message'].includes('No updates are to be performed')) {
CloudRunnerLogger.log(`No updates are to be performed`);
} else {
CloudRunnerLogger.log(`Update Failed (Stack name: ${baseStackName})`);
CloudRunnerLogger.log(error['message']);
}
CloudRunnerLogger.log(`Continuing...`);
}
} else {
CloudRunnerLogger.log(`No update required`);
}
stack = (await describeStack()).Stacks?.[0];
if (!stack) {
throw new Error(
`Base stack doesn't exist, even after updating and creation, stackExists check: ${stackExists}`,
);
}
if (stack.StackStatus === 'UPDATE_IN_PROGRESS') {
await waitUntilStackUpdateComplete(
{
client: CF,
maxWaitTime: 200,
},
describeStackInput,
);
}
}
CloudRunnerLogger.log('base stack is now ready');
} catch (error) {
core.error(JSON.stringify(await describeStack(), undefined, 4));
throw error;
}
}
}
@@ -0,0 +1,39 @@
import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation';
export class AWSCloudFormationTemplates {
public static getParameterTemplate(p1: string) {
return `
${p1}:
Type: String
Default: ''
`;
}
public static getSecretTemplate(p1: string) {
return `
${p1}Secret:
Type: AWS::SecretsManager::Secret
Properties:
Name: '${p1}'
SecretString: !Ref ${p1}
`;
}
public static getSecretDefinitionTemplate(p1: string, p2: string) {
return `
- Name: '${p1}'
ValueFrom: !Ref ${p2}Secret
`;
}
public static insertAtTemplate(template: string, insertionKey: string, insertion: string) {
const index = template.search(insertionKey) + insertionKey.length + '\n'.length;
template = [template.slice(0, index), insertion, template.slice(index)].join('');
return template;
}
public static readTaskCloudFormationTemplate(): string {
return TaskDefinitionFormation.formation;
}
}
@@ -0,0 +1,16 @@
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { CloudFormation, DescribeStackEventsCommand } from '@aws-sdk/client-cloudformation';
import * as core from '@actions/core';
import CloudRunner from '../../cloud-runner';
export class AWSError {
static async handleStackCreationFailure(error: any, CF: CloudFormation, taskDefStackName: string) {
CloudRunnerLogger.log('aws error: ');
core.error(JSON.stringify(error, undefined, 4));
if (CloudRunner.buildParameters.cloudRunnerDebug) {
CloudRunnerLogger.log('Getting events and resources for task stack');
const events = (await CF.send(new DescribeStackEventsCommand({ StackName: taskDefStackName }))).StackEvents;
CloudRunnerLogger.log(JSON.stringify(events, undefined, 4));
}
}
}
@@ -0,0 +1,227 @@
import {
CloudFormation,
CreateStackCommand,
CreateStackCommandInput,
DescribeStackResourcesCommand,
DescribeStacksCommand,
ListStacksCommand,
waitUntilStackCreateComplete,
} from '@aws-sdk/client-cloudformation';
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import { AWSCloudFormationTemplates } from './aws-cloud-formation-templates';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { AWSError } from './aws-error';
import CloudRunner from '../../cloud-runner';
import { CleanupCronFormation } from './cloud-formations/cleanup-cron-formation';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation';
export class AWSJobStack {
private baseStackName: string;
constructor(baseStackName: string) {
this.baseStackName = baseStackName;
}
public async setupCloudFormations(
CF: CloudFormation,
buildGuid: string,
image: string,
entrypoint: string[],
commands: string,
mountdir: string,
workingdir: string,
secrets: CloudRunnerSecret[],
): Promise<CloudRunnerAWSTaskDef> {
const taskDefStackName = `${this.baseStackName}-${buildGuid}`;
let taskDefCloudFormation = AWSCloudFormationTemplates.readTaskCloudFormationTemplate();
taskDefCloudFormation = taskDefCloudFormation.replace(
`ContainerCpu:
Default: 1024`,
`ContainerCpu:
Default: ${Number.parseInt(CloudRunner.buildParameters.containerCpu)}`,
);
taskDefCloudFormation = taskDefCloudFormation.replace(
`ContainerMemory:
Default: 2048`,
`ContainerMemory:
Default: ${Number.parseInt(CloudRunner.buildParameters.containerMemory)}`,
);
if (!CloudRunnerOptions.asyncCloudRunner) {
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
taskDefCloudFormation,
'# template resources logstream',
TaskDefinitionFormation.streamLogs,
);
}
for (const secret of secrets) {
secret.ParameterKey = `${buildGuid.replace(/[^\dA-Za-z]/g, '')}${secret.ParameterKey.replace(
/[^\dA-Za-z]/g,
'',
)}`;
if (typeof secret.ParameterValue == 'number') {
secret.ParameterValue = `${secret.ParameterValue}`;
}
if (!secret.ParameterValue || secret.ParameterValue === '') {
secrets = secrets.filter((x) => x !== secret);
continue;
}
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
taskDefCloudFormation,
'p1 - input',
AWSCloudFormationTemplates.getParameterTemplate(secret.ParameterKey),
);
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
taskDefCloudFormation,
'# template resources secrets',
AWSCloudFormationTemplates.getSecretTemplate(`${secret.ParameterKey}`),
);
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
taskDefCloudFormation,
'p3 - container def',
AWSCloudFormationTemplates.getSecretDefinitionTemplate(secret.EnvironmentVariable, secret.ParameterKey),
);
}
const secretsMappedToCloudFormationParameters = secrets.map((x) => {
return { ParameterKey: x.ParameterKey.replace(/[^\dA-Za-z]/g, ''), ParameterValue: x.ParameterValue };
});
const logGroupName = `${this.baseStackName}/${taskDefStackName}`;
const parameters = [
{
ParameterKey: 'EnvironmentName',
ParameterValue: this.baseStackName,
},
{
ParameterKey: 'ImageUrl',
ParameterValue: image,
},
{
ParameterKey: 'ServiceName',
ParameterValue: taskDefStackName,
},
{
ParameterKey: 'LogGroupName',
ParameterValue: logGroupName,
},
{
ParameterKey: 'Command',
ParameterValue: 'echo "this template should be overwritten when running a task"',
},
{
ParameterKey: 'EntryPoint',
ParameterValue: entrypoint.join(','),
},
{
ParameterKey: 'WorkingDirectory',
ParameterValue: workingdir,
},
{
ParameterKey: 'EFSMountDirectory',
ParameterValue: mountdir,
},
...secretsMappedToCloudFormationParameters,
];
CloudRunnerLogger.log(
`Starting AWS job with memory: ${CloudRunner.buildParameters.containerMemory} cpu: ${CloudRunner.buildParameters.containerCpu}`,
);
let previousStackExists = true;
while (previousStackExists) {
previousStackExists = false;
const stacks = await CF.send(new ListStacksCommand({}));
if (!stacks.StackSummaries) {
throw new Error('Faild to get stacks');
}
for (let index = 0; index < stacks.StackSummaries.length; index++) {
const element = stacks.StackSummaries[index];
if (element.StackName === taskDefStackName && element.StackStatus !== 'DELETE_COMPLETE') {
previousStackExists = true;
CloudRunnerLogger.log(`Previous stack still exists: ${JSON.stringify(element)}`);
await new Promise((promise) => setTimeout(promise, 5000));
}
}
}
const createStackInput: CreateStackCommandInput = {
StackName: taskDefStackName,
TemplateBody: taskDefCloudFormation,
Capabilities: ['CAPABILITY_IAM'],
Parameters: parameters,
};
try {
CloudRunnerLogger.log(`Creating job aws formation ${taskDefStackName}`);
await CF.send(new CreateStackCommand(createStackInput));
await waitUntilStackCreateComplete(
{
client: CF,
maxWaitTime: 200,
},
{ StackName: taskDefStackName },
);
const describeStack = await CF.send(new DescribeStacksCommand({ StackName: taskDefStackName }));
for (const parameter of parameters) {
if (!describeStack.Stacks?.[0].Parameters?.some((x) => x.ParameterKey === parameter.ParameterKey)) {
throw new Error(`Parameter ${parameter.ParameterKey} not found in stack`);
}
}
} catch (error) {
await AWSError.handleStackCreationFailure(error, CF, taskDefStackName);
throw error;
}
const createCleanupStackInput: CreateStackCommandInput = {
StackName: `${taskDefStackName}-cleanup`,
TemplateBody: CleanupCronFormation.formation,
Capabilities: ['CAPABILITY_IAM'],
Parameters: [
{
ParameterKey: 'StackName',
ParameterValue: taskDefStackName,
},
{
ParameterKey: 'DeleteStackName',
ParameterValue: `${taskDefStackName}-cleanup`,
},
{
ParameterKey: 'TTL',
ParameterValue: `1080`,
},
{
ParameterKey: 'BUILDGUID',
ParameterValue: CloudRunner.buildParameters.buildGuid,
},
{
ParameterKey: 'EnvironmentName',
ParameterValue: this.baseStackName,
},
],
};
if (CloudRunnerOptions.useCleanupCron) {
try {
CloudRunnerLogger.log(`Creating job cleanup formation`);
await CF.send(new CreateStackCommand(createCleanupStackInput));
// await CF.waitFor('stackCreateComplete', { StackName: createCleanupStackInput.StackName }).promise();
} catch (error) {
await AWSError.handleStackCreationFailure(error, CF, taskDefStackName);
throw error;
}
}
const taskDefResources = (
await CF.send(
new DescribeStackResourcesCommand({
StackName: taskDefStackName,
}),
)
).StackResources;
const baseResources = (await CF.send(new DescribeStackResourcesCommand({ StackName: this.baseStackName })))
.StackResources;
return {
taskDefStackName,
taskDefCloudFormation,
taskDefResources,
baseResources,
};
}
}
@@ -0,0 +1,268 @@
import {
DescribeTasksCommand,
ECS,
RunTaskCommand,
RunTaskCommandInput,
Task,
waitUntilTasksRunning,
} from '@aws-sdk/client-ecs';
import {
DescribeStreamCommand,
DescribeStreamCommandOutput,
GetRecordsCommand,
GetRecordsCommandOutput,
GetShardIteratorCommand,
Kinesis,
} from '@aws-sdk/client-kinesis';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import * as core from '@actions/core';
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
import * as zlib from 'node:zlib';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { Input } from '../../..';
import CloudRunner from '../../cloud-runner';
import { CommandHookService } from '../../services/hooks/command-hook-service';
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import GitHub from '../../../github';
class AWSTaskRunner {
public static ECS: ECS;
public static Kinesis: Kinesis;
private static readonly encodedUnderscore = `$252F`;
static async runTask(
taskDef: CloudRunnerAWSTaskDef,
environment: CloudRunnerEnvironmentVariable[],
commands: string,
): Promise<{ output: string; shouldCleanup: boolean }> {
const cluster = taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ECSCluster')?.PhysicalResourceId || '';
const taskDefinition =
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'TaskDefinition')?.PhysicalResourceId || '';
const SubnetOne =
taskDef.baseResources?.find((x) => x.LogicalResourceId === 'PublicSubnetOne')?.PhysicalResourceId || '';
const SubnetTwo =
taskDef.baseResources?.find((x) => x.LogicalResourceId === 'PublicSubnetTwo')?.PhysicalResourceId || '';
const ContainerSecurityGroup =
taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ContainerSecurityGroup')?.PhysicalResourceId || '';
const streamName =
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'KinesisStream')?.PhysicalResourceId || '';
const runParameters = {
cluster,
taskDefinition,
platformVersion: '1.4.0',
overrides: {
containerOverrides: [
{
name: taskDef.taskDefStackName,
environment,
command: ['-c', CommandHookService.ApplyHooksToCommands(commands, CloudRunner.buildParameters)],
},
],
},
launchType: 'FARGATE',
networkConfiguration: {
awsvpcConfiguration: {
subnets: [SubnetOne, SubnetTwo],
assignPublicIp: 'ENABLED',
securityGroups: [ContainerSecurityGroup],
},
},
};
if (JSON.stringify(runParameters.overrides.containerOverrides).length > 8192) {
CloudRunnerLogger.log(JSON.stringify(runParameters.overrides.containerOverrides, undefined, 4));
throw new Error(`Container Overrides length must be at most 8192`);
}
const task = await AWSTaskRunner.ECS.send(new RunTaskCommand(runParameters as RunTaskCommandInput));
const taskArn = task.tasks?.[0].taskArn || '';
CloudRunnerLogger.log('Cloud runner job is starting');
await AWSTaskRunner.waitUntilTaskRunning(taskArn, cluster);
CloudRunnerLogger.log(
`Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Async:${
CloudRunnerOptions.asyncCloudRunner
}`,
);
if (CloudRunnerOptions.asyncCloudRunner) {
const shouldCleanup: boolean = false;
const output: string = '';
CloudRunnerLogger.log(`Watch Cloud Runner To End: false`);
return { output, shouldCleanup };
}
CloudRunnerLogger.log(`Streaming...`);
const { output, shouldCleanup } = await this.streamLogsUntilTaskStops(cluster, taskArn, streamName);
let exitCode;
let containerState;
let taskData;
while (exitCode === undefined) {
await new Promise((resolve) => resolve(10000));
taskData = await AWSTaskRunner.describeTasks(cluster, taskArn);
containerState = taskData.containers?.[0];
exitCode = containerState?.exitCode;
}
CloudRunnerLogger.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`);
if (exitCode === undefined) {
CloudRunnerLogger.logWarning(`Undefined exitcode for container`);
}
const wasSuccessful = exitCode === 0;
if (wasSuccessful) {
CloudRunnerLogger.log(`Cloud runner job has finished successfully`);
return { output, shouldCleanup };
}
if (taskData?.stoppedReason === 'Essential container in task exited' && exitCode === 1) {
throw new Error('Container exited with code 1');
}
throw new Error(`Task failed`);
}
private static async waitUntilTaskRunning(taskArn: string, cluster: string) {
try {
await waitUntilTasksRunning(
{
client: AWSTaskRunner.ECS,
maxWaitTime: 120,
},
{ tasks: [taskArn], cluster },
);
} catch (error_) {
const error = error_ as Error;
await new Promise((resolve) => setTimeout(resolve, 3000));
CloudRunnerLogger.log(
`Cloud runner job has ended ${
(await AWSTaskRunner.describeTasks(cluster, taskArn)).containers?.[0].lastStatus
}`,
);
core.setFailed(error);
core.error(error);
}
}
static async describeTasks(clusterName: string, taskArn: string) {
const tasks = await AWSTaskRunner.ECS.send(new DescribeTasksCommand({ cluster: clusterName, tasks: [taskArn] }));
if (tasks.tasks?.[0]) {
return tasks.tasks?.[0];
} else {
throw new Error('No task found');
}
}
static async streamLogsUntilTaskStops(clusterName: string, taskArn: string, kinesisStreamName: string) {
await new Promise((resolve) => setTimeout(resolve, 3000));
CloudRunnerLogger.log(`Streaming...`);
const stream = await AWSTaskRunner.getLogStream(kinesisStreamName);
let iterator = await AWSTaskRunner.getLogIterator(stream);
const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsStackName}${AWSTaskRunner.encodedUnderscore}${CloudRunner.buildParameters.awsStackName}-${CloudRunner.buildParameters.buildGuid}`;
CloudRunnerLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`);
await GitHub.updateGitHubCheck(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`, ``);
let shouldReadLogs = true;
let shouldCleanup = true;
let timestamp: number = 0;
let output = '';
while (shouldReadLogs) {
await new Promise((resolve) => setTimeout(resolve, 1500));
const taskData = await AWSTaskRunner.describeTasks(clusterName, taskArn);
({ timestamp, shouldReadLogs } = AWSTaskRunner.checkStreamingShouldContinue(taskData, timestamp, shouldReadLogs));
({ iterator, shouldReadLogs, output, shouldCleanup } = await AWSTaskRunner.handleLogStreamIteration(
iterator,
shouldReadLogs,
output,
shouldCleanup,
));
}
return { output, shouldCleanup };
}
private static async handleLogStreamIteration(
iterator: string,
shouldReadLogs: boolean,
output: string,
shouldCleanup: boolean,
) {
const records = await AWSTaskRunner.Kinesis.send(new GetRecordsCommand({ ShardIterator: iterator }));
iterator = records.NextShardIterator || '';
({ shouldReadLogs, output, shouldCleanup } = AWSTaskRunner.logRecords(
records,
iterator,
shouldReadLogs,
output,
shouldCleanup,
));
return { iterator, shouldReadLogs, output, shouldCleanup };
}
private static checkStreamingShouldContinue(taskData: Task, timestamp: number, shouldReadLogs: boolean) {
if (taskData?.lastStatus === 'UNKNOWN') {
CloudRunnerLogger.log('## Cloud runner job unknwon');
}
if (taskData?.lastStatus !== 'RUNNING') {
if (timestamp === 0) {
CloudRunnerLogger.log('## Cloud runner job stopped, streaming end of logs');
timestamp = Date.now();
}
if (timestamp !== 0 && Date.now() - timestamp > 30000) {
CloudRunnerLogger.log('## Cloud runner status is not RUNNING for 30 seconds, last query for logs');
shouldReadLogs = false;
}
CloudRunnerLogger.log(`## Status of job: ${taskData.lastStatus}`);
}
return { timestamp, shouldReadLogs };
}
private static logRecords(
records: GetRecordsCommandOutput,
iterator: string,
shouldReadLogs: boolean,
output: string,
shouldCleanup: boolean,
) {
if ((records.Records ?? []).length > 0 && iterator) {
for (const record of records.Records ?? []) {
const json = JSON.parse(
zlib.gunzipSync(Buffer.from(record.Data as unknown as string, 'base64')).toString('utf8'),
);
if (json.messageType === 'DATA_MESSAGE') {
for (const logEvent of json.logEvents) {
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
logEvent.message,
shouldReadLogs,
shouldCleanup,
output,
));
}
}
}
}
return { shouldReadLogs, output, shouldCleanup };
}
private static async getLogStream(kinesisStreamName: string) {
return await AWSTaskRunner.Kinesis.send(new DescribeStreamCommand({ StreamName: kinesisStreamName }));
}
private static async getLogIterator(stream: DescribeStreamCommandOutput) {
return (
(
await AWSTaskRunner.Kinesis.send(
new GetShardIteratorCommand({
ShardIteratorType: 'TRIM_HORIZON',
StreamName: stream.StreamDescription?.StreamName ?? '',
ShardId: stream.StreamDescription?.Shards?.[0]?.ShardId || '',
}),
)
).ShardIterator || ''
);
}
}
export default AWSTaskRunner;
@@ -0,0 +1,397 @@
export class BaseStackFormation {
public static readonly baseStackDecription = `Game-CI base stack`;
public static readonly formation: string = `AWSTemplateFormatVersion: '2010-09-09'
Description: ${BaseStackFormation.baseStackDecription}
Parameters:
EnvironmentName:
Type: String
Default: development
Description: 'Your deployment environment: DEV, QA , PROD'
Version:
Type: String
Description: 'hash of template'
# ContainerPort:
# Type: Number
# Default: 80
# Description: What port number the application inside the docker container is binding to
Mappings:
# Hard values for the subnet masks. These masks define
# the range of internal IP addresses that can be assigned.
# The VPC can have all IP's from 10.0.0.0 to 10.0.255.255
# There are four subnets which cover the ranges:
#
# 10.0.0.0 - 10.0.0.255
# 10.0.1.0 - 10.0.1.255
# 10.0.2.0 - 10.0.2.255
# 10.0.3.0 - 10.0.3.255
SubnetConfig:
VPC:
CIDR: '10.0.0.0/16'
PublicOne:
CIDR: '10.0.0.0/24'
PublicTwo:
CIDR: '10.0.1.0/24'
Resources:
# VPC in which containers will be networked.
# It has two public subnets, and two private subnets.
# We distribute the subnets across the first two available subnets
# for the region, for high availability.
VPC:
Type: AWS::EC2::VPC
Properties:
EnableDnsSupport: true
EnableDnsHostnames: true
CidrBlock: !FindInMap ['SubnetConfig', 'VPC', 'CIDR']
MainBucket:
Type: "AWS::S3::Bucket"
Properties:
BucketName: !Ref EnvironmentName
EFSServerSecurityGroup:
Type: AWS::EC2::SecurityGroup
Properties:
GroupName: 'efs-server-endpoints'
GroupDescription: Which client ip addrs are allowed to access EFS server
VpcId: !Ref 'VPC'
SecurityGroupIngress:
- IpProtocol: tcp
FromPort: 2049
ToPort: 2049
SourceSecurityGroupId: !Ref ContainerSecurityGroup
#CidrIp: !FindInMap ['SubnetConfig', 'VPC', 'CIDR']
# A security group for the containers we will run in Fargate.
# Rules are added to this security group based on what ingress you
# add for the cluster.
ContainerSecurityGroup:
Type: AWS::EC2::SecurityGroup
Properties:
GroupName: 'task security group'
GroupDescription: Access to the Fargate containers
VpcId: !Ref 'VPC'
# SecurityGroupIngress:
# - IpProtocol: tcp
# FromPort: !Ref ContainerPort
# ToPort: !Ref ContainerPort
# CidrIp: 0.0.0.0/0
SecurityGroupEgress:
- IpProtocol: -1
FromPort: 2049
ToPort: 2049
CidrIp: '0.0.0.0/0'
# Two public subnets, where containers can have public IP addresses
PublicSubnetOne:
Type: AWS::EC2::Subnet
Properties:
AvailabilityZone: !Select
- 0
- Fn::GetAZs: !Ref 'AWS::Region'
VpcId: !Ref 'VPC'
CidrBlock: !FindInMap ['SubnetConfig', 'PublicOne', 'CIDR']
# MapPublicIpOnLaunch: true
PublicSubnetTwo:
Type: AWS::EC2::Subnet
Properties:
AvailabilityZone: !Select
- 1
- Fn::GetAZs: !Ref 'AWS::Region'
VpcId: !Ref 'VPC'
CidrBlock: !FindInMap ['SubnetConfig', 'PublicTwo', 'CIDR']
# MapPublicIpOnLaunch: true
# Setup networking resources for the public subnets. Containers
# in the public subnets have public IP addresses and the routing table
# sends network traffic via the internet gateway.
InternetGateway:
Type: AWS::EC2::InternetGateway
GatewayAttachement:
Type: AWS::EC2::VPCGatewayAttachment
Properties:
VpcId: !Ref 'VPC'
InternetGatewayId: !Ref 'InternetGateway'
# Attaching a Internet Gateway to route table makes it public.
PublicRouteTable:
Type: AWS::EC2::RouteTable
Properties:
VpcId: !Ref 'VPC'
PublicRoute:
Type: AWS::EC2::Route
DependsOn: GatewayAttachement
Properties:
RouteTableId: !Ref 'PublicRouteTable'
DestinationCidrBlock: '0.0.0.0/0'
GatewayId: !Ref 'InternetGateway'
# Attaching a public route table makes a subnet public.
PublicSubnetOneRouteTableAssociation:
Type: AWS::EC2::SubnetRouteTableAssociation
Properties:
SubnetId: !Ref PublicSubnetOne
RouteTableId: !Ref PublicRouteTable
PublicSubnetTwoRouteTableAssociation:
Type: AWS::EC2::SubnetRouteTableAssociation
Properties:
SubnetId: !Ref PublicSubnetTwo
RouteTableId: !Ref PublicRouteTable
# ECS Resources
ECSCluster:
Type: AWS::ECS::Cluster
# A role used to allow AWS Autoscaling to inspect stats and adjust scaleable targets
# on your AWS account
AutoscalingRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Statement:
- Effect: Allow
Principal:
Service: [application-autoscaling.amazonaws.com]
Action: ['sts:AssumeRole']
Path: /
Policies:
- PolicyName: service-autoscaling
PolicyDocument:
Statement:
- Effect: Allow
Action:
- 'application-autoscaling:*'
- 'cloudwatch:DescribeAlarms'
- 'cloudwatch:PutMetricAlarm'
- 'ecs:DescribeServices'
- 'ecs:UpdateService'
Resource: '*'
# This is an IAM role which authorizes ECS to manage resources on your
# account on your behalf, such as updating your load balancer with the
# details of where your containers are, so that traffic can reach your
# containers.
ECSRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Statement:
- Effect: Allow
Principal:
Service: [ecs.amazonaws.com]
Action: ['sts:AssumeRole']
Path: /
Policies:
- PolicyName: ecs-service
PolicyDocument:
Statement:
- Effect: Allow
Action:
# Rules which allow ECS to attach network interfaces to instances
# on your behalf in order for awsvpc networking mode to work right
- 'ec2:AttachNetworkInterface'
- 'ec2:CreateNetworkInterface'
- 'ec2:CreateNetworkInterfacePermission'
- 'ec2:DeleteNetworkInterface'
- 'ec2:DeleteNetworkInterfacePermission'
- 'ec2:Describe*'
- 'ec2:DetachNetworkInterface'
# Rules which allow ECS to update load balancers on your behalf
# with the information sabout how to send traffic to your containers
- 'elasticloadbalancing:DeregisterInstancesFromLoadBalancer'
- 'elasticloadbalancing:DeregisterTargets'
- 'elasticloadbalancing:Describe*'
- 'elasticloadbalancing:RegisterInstancesWithLoadBalancer'
- 'elasticloadbalancing:RegisterTargets'
Resource: '*'
# This is a role which is used by the ECS tasks themselves.
ECSTaskExecutionRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Statement:
- Effect: Allow
Principal:
Service: [ecs-tasks.amazonaws.com]
Action: ['sts:AssumeRole']
Path: /
Policies:
- PolicyName: AmazonECSTaskExecutionRolePolicy
PolicyDocument:
Statement:
- Effect: Allow
Action:
# Allow the use of secret manager
- 'secretsmanager:GetSecretValue'
- 'kms:Decrypt'
# Allow the ECS Tasks to download images from ECR
- 'ecr:GetAuthorizationToken'
- 'ecr:BatchCheckLayerAvailability'
- 'ecr:GetDownloadUrlForLayer'
- 'ecr:BatchGetImage'
# Allow the ECS tasks to upload logs to CloudWatch
- 'logs:CreateLogStream'
- 'logs:PutLogEvents'
Resource: '*'
DeleteCFNLambdaExecutionRole:
Type: 'AWS::IAM::Role'
Properties:
AssumeRolePolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: 'Allow'
Principal:
Service: ['lambda.amazonaws.com']
Action: 'sts:AssumeRole'
Path: '/'
Policies:
- PolicyName: DeleteCFNLambdaExecutionRole
PolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: 'Allow'
Action:
- 'logs:CreateLogGroup'
- 'logs:CreateLogStream'
- 'logs:PutLogEvents'
Resource: 'arn:aws:logs:*:*:*'
- Effect: 'Allow'
Action:
- 'cloudformation:DeleteStack'
- 'kinesis:DeleteStream'
- 'secretsmanager:DeleteSecret'
- 'kinesis:DescribeStreamSummary'
- 'logs:DeleteLogGroup'
- 'logs:DeleteSubscriptionFilter'
- 'ecs:DeregisterTaskDefinition'
- 'lambda:DeleteFunction'
- 'lambda:InvokeFunction'
- 'events:RemoveTargets'
- 'events:DeleteRule'
- 'lambda:RemovePermission'
Resource: '*'
### cloud watch to kinesis role
CloudWatchIAMRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Statement:
- Effect: Allow
Principal:
Service: [logs.amazonaws.com]
Action: ['sts:AssumeRole']
Path: /
Policies:
- PolicyName: service-autoscaling
PolicyDocument:
Statement:
- Effect: Allow
Action:
- 'kinesis:PutRecord'
Resource: '*'
#####################EFS#####################
EfsFileStorage:
Type: 'AWS::EFS::FileSystem'
Properties:
BackupPolicy:
Status: ENABLED
PerformanceMode: maxIO
Encrypted: false
FileSystemPolicy:
Version: '2012-10-17'
Statement:
- Effect: 'Allow'
Action:
- 'elasticfilesystem:ClientMount'
- 'elasticfilesystem:ClientWrite'
- 'elasticfilesystem:ClientRootAccess'
Principal:
AWS: '*'
MountTargetResource1:
Type: AWS::EFS::MountTarget
Properties:
FileSystemId: !Ref EfsFileStorage
SubnetId: !Ref PublicSubnetOne
SecurityGroups:
- !Ref EFSServerSecurityGroup
MountTargetResource2:
Type: AWS::EFS::MountTarget
Properties:
FileSystemId: !Ref EfsFileStorage
SubnetId: !Ref PublicSubnetTwo
SecurityGroups:
- !Ref EFSServerSecurityGroup
Outputs:
EfsFileStorageId:
Description: 'The connection endpoint for the database.'
Value: !Ref EfsFileStorage
Export:
Name: !Sub ${'${EnvironmentName}'}:EfsFileStorageId
ClusterName:
Description: The name of the ECS cluster
Value: !Ref 'ECSCluster'
Export:
Name: !Sub${' ${EnvironmentName}'}:ClusterName
AutoscalingRole:
Description: The ARN of the role used for autoscaling
Value: !GetAtt 'AutoscalingRole.Arn'
Export:
Name: !Sub ${'${EnvironmentName}'}:AutoscalingRole
ECSRole:
Description: The ARN of the ECS role
Value: !GetAtt 'ECSRole.Arn'
Export:
Name: !Sub ${'${EnvironmentName}'}:ECSRole
ECSTaskExecutionRole:
Description: The ARN of the ECS role tsk execution role
Value: !GetAtt 'ECSTaskExecutionRole.Arn'
Export:
Name: !Sub ${'${EnvironmentName}'}:ECSTaskExecutionRole
DeleteCFNLambdaExecutionRole:
Description: Lambda execution role for cleaning up cloud formations
Value: !GetAtt 'DeleteCFNLambdaExecutionRole.Arn'
Export:
Name: !Sub ${'${EnvironmentName}'}:DeleteCFNLambdaExecutionRole
CloudWatchIAMRole:
Description: The ARN of the CloudWatch role for subscription filter
Value: !GetAtt 'CloudWatchIAMRole.Arn'
Export:
Name: !Sub ${'${EnvironmentName}'}:CloudWatchIAMRole
VpcId:
Description: The ID of the VPC that this stack is deployed in
Value: !Ref 'VPC'
Export:
Name: !Sub ${'${EnvironmentName}'}:VpcId
PublicSubnetOne:
Description: Public subnet one
Value: !Ref 'PublicSubnetOne'
Export:
Name: !Sub ${'${EnvironmentName}'}:PublicSubnetOne
PublicSubnetTwo:
Description: Public subnet two
Value: !Ref 'PublicSubnetTwo'
Export:
Name: !Sub ${'${EnvironmentName}'}:PublicSubnetTwo
ContainerSecurityGroup:
Description: A security group used to allow Fargate containers to receive traffic
Value: !Ref 'ContainerSecurityGroup'
Export:
Name: !Sub ${'${EnvironmentName}'}:ContainerSecurityGroup
`;
}
@@ -0,0 +1,146 @@
export class CleanupCronFormation {
public static readonly formation: string = `AWSTemplateFormatVersion: '2010-09-09'
Description: Schedule automatic deletion of CloudFormation stacks
Metadata:
AWS::CloudFormation::Interface:
ParameterGroups:
- Label:
default: Input configuration
Parameters:
- StackName
- TTL
ParameterLabels:
StackName:
default: Stack name
TTL:
default: Time-to-live
Parameters:
EnvironmentName:
Type: String
Default: development
Description: 'Your deployment environment: DEV, QA , PROD'
BUILDGUID:
Type: String
Default: ''
StackName:
Type: String
Description: Stack name that will be deleted.
DeleteStackName:
Type: String
Description: Stack name that will be deleted.
TTL:
Type: Number
Description: Time-to-live in minutes for the stack.
Resources:
DeleteCFNLambda:
Type: "AWS::Lambda::Function"
Properties:
FunctionName: !Join [ "", [ 'DeleteCFNLambda', !Ref BUILDGUID ] ]
Code:
ZipFile: |
import boto3
import os
import json
stack_name = os.environ['stackName']
delete_stack_name = os.environ['deleteStackName']
def delete_cfn(stack_name):
try:
cfn = boto3.resource('cloudformation')
stack = cfn.Stack(stack_name)
stack.delete()
return "SUCCESS"
except:
return "ERROR"
def handler(event, context):
print("Received event:")
print(json.dumps(event))
result = delete_cfn(stack_name)
delete_cfn(delete_stack_name)
return result
Environment:
Variables:
stackName: !Ref 'StackName'
deleteStackName: !Ref 'DeleteStackName'
Handler: "index.handler"
Runtime: "python3.9"
Timeout: "5"
Role:
'Fn::ImportValue': !Sub '\${EnvironmentName}:DeleteCFNLambdaExecutionRole'
DeleteStackEventRule:
DependsOn:
- DeleteCFNLambda
- GenerateCronExpression
Type: "AWS::Events::Rule"
Properties:
Name: !Join [ "", [ 'DeleteStackEventRule', !Ref BUILDGUID ] ]
Description: Delete stack event
ScheduleExpression: !GetAtt GenerateCronExpression.cron_exp
State: "ENABLED"
Targets:
-
Arn: !GetAtt DeleteCFNLambda.Arn
Id: 'DeleteCFNLambda'
PermissionForDeleteCFNLambda:
Type: "AWS::Lambda::Permission"
DependsOn:
- DeleteStackEventRule
Properties:
FunctionName: !Join [ "", [ 'DeleteCFNLambda', !Ref BUILDGUID ] ]
Action: "lambda:InvokeFunction"
Principal: "events.amazonaws.com"
SourceArn: !GetAtt DeleteStackEventRule.Arn
GenerateCronExpLambda:
Type: "AWS::Lambda::Function"
Properties:
FunctionName: !Join [ "", [ 'GenerateCronExpressionLambda', !Ref BUILDGUID ] ]
Code:
ZipFile: |
from datetime import datetime, timedelta
import os
import logging
import json
import cfnresponse
def deletion_time(ttl):
delete_at_time = datetime.now() + timedelta(minutes=int(ttl))
hh = delete_at_time.hour
mm = delete_at_time.minute
yyyy = delete_at_time.year
month = delete_at_time.month
dd = delete_at_time.day
# minutes hours day month day-of-week year
cron_exp = "cron({} {} {} {} ? {})".format(mm, hh, dd, month, yyyy)
return cron_exp
def handler(event, context):
print('Received event: %s' % json.dumps(event))
status = cfnresponse.SUCCESS
try:
if event['RequestType'] == 'Delete':
cfnresponse.send(event, context, status, {})
else:
ttl = event['ResourceProperties']['ttl']
responseData = {}
responseData['cron_exp'] = deletion_time(ttl)
cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData)
except Exception as e:
logging.error('Exception: %s' % e, exc_info=True)
status = cfnresponse.FAILED
cfnresponse.send(event, context, status, {}, None)
Handler: "index.handler"
Runtime: "python3.9"
Timeout: "5"
Role:
'Fn::ImportValue': !Sub '\${EnvironmentName}:DeleteCFNLambdaExecutionRole'
GenerateCronExpression:
Type: "Custom::GenerateCronExpression"
Version: "1.0"
Properties:
Name: !Join [ "", [ 'GenerateCronExpression', !Ref BUILDGUID ] ]
ServiceToken: !GetAtt GenerateCronExpLambda.Arn
ttl: !Ref 'TTL'
`;
}
@@ -0,0 +1,169 @@
import CloudRunner from '../../../cloud-runner';
export class TaskDefinitionFormation {
public static readonly description: string = `Game CI Cloud Runner Task Stack`;
public static get formation(): string {
return `AWSTemplateFormatVersion: 2010-09-09
Description: ${TaskDefinitionFormation.description}
Parameters:
EnvironmentName:
Type: String
Default: development
Description: 'Your deployment environment: DEV, QA , PROD'
ServiceName:
Type: String
Default: example
Description: A name for the service
LogGroupName:
Type: String
Default: example
Description: Name to use for the log group created for this task
ImageUrl:
Type: String
Default: nginx
Description: >-
The url of a docker image that contains the application process that will
handle the traffic for this service
ContainerPort:
Type: Number
Default: 80
Description: What port number the application inside the docker container is binding to
ContainerCpu:
Default: ${CloudRunner.buildParameters.containerCpu}
Type: Number
Description: How much CPU to give the container. 1024 is 1 CPU
ContainerMemory:
Default: ${CloudRunner.buildParameters.containerMemory}
Type: Number
Description: How much memory in megabytes to give the container
BUILDGUID:
Type: String
Default: ''
Command:
Type: String
Default: 'ls'
EntryPoint:
Type: String
Default: '/bin/sh'
WorkingDirectory:
Type: String
Default: '/efsdata/'
Role:
Type: String
Default: ''
Description: >-
(Optional) An IAM role to give the service's containers if the code within
needs to access other AWS resources like S3 buckets, DynamoDB tables, etc
EFSMountDirectory:
Type: String
Default: '/efsdata'
# template secrets p1 - input
Mappings:
SubnetConfig:
VPC:
CIDR: 10.0.0.0/16
PublicOne:
CIDR: 10.0.0.0/24
PublicTwo:
CIDR: 10.0.1.0/24
Conditions:
HasCustomRole: !Not
- !Equals
- Ref: Role
- ''
Resources:
LogGroup:
Type: 'AWS::Logs::LogGroup'
Properties:
LogGroupName: !Ref LogGroupName
Metadata:
'AWS::CloudFormation::Designer':
id: aece53ae-b82d-4267-bc16-ed964b05db27
# template resources secrets
# template resources logstream
TaskDefinition:
Type: 'AWS::ECS::TaskDefinition'
Properties:
Family: !Ref ServiceName
Cpu: !Ref ContainerCpu
Memory: !Ref ContainerMemory
NetworkMode: awsvpc
Volumes:
- Name: efs-data
EFSVolumeConfiguration:
FilesystemId:
'Fn::ImportValue': !Sub '${'${EnvironmentName}'}:EfsFileStorageId'
TransitEncryption: DISABLED
RequiresCompatibilities:
- FARGATE
ExecutionRoleArn:
'Fn::ImportValue': !Sub '${'${EnvironmentName}'}:ECSTaskExecutionRole'
TaskRoleArn:
'Fn::If':
- HasCustomRole
- !Ref Role
- !Ref 'AWS::NoValue'
ContainerDefinitions:
- Name: !Ref ServiceName
Cpu: !Ref ContainerCpu
Memory: !Ref ContainerMemory
Image: !Ref ImageUrl
EntryPoint:
Fn::Split:
- ','
- !Ref EntryPoint
Command:
Fn::Split:
- ','
- !Ref Command
WorkingDirectory: !Ref WorkingDirectory
Environment:
- Name: ALLOW_EMPTY_PASSWORD
Value: 'yes'
# template - env vars
MountPoints:
- SourceVolume: efs-data
ContainerPath: !Ref EFSMountDirectory
ReadOnly: false
Secrets:
# template secrets p3 - container def
LogConfiguration:
LogDriver: awslogs
Options:
awslogs-group: !Ref LogGroupName
awslogs-region: !Ref 'AWS::Region'
awslogs-stream-prefix: !Ref ServiceName
DependsOn:
- LogGroup
`;
}
public static streamLogs = `
SubscriptionFilter:
Type: 'AWS::Logs::SubscriptionFilter'
Properties:
FilterPattern: ''
RoleArn:
'Fn::ImportValue': !Sub '${'${EnvironmentName}'}:CloudWatchIAMRole'
LogGroupName: !Ref LogGroupName
DestinationArn:
'Fn::GetAtt':
- KinesisStream
- Arn
Metadata:
'AWS::CloudFormation::Designer':
id: 7f809e91-9e5d-4678-98c1-c5085956c480
DependsOn:
- LogGroup
- KinesisStream
KinesisStream:
Type: 'AWS::Kinesis::Stream'
Properties:
Name: !Ref ServiceName
ShardCount: 1
Metadata:
'AWS::CloudFormation::Designer':
id: c6f18447-b879-4696-8873-f981b2cedd2b
`;
}
@@ -0,0 +1,9 @@
import { StackResource } from '@aws-sdk/client-cloudformation';
class CloudRunnerAWSTaskDef {
public taskDefStackName!: string;
public taskDefCloudFormation!: string;
public taskDefResources: StackResource[] | undefined;
public baseResources: StackResource[] | undefined;
}
export default CloudRunnerAWSTaskDef;
@@ -0,0 +1,163 @@
import { CloudFormation, DeleteStackCommand, waitUntilStackDeleteComplete } from '@aws-sdk/client-cloudformation';
import { ECS as ECSClient } from '@aws-sdk/client-ecs';
import { Kinesis } from '@aws-sdk/client-kinesis';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
import AwsTaskRunner from './aws-task-runner';
import { ProviderInterface } from '../provider-interface';
import BuildParameters from '../../../build-parameters';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { AWSJobStack as AwsJobStack } from './aws-job-stack';
import { AWSBaseStack as AwsBaseStack } from './aws-base-stack';
import { Input } from '../../..';
import { GarbageCollectionService } from './services/garbage-collection-service';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
import { TaskService } from './services/task-service';
import CloudRunnerOptions from '../../options/cloud-runner-options';
class AWSBuildEnvironment implements ProviderInterface {
private baseStackName: string;
constructor(buildParameters: BuildParameters) {
this.baseStackName = buildParameters.awsStackName;
}
async listResources(): Promise<ProviderResource[]> {
await TaskService.getCloudFormationJobStacks();
await TaskService.getLogGroups();
await TaskService.getTasks();
return [];
}
listWorkflow(): Promise<ProviderWorkflow[]> {
throw new Error('Method not implemented.');
}
async watchWorkflow(): Promise<string> {
return await TaskService.watch();
}
async listOtherResources(): Promise<string> {
await TaskService.getLogGroups();
return '';
}
async garbageCollect(
filter: string,
previewOnly: boolean,
// eslint-disable-next-line no-unused-vars
olderThan: Number,
// eslint-disable-next-line no-unused-vars
fullCache: boolean,
// eslint-disable-next-line no-unused-vars
baseDependencies: boolean,
): Promise<string> {
await GarbageCollectionService.cleanup(!previewOnly);
return ``;
}
async cleanupWorkflow(
// eslint-disable-next-line no-unused-vars
buildParameters: BuildParameters,
// eslint-disable-next-line no-unused-vars
branchName: string,
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {}
async setupWorkflow(
// eslint-disable-next-line no-unused-vars
buildGuid: string,
// eslint-disable-next-line no-unused-vars
buildParameters: BuildParameters,
// eslint-disable-next-line no-unused-vars
branchName: string,
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {
process.env.AWS_REGION = Input.region;
const CF = new CloudFormation({ region: Input.region });
await new AwsBaseStack(this.baseStackName).setupBaseStack(CF);
}
async runTaskInWorkflow(
buildGuid: string,
image: string,
commands: string,
mountdir: string,
workingdir: string,
environment: CloudRunnerEnvironmentVariable[],
secrets: CloudRunnerSecret[],
): Promise<string> {
process.env.AWS_REGION = Input.region;
const ECS = new ECSClient({ region: Input.region });
const CF = new CloudFormation({ region: Input.region });
AwsTaskRunner.ECS = ECS;
AwsTaskRunner.Kinesis = new Kinesis({ region: Input.region });
CloudRunnerLogger.log(`AWS Region: ${CF.config.region}`);
const entrypoint = ['/bin/sh'];
const startTimeMs = Date.now();
const taskDef = await new AwsJobStack(this.baseStackName).setupCloudFormations(
CF,
buildGuid,
image,
entrypoint,
commands,
mountdir,
workingdir,
secrets,
);
let postRunTaskTimeMs;
try {
const postSetupStacksTimeMs = Date.now();
CloudRunnerLogger.log(`Setup job time: ${Math.floor((postSetupStacksTimeMs - startTimeMs) / 1000)}s`);
const { output, shouldCleanup } = await AwsTaskRunner.runTask(taskDef, environment, commands);
postRunTaskTimeMs = Date.now();
CloudRunnerLogger.log(`Run job time: ${Math.floor((postRunTaskTimeMs - postSetupStacksTimeMs) / 1000)}s`);
if (shouldCleanup) {
await this.cleanupResources(CF, taskDef);
}
const postCleanupTimeMs = Date.now();
if (postRunTaskTimeMs !== undefined)
CloudRunnerLogger.log(`Cleanup job time: ${Math.floor((postCleanupTimeMs - postRunTaskTimeMs) / 1000)}s`);
return output;
} catch (error) {
CloudRunnerLogger.log(`error running task ${error}`);
await this.cleanupResources(CF, taskDef);
throw error;
}
}
async cleanupResources(CF: CloudFormation, taskDef: CloudRunnerAWSTaskDef) {
CloudRunnerLogger.log('Cleanup starting');
await CF.send(new DeleteStackCommand({ StackName: taskDef.taskDefStackName }));
if (CloudRunnerOptions.useCleanupCron) {
await CF.send(new DeleteStackCommand({ StackName: `${taskDef.taskDefStackName}-cleanup` }));
}
await waitUntilStackDeleteComplete(
{
client: CF,
maxWaitTime: 200,
},
{
StackName: taskDef.taskDefStackName,
},
);
await waitUntilStackDeleteComplete(
{
client: CF,
maxWaitTime: 200,
},
{
StackName: `${taskDef.taskDefStackName}-cleanup`,
},
);
CloudRunnerLogger.log(`Deleted Stack: ${taskDef.taskDefStackName}`);
CloudRunnerLogger.log('Cleanup complete');
}
}
export default AWSBuildEnvironment;
@@ -0,0 +1,80 @@
import {
CloudFormation,
DeleteStackCommand,
DeleteStackCommandInput,
DescribeStackResourcesCommand,
} from '@aws-sdk/client-cloudformation';
import { CloudWatchLogs, DeleteLogGroupCommand } from '@aws-sdk/client-cloudwatch-logs';
import { ECS, StopTaskCommand } from '@aws-sdk/client-ecs';
import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
import { TaskService } from './task-service';
export class GarbageCollectionService {
static isOlderThan1day(date: Date) {
const ageDate = new Date(date.getTime() - Date.now());
return ageDate.getDay() > 0;
}
public static async cleanup(deleteResources = false, OneDayOlderOnly: boolean = false) {
process.env.AWS_REGION = Input.region;
const CF = new CloudFormation({ region: Input.region });
const ecs = new ECS({ region: Input.region });
const cwl = new CloudWatchLogs({ region: Input.region });
const taskDefinitionsInUse = new Array();
const tasks = await TaskService.getTasks();
for (const task of tasks) {
const { taskElement, element } = task;
taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.createdAt!))) {
CloudRunnerLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
await ecs.send(new StopTaskCommand({ task: taskElement.taskArn || '', cluster: element }));
}
}
const jobStacks = await TaskService.getCloudFormationJobStacks();
for (const element of jobStacks) {
if (
(await CF.send(new DescribeStackResourcesCommand({ StackName: element.StackName }))).StackResources?.some(
(x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId),
)
) {
CloudRunnerLogger.log(`Skipping ${element.StackName} - active task was running not deleting`);
return;
}
if (
deleteResources &&
(!OneDayOlderOnly || (element.CreationTime && GarbageCollectionService.isOlderThan1day(element.CreationTime)))
) {
if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') {
CloudRunnerLogger.log(`Skipping ${element.StackName} ignore list`);
return;
}
CloudRunnerLogger.log(`Deleting ${element.StackName}`);
const deleteStackInput: DeleteStackCommandInput = { StackName: element.StackName };
await CF.send(new DeleteStackCommand(deleteStackInput));
}
}
const logGroups = await TaskService.getLogGroups();
for (const element of logGroups) {
if (
deleteResources &&
(!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.creationTime!)))
) {
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
await cwl.send(new DeleteLogGroupCommand({ logGroupName: element.logGroupName || '' }));
}
}
const locks = await TaskService.getLocks();
for (const element of locks) {
CloudRunnerLogger.log(`Lock: ${element.Key}`);
}
}
}
@@ -0,0 +1,209 @@
import {
CloudFormation,
DescribeStackResourcesCommand,
DescribeStacksCommand,
ListStacksCommand,
StackSummary,
} from '@aws-sdk/client-cloudformation';
import {
CloudWatchLogs,
DescribeLogGroupsCommand,
DescribeLogGroupsCommandInput,
LogGroup,
} from '@aws-sdk/client-cloudwatch-logs';
import {
DescribeTasksCommand,
DescribeTasksCommandInput,
ECS,
ListClustersCommand,
ListTasksCommand,
ListTasksCommandInput,
Task,
} from '@aws-sdk/client-ecs';
import { ListObjectsCommand, ListObjectsCommandInput, S3 } from '@aws-sdk/client-s3';
import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
import AwsTaskRunner from '../aws-task-runner';
import CloudRunner from '../../../cloud-runner';
export class TaskService {
static async watch() {
// eslint-disable-next-line no-unused-vars
const { output, shouldCleanup } = await AwsTaskRunner.streamLogsUntilTaskStops(
process.env.cluster || ``,
process.env.taskArn || ``,
process.env.streamName || ``,
);
return output;
}
public static async getCloudFormationJobStacks() {
const result: StackSummary[] = [];
CloudRunnerLogger.log(``);
CloudRunnerLogger.log(`List Cloud Formation Stacks`);
process.env.AWS_REGION = Input.region;
const CF = new CloudFormation({ region: Input.region });
const stacks =
(await CF.send(new ListStacksCommand({}))).StackSummaries?.filter(
(_x) =>
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription,
) || [];
CloudRunnerLogger.log(``);
CloudRunnerLogger.log(`Cloud Formation Stacks ${stacks.length}`);
for (const element of stacks) {
if (!element.CreationTime) {
CloudRunnerLogger.log(`${element.StackName} due to undefined CreationTime`);
}
const ageDate: Date = new Date(Date.now() - (element.CreationTime?.getTime() ?? 0));
CloudRunnerLogger.log(
`Task Stack ${element.StackName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
result.push(element);
}
const baseStacks =
(await CF.send(new ListStacksCommand({}))).StackSummaries?.filter(
(_x) =>
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription,
) || [];
CloudRunnerLogger.log(``);
CloudRunnerLogger.log(`Base Stacks ${baseStacks.length}`);
for (const element of baseStacks) {
if (!element.CreationTime) {
CloudRunnerLogger.log(`${element.StackName} due to undefined CreationTime`);
}
const ageDate: Date = new Date(Date.now() - (element.CreationTime?.getTime() ?? 0));
CloudRunnerLogger.log(
`Task Stack ${element.StackName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
result.push(element);
}
CloudRunnerLogger.log(``);
return result;
}
public static async getTasks() {
const result: { taskElement: Task; element: string }[] = [];
CloudRunnerLogger.log(``);
CloudRunnerLogger.log(`List Tasks`);
process.env.AWS_REGION = Input.region;
const ecs = new ECS({ region: Input.region });
const clusters = (await ecs.send(new ListClustersCommand({}))).clusterArns || [];
CloudRunnerLogger.log(`Task Clusters ${clusters.length}`);
for (const element of clusters) {
const input: ListTasksCommandInput = {
cluster: element,
};
const list = (await ecs.send(new ListTasksCommand(input))).taskArns || [];
if (list.length > 0) {
const describeInput: DescribeTasksCommandInput = { tasks: list, cluster: element };
const describeList = (await ecs.send(new DescribeTasksCommand(describeInput))).tasks || [];
if (describeList.length === 0) {
CloudRunnerLogger.log(`No Tasks`);
continue;
}
CloudRunnerLogger.log(`Tasks ${describeList.length}`);
for (const taskElement of describeList) {
if (taskElement === undefined) {
continue;
}
taskElement.overrides = {};
taskElement.attachments = [];
if (taskElement.createdAt === undefined) {
CloudRunnerLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
continue;
}
result.push({ taskElement, element });
}
}
}
CloudRunnerLogger.log(``);
return result;
}
public static async awsDescribeJob(job: string) {
process.env.AWS_REGION = Input.region;
const CF = new CloudFormation({ region: Input.region });
try {
const stack =
(await CF.send(new ListStacksCommand({}))).StackSummaries?.find((_x) => _x.StackName === job) || undefined;
const stackInfo = (await CF.send(new DescribeStackResourcesCommand({ StackName: job }))) || undefined;
const stackInfo2 = (await CF.send(new DescribeStacksCommand({ StackName: job }))) || undefined;
if (stack === undefined) {
throw new Error('stack not defined');
}
if (!stack.CreationTime) {
CloudRunnerLogger.log(`${stack.StackName} due to undefined CreationTime`);
}
const ageDate: Date = new Date(Date.now() - (stack.CreationTime?.getTime() ?? 0));
const message = `
Task Stack ${stack.StackName}
Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}
${JSON.stringify(stack, undefined, 4)}
${JSON.stringify(stackInfo, undefined, 4)}
${JSON.stringify(stackInfo2, undefined, 4)}
`;
CloudRunnerLogger.log(message);
return message;
} catch (error) {
CloudRunnerLogger.error(
`Failed to describe job ${job}: ${error instanceof Error ? error.message : String(error)}`,
);
throw error;
}
}
public static async getLogGroups() {
const result: Array<LogGroup> = [];
process.env.AWS_REGION = Input.region;
const ecs = new CloudWatchLogs();
let logStreamInput: DescribeLogGroupsCommandInput = {
/* logGroupNamePrefix: 'game-ci' */
};
let logGroupsDescribe = await ecs.send(new DescribeLogGroupsCommand(logStreamInput));
const logGroups = logGroupsDescribe.logGroups || [];
while (logGroupsDescribe.nextToken) {
logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
logGroupsDescribe = await ecs.send(new DescribeLogGroupsCommand(logStreamInput));
logGroups.push(...(logGroupsDescribe?.logGroups || []));
}
CloudRunnerLogger.log(`Log Groups ${logGroups.length}`);
for (const element of logGroups) {
if (element.creationTime === undefined) {
CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
continue;
}
const ageDate: Date = new Date(Date.now() - element.creationTime);
CloudRunnerLogger.log(
`Task Stack ${element.logGroupName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
result.push(element);
}
return result;
}
public static async getLocks() {
process.env.AWS_REGION = Input.region;
const s3 = new S3({ region: Input.region });
const listRequest: ListObjectsCommandInput = {
Bucket: CloudRunner.buildParameters.awsStackName,
};
const results = await s3.send(new ListObjectsCommand(listRequest));
return results.Contents || [];
}
}
@@ -0,0 +1,163 @@
import BuildParameters from '../../../build-parameters';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import Docker from '../../../docker';
import { Action } from '../../..';
import { writeFileSync } from 'node:fs';
import CloudRunner from '../../cloud-runner';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
import * as fs from 'node:fs';
import { CommandHookService } from '../../services/hooks/command-hook-service';
import { StringKeyValuePair } from '../../../shared-types';
class LocalDockerCloudRunner implements ProviderInterface {
public buildParameters!: BuildParameters;
listResources(): Promise<ProviderResource[]> {
return new Promise((resolve) => resolve([]));
}
listWorkflow(): Promise<ProviderWorkflow[]> {
throw new Error('Method not implemented.');
}
watchWorkflow(): Promise<string> {
throw new Error('Method not implemented.');
}
garbageCollect(
// eslint-disable-next-line no-unused-vars
filter: string,
// eslint-disable-next-line no-unused-vars
previewOnly: boolean,
// eslint-disable-next-line no-unused-vars
olderThan: Number,
// eslint-disable-next-line no-unused-vars
fullCache: boolean,
// eslint-disable-next-line no-unused-vars
baseDependencies: boolean,
): Promise<string> {
return new Promise((result) => result(``));
}
async cleanupWorkflow(
buildParameters: BuildParameters,
// eslint-disable-next-line no-unused-vars
branchName: string,
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {
const { workspace } = Action;
if (
fs.existsSync(
`${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}`,
)
) {
await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache/cache/build/`);
await CloudRunnerSystem.Run(
`rm -r ${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}`,
);
}
}
setupWorkflow(
buildGuid: string,
buildParameters: BuildParameters,
// eslint-disable-next-line no-unused-vars
branchName: string,
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {
this.buildParameters = buildParameters;
}
public async runTaskInWorkflow(
buildGuid: string,
image: string,
commands: string,
mountdir: string,
workingdir: string,
environment: CloudRunnerEnvironmentVariable[],
secrets: CloudRunnerSecret[],
): Promise<string> {
CloudRunnerLogger.log(buildGuid);
CloudRunnerLogger.log(commands);
const { workspace, actionFolder } = Action;
const content: StringKeyValuePair[] = [];
for (const x of secrets) {
content.push({ name: x.EnvironmentVariable, value: x.ParameterValue });
}
for (const x of environment) {
content.push({ name: x.name, value: x.value });
}
// if (this.buildParameters?.cloudRunnerIntegrationTests) {
// core.info(JSON.stringify(content, undefined, 4));
// core.info(JSON.stringify(secrets, undefined, 4));
// core.info(JSON.stringify(environment, undefined, 4));
// }
// eslint-disable-next-line unicorn/no-for-loop
for (let index = 0; index < content.length; index++) {
if (content[index] === undefined) {
delete content[index];
}
}
let myOutput = '';
const sharedFolder = `/data/`;
// core.info(JSON.stringify({ workspace, actionFolder, ...this.buildParameters, ...content }, undefined, 4));
const entrypointFilePath = `start.sh`;
const fileContents = `#!/bin/bash
set -e
mkdir -p /github/workspace/cloud-runner-cache
mkdir -p /data/cache
cp -a /github/workspace/cloud-runner-cache/. ${sharedFolder}
${CommandHookService.ApplyHooksToCommands(commands, this.buildParameters)}
cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
`;
writeFileSync(`${workspace}/${entrypointFilePath}`, fileContents, {
flag: 'w',
});
if (CloudRunner.buildParameters.cloudRunnerDebug) {
CloudRunnerLogger.log(`Running local-docker: \n ${fileContents}`);
}
if (fs.existsSync(`${workspace}/cloud-runner-cache`)) {
await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache && du -sh ${workspace}/cloud-runner-cache`);
}
const exitCode = await Docker.run(
image,
{ workspace, actionFolder, ...this.buildParameters },
false,
`chmod +x /github/workspace/${entrypointFilePath} && /github/workspace/${entrypointFilePath}`,
content,
{
listeners: {
stdout: (data: Buffer) => {
myOutput += data.toString();
},
stderr: (data: Buffer) => {
myOutput += `[LOCAL-DOCKER-ERROR]${data.toString()}`;
},
},
},
true,
);
// Docker doesn't exit on fail now so adding this to ensure behavior is unchanged
// TODO: Is there a helpful way to consume the exit code or is it best to except
if (exitCode !== 0) {
throw new Error(`Build failed with exit code ${exitCode}`);
}
return myOutput;
}
}
export default LocalDockerCloudRunner;
@@ -0,0 +1,337 @@
import * as k8s from '@kubernetes/client-node';
import { BuildParameters } from '../../..';
import * as core from '@actions/core';
import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import KubernetesStorage from './kubernetes-storage';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import KubernetesTaskRunner from './kubernetes-task-runner';
import KubernetesSecret from './kubernetes-secret';
import KubernetesJobSpecFactory from './kubernetes-job-spec-factory';
import KubernetesServiceAccount from './kubernetes-service-account';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { CoreV1Api } from '@kubernetes/client-node';
import CloudRunner from '../../cloud-runner';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
import { KubernetesRole } from './kubernetes-role';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
class Kubernetes implements ProviderInterface {
public static Instance: Kubernetes;
public kubeConfig!: k8s.KubeConfig;
public kubeClient!: k8s.CoreV1Api;
public kubeClientApps!: k8s.AppsV1Api;
public kubeClientBatch!: k8s.BatchV1Api;
public rbacAuthorizationV1Api!: k8s.RbacAuthorizationV1Api;
public buildGuid: string = '';
public buildParameters!: BuildParameters;
public pvcName: string = '';
public secretName: string = '';
public jobName: string = '';
public namespace!: string;
public podName: string = '';
public containerName: string = '';
public cleanupCronJobName: string = '';
public serviceAccountName: string = '';
public ip: string = '';
// eslint-disable-next-line no-unused-vars
constructor(buildParameters: BuildParameters) {
Kubernetes.Instance = this;
this.kubeConfig = new k8s.KubeConfig();
this.kubeConfig.loadFromDefault();
this.kubeClient = this.kubeConfig.makeApiClient(k8s.CoreV1Api);
this.kubeClientApps = this.kubeConfig.makeApiClient(k8s.AppsV1Api);
this.kubeClientBatch = this.kubeConfig.makeApiClient(k8s.BatchV1Api);
this.rbacAuthorizationV1Api = this.kubeConfig.makeApiClient(k8s.RbacAuthorizationV1Api);
this.namespace = 'default';
CloudRunnerLogger.log('Loaded default Kubernetes configuration for this environment');
}
async PushLogUpdate(logs: string) {
// push logs to nginx file server via 'LOG_SERVICE_IP' env var
const ip = process.env[`LOG_SERVICE_IP`];
if (ip === undefined) {
RemoteClientLogger.logWarning(`LOG_SERVICE_IP not set, skipping log push`);
return;
}
const url = `http://${ip}/api/log`;
RemoteClientLogger.log(`Pushing logs to ${url}`);
// logs to base64
logs = Buffer.from(logs).toString('base64');
const response = await CloudRunnerSystem.Run(`curl -X POST -d "${logs}" ${url}`, false, true);
RemoteClientLogger.log(`Pushed logs to ${url} ${response}`);
}
async listResources(): Promise<ProviderResource[]> {
const pods = await this.kubeClient.listNamespacedPod(this.namespace);
const serviceAccounts = await this.kubeClient.listNamespacedServiceAccount(this.namespace);
const secrets = await this.kubeClient.listNamespacedSecret(this.namespace);
const jobs = await this.kubeClientBatch.listNamespacedJob(this.namespace);
return [
...pods.body.items.map((x) => {
return { Name: x.metadata?.name || `` };
}),
...serviceAccounts.body.items.map((x) => {
return { Name: x.metadata?.name || `` };
}),
...secrets.body.items.map((x) => {
return { Name: x.metadata?.name || `` };
}),
...jobs.body.items.map((x) => {
return { Name: x.metadata?.name || `` };
}),
];
}
listWorkflow(): Promise<ProviderWorkflow[]> {
throw new Error('Method not implemented.');
}
watchWorkflow(): Promise<string> {
throw new Error('Method not implemented.');
}
garbageCollect(
// eslint-disable-next-line no-unused-vars
filter: string,
// eslint-disable-next-line no-unused-vars
previewOnly: boolean,
// eslint-disable-next-line no-unused-vars
olderThan: Number,
// eslint-disable-next-line no-unused-vars
fullCache: boolean,
// eslint-disable-next-line no-unused-vars
baseDependencies: boolean,
): Promise<string> {
return new Promise((result) => result(``));
}
public async setupWorkflow(
buildGuid: string,
buildParameters: BuildParameters,
// eslint-disable-next-line no-unused-vars
branchName: string,
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {
try {
this.buildParameters = buildParameters;
this.cleanupCronJobName = `unity-builder-cronjob-${buildParameters.buildGuid}`;
this.serviceAccountName = `service-account-${buildParameters.buildGuid}`;
await KubernetesServiceAccount.createServiceAccount(this.serviceAccountName, this.namespace, this.kubeClient);
} catch (error) {
throw error;
}
}
async runTaskInWorkflow(
buildGuid: string,
image: string,
commands: string,
mountdir: string,
workingdir: string,
environment: CloudRunnerEnvironmentVariable[],
secrets: CloudRunnerSecret[],
): Promise<string> {
try {
CloudRunnerLogger.log('Cloud Runner K8s workflow!');
// Setup
const id =
BuildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(this.buildParameters)
? CloudRunner.lockedWorkspace
: this.buildParameters.buildGuid;
this.pvcName = `unity-builder-pvc-${id}`;
await KubernetesStorage.createPersistentVolumeClaim(
this.buildParameters,
this.pvcName,
this.kubeClient,
this.namespace,
);
this.buildGuid = buildGuid;
this.secretName = `build-credentials-${this.buildGuid}`;
this.jobName = `unity-builder-job-${this.buildGuid}`;
this.containerName = `main`;
await KubernetesSecret.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
let output = '';
try {
CloudRunnerLogger.log('Job does not exist');
await this.createJob(commands, image, mountdir, workingdir, environment, secrets);
CloudRunnerLogger.log('Watching pod until running');
await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
CloudRunnerLogger.log('Pod is running');
output += await KubernetesTaskRunner.runTask(
this.kubeConfig,
this.kubeClient,
this.jobName,
this.podName,
this.containerName,
this.namespace,
);
} catch (error: any) {
CloudRunnerLogger.log(`error running k8s workflow ${error}`);
await new Promise((resolve) => setTimeout(resolve, 3000));
CloudRunnerLogger.log(
JSON.stringify(
(await this.kubeClient.listNamespacedEvent(this.namespace)).body.items
.map((x) => {
return {
message: x.message || ``,
name: x.metadata.name || ``,
reason: x.reason || ``,
};
})
.filter((x) => x.name.includes(this.podName)),
undefined,
4,
),
);
await this.cleanupTaskResources();
throw error;
}
await this.cleanupTaskResources();
return output;
} catch (error) {
CloudRunnerLogger.log('Running job failed');
core.error(JSON.stringify(error, undefined, 4));
// await this.cleanupTaskResources();
throw error;
}
}
private async createJob(
commands: string,
image: string,
mountdir: string,
workingdir: string,
environment: CloudRunnerEnvironmentVariable[],
secrets: CloudRunnerSecret[],
) {
await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
const find = await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace);
this.setPodNameAndContainerName(find);
}
private async doesJobExist(name: string) {
const jobs = await this.kubeClientBatch.listNamespacedJob(this.namespace);
return jobs.body.items.some((x) => x.metadata?.name === name);
}
private async doesFailedJobExist() {
const podStatus = await this.kubeClient.readNamespacedPodStatus(this.podName, this.namespace);
return podStatus.body.status?.phase === `Failed`;
}
private async createNamespacedJob(
commands: string,
image: string,
mountdir: string,
workingdir: string,
environment: CloudRunnerEnvironmentVariable[],
secrets: CloudRunnerSecret[],
) {
for (let index = 0; index < 3; index++) {
try {
const jobSpec = KubernetesJobSpecFactory.getJobSpec(
commands,
image,
mountdir,
workingdir,
environment,
secrets,
this.buildGuid,
this.buildParameters,
this.secretName,
this.pvcName,
this.jobName,
k8s,
this.containerName,
this.ip,
);
await new Promise((promise) => setTimeout(promise, 15000));
// await KubernetesRole.createRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api);
const result = await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
CloudRunnerLogger.log(`Build job created`);
await new Promise((promise) => setTimeout(promise, 5000));
CloudRunnerLogger.log('Job created');
return result.body.metadata?.name;
} catch (error) {
CloudRunnerLogger.log(`Error occured creating job: ${error}`);
throw error;
}
}
}
setPodNameAndContainerName(pod: k8s.V1Pod) {
this.podName = pod.metadata?.name || '';
this.containerName = pod.status?.containerStatuses?.[0].name || this.containerName;
}
async cleanupTaskResources() {
CloudRunnerLogger.log('cleaning up');
try {
await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
await this.kubeClient.deleteNamespacedPod(this.podName, this.namespace);
await KubernetesRole.deleteRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api);
} catch (error: any) {
CloudRunnerLogger.log(`Failed to cleanup`);
if (error.response.body.reason !== `NotFound`) {
CloudRunnerLogger.log(`Wasn't a not found error: ${error.response.body.reason}`);
throw error;
}
}
try {
await this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace);
} catch (error: any) {
CloudRunnerLogger.log(`Failed to cleanup secret`);
CloudRunnerLogger.log(error.response.body.reason);
}
CloudRunnerLogger.log('cleaned up Secret, Job and Pod');
CloudRunnerLogger.log('cleaning up finished');
}
async cleanupWorkflow(
buildParameters: BuildParameters,
// eslint-disable-next-line no-unused-vars
branchName: string,
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {
if (BuildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
return;
}
CloudRunnerLogger.log(`deleting PVC`);
try {
await this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace);
await this.kubeClient.deleteNamespacedServiceAccount(this.serviceAccountName, this.namespace);
CloudRunnerLogger.log('cleaned up PVC and Service Account');
} catch (error: any) {
CloudRunnerLogger.log(`Cleanup failed ${JSON.stringify(error, undefined, 4)}`);
throw error;
}
}
static async findPodFromJob(kubeClient: CoreV1Api, jobName: string, namespace: string) {
const namespacedPods = await kubeClient.listNamespacedPod(namespace);
const pod = namespacedPods.body.items.find((x) => x.metadata?.labels?.['job-name'] === jobName);
if (pod === undefined) {
throw new Error("pod with job-name label doesn't exist");
}
return pod;
}
}
export default Kubernetes;
@@ -0,0 +1,127 @@
import { V1EnvVar, V1EnvVarSource, V1SecretKeySelector } from '@kubernetes/client-node';
import BuildParameters from '../../../build-parameters';
import { CommandHookService } from '../../services/hooks/command-hook-service';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import CloudRunner from '../../cloud-runner';
class KubernetesJobSpecFactory {
static getJobSpec(
command: string,
image: string,
mountdir: string,
workingDirectory: string,
environment: CloudRunnerEnvironmentVariable[],
secrets: CloudRunnerSecret[],
buildGuid: string,
buildParameters: BuildParameters,
secretName: string,
pvcName: string,
jobName: string,
k8s: any,
containerName: string,
ip: string = '',
) {
const job = new k8s.V1Job();
job.apiVersion = 'batch/v1';
job.kind = 'Job';
job.metadata = {
name: jobName,
labels: {
app: 'unity-builder',
buildGuid,
},
};
job.spec = {
ttlSecondsAfterFinished: 9999,
backoffLimit: 0,
template: {
spec: {
volumes: [
{
name: 'build-mount',
persistentVolumeClaim: {
claimName: pvcName,
},
},
],
containers: [
{
ttlSecondsAfterFinished: 9999,
name: containerName,
image,
command: ['/bin/sh'],
args: [
'-c',
`${CommandHookService.ApplyHooksToCommands(`${command}\nsleep 2m`, CloudRunner.buildParameters)}`,
],
workingDir: `${workingDirectory}`,
resources: {
requests: {
memory: `${Number.parseInt(buildParameters.containerMemory) / 1024}G` || '750M',
cpu: Number.parseInt(buildParameters.containerCpu) / 1024 || '1',
},
},
env: [
...environment.map((x) => {
const environmentVariable = new V1EnvVar();
environmentVariable.name = x.name;
environmentVariable.value = x.value;
return environmentVariable;
}),
...secrets.map((x) => {
const secret = new V1EnvVarSource();
secret.secretKeyRef = new V1SecretKeySelector();
secret.secretKeyRef.key = x.ParameterKey;
secret.secretKeyRef.name = secretName;
const environmentVariable = new V1EnvVar();
environmentVariable.name = x.EnvironmentVariable;
environmentVariable.valueFrom = secret;
return environmentVariable;
}),
{ name: 'LOG_SERVICE_IP', value: ip },
],
volumeMounts: [
{
name: 'build-mount',
mountPath: `${mountdir}`,
},
],
lifecycle: {
preStop: {
exec: {
command: [
`wait 60s;
cd /data/builder/action/steps;
chmod +x /return_license.sh;
/return_license.sh;`,
],
},
},
},
},
],
restartPolicy: 'Never',
},
},
};
if (process.env['CLOUD_RUNNER_MINIKUBE']) {
job.spec.template.spec.volumes[0] = {
name: 'build-mount',
hostPath: {
path: `/data`,
type: `Directory`,
},
};
}
job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '10Gi';
return job;
}
}
export default KubernetesJobSpecFactory;
@@ -0,0 +1,23 @@
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { CoreV1Api } from '@kubernetes/client-node';
class KubernetesPods {
public static async IsPodRunning(podName: string, namespace: string, kubeClient: CoreV1Api) {
const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.filter((x) => podName === x.metadata?.name);
const running = pods.length > 0 && (pods[0].status?.phase === `Running` || pods[0].status?.phase === `Pending`);
const phase = pods[0]?.status?.phase || 'undefined status';
CloudRunnerLogger.log(`Getting pod status: ${phase}`);
if (phase === `Failed`) {
throw new Error(`K8s pod failed`);
}
return running;
}
public static async GetPodStatus(podName: string, namespace: string, kubeClient: CoreV1Api) {
const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.find((x) => podName === x.metadata?.name);
const phase = pods?.status?.phase || 'undefined status';
return phase;
}
}
export default KubernetesPods;
@@ -0,0 +1,53 @@
import { RbacAuthorizationV1Api } from '@kubernetes/client-node';
class KubernetesRole {
static async createRole(serviceAccountName: string, namespace: string, rbac: RbacAuthorizationV1Api) {
// create admin kubernetes role and role binding
const roleBinding = {
apiVersion: 'rbac.authorization.k8s.io/v1',
kind: 'RoleBinding',
metadata: {
name: `${serviceAccountName}-admin`,
namespace,
},
subjects: [
{
kind: 'ServiceAccount',
name: serviceAccountName,
namespace,
},
],
roleRef: {
apiGroup: 'rbac.authorization.k8s.io',
kind: 'Role',
name: `${serviceAccountName}-admin`,
},
};
const role = {
apiVersion: 'rbac.authorization.k8s.io/v1',
kind: 'Role',
metadata: {
name: `${serviceAccountName}-admin`,
namespace,
},
rules: [
{
apiGroups: ['*'],
resources: ['*'],
verbs: ['*'],
},
],
};
const roleBindingResponse = await rbac.createNamespacedRoleBinding(namespace, roleBinding);
const roleResponse = await rbac.createNamespacedRole(namespace, role);
return { roleBindingResponse, roleResponse };
}
public static async deleteRole(serviceAccountName: string, namespace: string, rbac: RbacAuthorizationV1Api) {
await rbac.deleteNamespacedRoleBinding(`${serviceAccountName}-admin`, namespace);
await rbac.deleteNamespacedRole(`${serviceAccountName}-admin`, namespace);
}
}
export { KubernetesRole };
@@ -0,0 +1,45 @@
import { CoreV1Api } from '@kubernetes/client-node';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import * as k8s from '@kubernetes/client-node';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import * as base64 from 'base-64';
class KubernetesSecret {
static async createSecret(
secrets: CloudRunnerSecret[],
secretName: string,
namespace: string,
kubeClient: CoreV1Api,
) {
try {
const secret = new k8s.V1Secret();
secret.apiVersion = 'v1';
secret.kind = 'Secret';
secret.type = 'Opaque';
secret.metadata = {
name: secretName,
};
secret.data = {};
for (const buildSecret of secrets) {
secret.data[buildSecret.ParameterKey] = base64.encode(buildSecret.ParameterValue);
}
CloudRunnerLogger.log(`Creating secret: ${secretName}`);
const existingSecrets = await kubeClient.listNamespacedSecret(namespace);
const mappedSecrets = existingSecrets.body.items.map((x) => {
return x.metadata?.name || `no name`;
});
CloudRunnerLogger.log(
`ExistsAlready: ${mappedSecrets.includes(secretName)} SecretsCount: ${mappedSecrets.length}`,
);
await new Promise((promise) => setTimeout(promise, 15000));
await kubeClient.createNamespacedSecret(namespace, secret);
CloudRunnerLogger.log('Created secret');
} catch (error) {
CloudRunnerLogger.log(`Created secret failed ${error}`);
throw new Error(`Failed to create kubernetes secret`);
}
}
}
export default KubernetesSecret;
@@ -0,0 +1,18 @@
import { CoreV1Api } from '@kubernetes/client-node';
import * as k8s from '@kubernetes/client-node';
class KubernetesServiceAccount {
static async createServiceAccount(serviceAccountName: string, namespace: string, kubeClient: CoreV1Api) {
const serviceAccount = new k8s.V1ServiceAccount();
serviceAccount.apiVersion = 'v1';
serviceAccount.kind = 'ServiceAccount';
serviceAccount.metadata = {
name: serviceAccountName,
};
serviceAccount.automountServiceAccountToken = true;
return kubeClient.createNamespacedServiceAccount(namespace, serviceAccount);
}
}
export default KubernetesServiceAccount;
@@ -0,0 +1,116 @@
import { waitUntil } from 'async-wait-until';
import * as core from '@actions/core';
import * as k8s from '@kubernetes/client-node';
import BuildParameters from '../../../build-parameters';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { IncomingMessage } from 'node:http';
import GitHub from '../../../github';
class KubernetesStorage {
public static async createPersistentVolumeClaim(
buildParameters: BuildParameters,
pvcName: string,
kubeClient: k8s.CoreV1Api,
namespace: string,
) {
if (buildParameters.kubeVolume !== ``) {
CloudRunnerLogger.log(`Kube Volume was input was set ${buildParameters.kubeVolume} overriding ${pvcName}`);
pvcName = buildParameters.kubeVolume;
return;
}
const allPvc = (await kubeClient.listNamespacedPersistentVolumeClaim(namespace)).body.items;
const pvcList = allPvc.map((x) => x.metadata?.name);
CloudRunnerLogger.log(`Current PVCs in namespace ${namespace}`);
CloudRunnerLogger.log(JSON.stringify(pvcList, undefined, 4));
if (pvcList.includes(pvcName)) {
CloudRunnerLogger.log(`pvc ${pvcName} already exists`);
if (GitHub.githubInputEnabled) {
core.setOutput('volume', pvcName);
}
return;
}
CloudRunnerLogger.log(`Creating PVC ${pvcName} (does not exist)`);
const result = await KubernetesStorage.createPVC(pvcName, buildParameters, kubeClient, namespace);
await KubernetesStorage.handleResult(result, kubeClient, namespace, pvcName);
}
public static async getPVCPhase(kubeClient: k8s.CoreV1Api, name: string, namespace: string) {
try {
return (await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace)).body.status?.phase;
} catch (error) {
core.error('Failed to get PVC phase');
core.error(JSON.stringify(error, undefined, 4));
throw error;
}
}
public static async watchUntilPVCNotPending(kubeClient: k8s.CoreV1Api, name: string, namespace: string) {
try {
CloudRunnerLogger.log(`watch Until PVC Not Pending ${name} ${namespace}`);
CloudRunnerLogger.log(`${await this.getPVCPhase(kubeClient, name, namespace)}`);
await waitUntil(
async () => {
return (await this.getPVCPhase(kubeClient, name, namespace)) === 'Pending';
},
{
timeout: 750000,
intervalBetweenAttempts: 15000,
},
);
} catch (error: any) {
core.error('Failed to watch PVC');
core.error(error.toString());
core.error(
`PVC Body: ${JSON.stringify(
(await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace)).body,
undefined,
4,
)}`,
);
throw error;
}
}
private static async createPVC(
pvcName: string,
buildParameters: BuildParameters,
kubeClient: k8s.CoreV1Api,
namespace: string,
) {
const pvc = new k8s.V1PersistentVolumeClaim();
pvc.apiVersion = 'v1';
pvc.kind = 'PersistentVolumeClaim';
pvc.metadata = {
name: pvcName,
};
pvc.spec = {
accessModes: ['ReadWriteOnce'],
storageClassName: buildParameters.kubeStorageClass === '' ? 'standard' : buildParameters.kubeStorageClass,
resources: {
requests: {
storage: buildParameters.kubeVolumeSize,
},
},
};
const result = await kubeClient.createNamespacedPersistentVolumeClaim(namespace, pvc);
return result;
}
private static async handleResult(
result: { response: IncomingMessage; body: k8s.V1PersistentVolumeClaim },
kubeClient: k8s.CoreV1Api,
namespace: string,
pvcName: string,
) {
const name = result.body.metadata?.name || '';
CloudRunnerLogger.log(`PVC ${name} created`);
await this.watchUntilPVCNotPending(kubeClient, name, namespace);
CloudRunnerLogger.log(`PVC ${name} is ready and not pending`);
core.setOutput('volume', pvcName);
}
}
export default KubernetesStorage;
@@ -0,0 +1,119 @@
import { CoreV1Api, KubeConfig } from '@kubernetes/client-node';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { waitUntil } from 'async-wait-until';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
import CloudRunner from '../../cloud-runner';
import KubernetesPods from './kubernetes-pods';
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
class KubernetesTaskRunner {
static readonly maxRetry: number = 3;
static lastReceivedMessage: string = ``;
static async runTask(
kubeConfig: KubeConfig,
kubeClient: CoreV1Api,
jobName: string,
podName: string,
containerName: string,
namespace: string,
) {
let output = '';
let shouldReadLogs = true;
let shouldCleanup = true;
let retriesAfterFinish = 0;
// eslint-disable-next-line no-constant-condition
while (true) {
await new Promise((resolve) => setTimeout(resolve, 3000));
CloudRunnerLogger.log(
`Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace} ${CloudRunner.buildParameters.kubeVolumeSize}/${CloudRunner.buildParameters.containerCpu}/${CloudRunner.buildParameters.containerMemory}`,
);
let extraFlags = ``;
extraFlags += (await KubernetesPods.IsPodRunning(podName, namespace, kubeClient))
? ` -f -c ${containerName}`
: ` --previous`;
const callback = (outputChunk: string) => {
output += outputChunk;
// split output chunk and handle per line
for (const chunk of outputChunk.split(`\n`)) {
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
chunk,
shouldReadLogs,
shouldCleanup,
output,
));
}
};
try {
await CloudRunnerSystem.Run(`kubectl logs ${podName}${extraFlags}`, false, true, callback);
} catch (error: any) {
await new Promise((resolve) => setTimeout(resolve, 3000));
const continueStreaming = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient);
CloudRunnerLogger.log(`K8s logging error ${error} ${continueStreaming}`);
if (continueStreaming) {
continue;
}
if (retriesAfterFinish < KubernetesTaskRunner.maxRetry) {
retriesAfterFinish++;
continue;
}
throw error;
}
if (FollowLogStreamService.DidReceiveEndOfTransmission) {
CloudRunnerLogger.log('end of log stream');
break;
}
}
return output;
}
static async watchUntilPodRunning(kubeClient: CoreV1Api, podName: string, namespace: string) {
let waitComplete: boolean = false;
let message = ``;
CloudRunnerLogger.log(`Watching ${podName} ${namespace}`);
await waitUntil(
async () => {
const status = await kubeClient.readNamespacedPodStatus(podName, namespace);
const phase = status?.body.status?.phase;
waitComplete = phase !== 'Pending';
message = `Phase:${status.body.status?.phase} \n Reason:${
status.body.status?.conditions?.[0].reason || ''
} \n Message:${status.body.status?.conditions?.[0].message || ''}`;
// CloudRunnerLogger.log(
// JSON.stringify(
// (await kubeClient.listNamespacedEvent(namespace)).body.items
// .map((x) => {
// return {
// message: x.message || ``,
// name: x.metadata.name || ``,
// reason: x.reason || ``,
// };
// })
// .filter((x) => x.name.includes(podName)),
// undefined,
// 4,
// ),
// );
if (waitComplete || phase !== 'Pending') return true;
return false;
},
{
timeout: 2000000,
intervalBetweenAttempts: 15000,
},
);
if (!waitComplete) {
CloudRunnerLogger.log(message);
}
return waitComplete;
}
}
export default KubernetesTaskRunner;
@@ -0,0 +1,72 @@
import BuildParameters from '../../../build-parameters';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
class LocalCloudRunner implements ProviderInterface {
listResources(): Promise<ProviderResource[]> {
throw new Error('Method not implemented.');
}
listWorkflow(): Promise<ProviderWorkflow[]> {
throw new Error('Method not implemented.');
}
watchWorkflow(): Promise<string> {
throw new Error('Method not implemented.');
}
garbageCollect(
// eslint-disable-next-line no-unused-vars
filter: string,
// eslint-disable-next-line no-unused-vars
previewOnly: boolean,
// eslint-disable-next-line no-unused-vars
olderThan: Number,
// eslint-disable-next-line no-unused-vars
fullCache: boolean,
// eslint-disable-next-line no-unused-vars
baseDependencies: boolean,
): Promise<string> {
throw new Error('Method not implemented.');
}
cleanupWorkflow(
// eslint-disable-next-line no-unused-vars
buildParameters: BuildParameters,
// eslint-disable-next-line no-unused-vars
branchName: string,
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {}
public setupWorkflow(
// eslint-disable-next-line no-unused-vars
buildGuid: string,
// eslint-disable-next-line no-unused-vars
buildParameters: BuildParameters,
// eslint-disable-next-line no-unused-vars
branchName: string,
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {}
public async runTaskInWorkflow(
buildGuid: string,
image: string,
commands: string,
// eslint-disable-next-line no-unused-vars
mountdir: string,
// eslint-disable-next-line no-unused-vars
workingdir: string,
// eslint-disable-next-line no-unused-vars
environment: CloudRunnerEnvironmentVariable[],
// eslint-disable-next-line no-unused-vars
secrets: CloudRunnerSecret[],
): Promise<string> {
CloudRunnerLogger.log(image);
CloudRunnerLogger.log(buildGuid);
CloudRunnerLogger.log(commands);
return await CloudRunnerSystem.Run(commands);
}
}
export default LocalCloudRunner;
@@ -0,0 +1,57 @@
import BuildParameters from '../../build-parameters';
import CloudRunnerEnvironmentVariable from '../options/cloud-runner-environment-variable';
import CloudRunnerSecret from '../options/cloud-runner-secret';
import { ProviderResource } from './provider-resource';
import { ProviderWorkflow } from './provider-workflow';
export interface ProviderInterface {
cleanupWorkflow(
// eslint-disable-next-line no-unused-vars
buildParameters: BuildParameters,
// eslint-disable-next-line no-unused-vars
branchName: string,
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
): any;
setupWorkflow(
// eslint-disable-next-line no-unused-vars
buildGuid: string,
// eslint-disable-next-line no-unused-vars
buildParameters: BuildParameters,
// eslint-disable-next-line no-unused-vars
branchName: string,
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
): any;
runTaskInWorkflow(
// eslint-disable-next-line no-unused-vars
buildGuid: string,
// eslint-disable-next-line no-unused-vars
image: string,
// eslint-disable-next-line no-unused-vars
commands: string,
// eslint-disable-next-line no-unused-vars
mountdir: string,
// eslint-disable-next-line no-unused-vars
workingdir: string,
// eslint-disable-next-line no-unused-vars
environment: CloudRunnerEnvironmentVariable[],
// eslint-disable-next-line no-unused-vars
secrets: CloudRunnerSecret[],
): Promise<string>;
garbageCollect(
// eslint-disable-next-line no-unused-vars
filter: string,
// eslint-disable-next-line no-unused-vars
previewOnly: boolean,
// eslint-disable-next-line no-unused-vars
olderThan: Number,
// eslint-disable-next-line no-unused-vars
fullCache: boolean,
// eslint-disable-next-line no-unused-vars
baseDependencies: boolean,
): Promise<string>;
listResources(): Promise<ProviderResource[]>;
listWorkflow(): Promise<ProviderWorkflow[]>;
watchWorkflow(): Promise<string>;
}
@@ -0,0 +1,3 @@
export class ProviderResource {
public Name!: string;
}
@@ -0,0 +1,3 @@
export class ProviderWorkflow {
public Name!: string;
}
@@ -0,0 +1,67 @@
import BuildParameters from '../../../build-parameters';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
class TestCloudRunner implements ProviderInterface {
listResources(): Promise<ProviderResource[]> {
throw new Error('Method not implemented.');
}
listWorkflow(): Promise<ProviderWorkflow[]> {
throw new Error('Method not implemented.');
}
watchWorkflow(): Promise<string> {
throw new Error('Method not implemented.');
}
garbageCollect(
// eslint-disable-next-line no-unused-vars
filter: string,
// eslint-disable-next-line no-unused-vars
previewOnly: boolean,
): Promise<string> {
throw new Error('Method not implemented.');
}
cleanupWorkflow(
// eslint-disable-next-line no-unused-vars
buildParameters: BuildParameters,
// eslint-disable-next-line no-unused-vars
branchName: string,
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {}
setupWorkflow(
// eslint-disable-next-line no-unused-vars
buildGuid: string,
// eslint-disable-next-line no-unused-vars
buildParameters: BuildParameters,
// eslint-disable-next-line no-unused-vars
branchName: string,
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {}
public async runTaskInWorkflow(
commands: string,
buildGuid: string,
image: string,
// eslint-disable-next-line no-unused-vars
mountdir: string,
// eslint-disable-next-line no-unused-vars
workingdir: string,
// eslint-disable-next-line no-unused-vars
environment: CloudRunnerEnvironmentVariable[],
// eslint-disable-next-line no-unused-vars
secrets: CloudRunnerSecret[],
): Promise<string> {
CloudRunnerLogger.log(image);
CloudRunnerLogger.log(buildGuid);
CloudRunnerLogger.log(commands);
return await new Promise((result) => {
result(commands);
});
}
}
export default TestCloudRunner;
@@ -0,0 +1,181 @@
import { assert } from 'node:console';
import fs from 'node:fs';
import path from 'node:path';
import CloudRunner from '../cloud-runner';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import { CloudRunnerFolders } from '../options/cloud-runner-folders';
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
import { LfsHashing } from '../services/utility/lfs-hashing';
import { RemoteClientLogger } from './remote-client-logger';
import { Cli } from '../../cli/cli';
import { CliFunction } from '../../cli/cli-functions-repository';
// eslint-disable-next-line github/no-then
const fileExists = async (fpath: fs.PathLike) => !!(await fs.promises.stat(fpath).catch(() => false));
export class Caching {
@CliFunction(`cache-push`, `push to cache`)
static async cachePush() {
try {
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
CloudRunner.buildParameters = buildParameter;
await Caching.PushToCache(
Cli.options!['cachePushTo'],
Cli.options!['cachePushFrom'],
Cli.options!['artifactName'] || '',
);
} catch (error: any) {
CloudRunnerLogger.log(`${error}`);
}
}
@CliFunction(`cache-pull`, `pull from cache`)
static async cachePull() {
try {
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
CloudRunner.buildParameters = buildParameter;
await Caching.PullFromCache(
Cli.options!['cachePushFrom'],
Cli.options!['cachePushTo'],
Cli.options!['artifactName'] || '',
);
} catch (error: any) {
CloudRunnerLogger.log(`${error}`);
}
}
public static async PushToCache(cacheFolder: string, sourceFolder: string, cacheArtifactName: string) {
CloudRunnerLogger.log(`Pushing to cache ${sourceFolder}`);
cacheArtifactName = cacheArtifactName.replace(' ', '');
const startPath = process.cwd();
let compressionSuffix = '';
if (CloudRunner.buildParameters.useCompressionStrategy === true) {
compressionSuffix = `.lz4`;
}
CloudRunnerLogger.log(`Compression: ${CloudRunner.buildParameters.useCompressionStrategy} ${compressionSuffix}`);
try {
if (!(await fileExists(cacheFolder))) {
await CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
}
process.chdir(path.resolve(sourceFolder, '..'));
if (CloudRunner.buildParameters.cloudRunnerDebug === true) {
CloudRunnerLogger.log(
`Hashed cache folder ${await LfsHashing.hashAllFiles(sourceFolder)} ${sourceFolder} ${path.basename(
sourceFolder,
)}`,
);
}
const contents = await fs.promises.readdir(path.basename(sourceFolder));
CloudRunnerLogger.log(
`There is ${contents.length} files/dir in the source folder ${path.basename(sourceFolder)}`,
);
if (contents.length === 0) {
CloudRunnerLogger.log(
`Did not push source folder to cache because it was empty ${path.basename(sourceFolder)}`,
);
process.chdir(`${startPath}`);
return;
}
await CloudRunnerSystem.Run(
`tar -cf ${cacheArtifactName}.tar${compressionSuffix} "${path.basename(sourceFolder)}"`,
);
await CloudRunnerSystem.Run(`du ${cacheArtifactName}.tar${compressionSuffix}`);
assert(await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`), 'cache archive exists');
assert(await fileExists(path.basename(sourceFolder)), 'source folder exists');
await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar${compressionSuffix} ${cacheFolder}`);
RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`);
assert(
await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.tar${compressionSuffix}`),
'cache archive exists inside cache folder',
);
} catch (error) {
process.chdir(`${startPath}`);
throw error;
}
process.chdir(`${startPath}`);
}
public static async PullFromCache(cacheFolder: string, destinationFolder: string, cacheArtifactName: string = ``) {
CloudRunnerLogger.log(`Pulling from cache ${destinationFolder} ${CloudRunner.buildParameters.skipCache}`);
if (`${CloudRunner.buildParameters.skipCache}` === `true`) {
CloudRunnerLogger.log(`Skipping cache debugSkipCache is true`);
return;
}
cacheArtifactName = cacheArtifactName.replace(' ', '');
let compressionSuffix = '';
if (CloudRunner.buildParameters.useCompressionStrategy === true) {
compressionSuffix = `.lz4`;
}
const startPath = process.cwd();
RemoteClientLogger.log(`Caching for (lz4 ${compressionSuffix}) ${path.basename(destinationFolder)}`);
try {
if (!(await fileExists(cacheFolder))) {
await fs.promises.mkdir(cacheFolder);
}
if (!(await fileExists(destinationFolder))) {
await fs.promises.mkdir(destinationFolder);
}
const latestInBranch = await (
await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar${compressionSuffix}$ | head -1`)
)
.replace(/\n/g, ``)
.replace(`.tar${compressionSuffix}`, '');
process.chdir(cacheFolder);
const cacheSelection =
cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`))
? cacheArtifactName
: latestInBranch;
await CloudRunnerLogger.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`);
if (await fileExists(`${cacheSelection}.tar${compressionSuffix}`)) {
const resultsFolder = `results${CloudRunner.buildParameters.buildGuid}`;
await CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`);
RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar${compressionSuffix}`);
const fullResultsFolder = path.join(cacheFolder, resultsFolder);
await CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar${compressionSuffix} -C ${fullResultsFolder}`);
RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
assert(await fileExists(fullResultsFolder), `cache extraction results folder exists`);
const destinationParentFolder = path.resolve(destinationFolder, '..');
if (await fileExists(destinationFolder)) {
await fs.promises.rmdir(destinationFolder, { recursive: true });
}
await CloudRunnerSystem.Run(
`mv "${path.join(fullResultsFolder, path.basename(destinationFolder))}" "${destinationParentFolder}"`,
);
const contents = await fs.promises.readdir(
path.join(destinationParentFolder, path.basename(destinationFolder)),
);
CloudRunnerLogger.log(
`There is ${contents.length} files/dir in the cache pulled contents for ${path.basename(destinationFolder)}`,
);
} else {
RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`);
if (cacheSelection !== ``) {
RemoteClientLogger.logWarning(
`cache item ${cacheArtifactName}.tar${compressionSuffix} doesn't exist ${destinationFolder}`,
);
throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
}
}
} catch (error) {
process.chdir(startPath);
throw error;
}
process.chdir(startPath);
}
public static async handleCachePurging() {
if (process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined) {
RemoteClientLogger.log(`purging ${CloudRunnerFolders.purgeRemoteCaching}`);
fs.promises.rmdir(CloudRunnerFolders.cacheFolder, { recursive: true });
}
}
}
@@ -0,0 +1,251 @@
import fs from 'node:fs';
import CloudRunner from '../cloud-runner';
import { CloudRunnerFolders } from '../options/cloud-runner-folders';
import { Caching } from './caching';
import { LfsHashing } from '../services/utility/lfs-hashing';
import { RemoteClientLogger } from './remote-client-logger';
import path from 'node:path';
import { assert } from 'node:console';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import { CliFunction } from '../../cli/cli-functions-repository';
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
import YAML from 'yaml';
import GitHub from '../../github';
import BuildParameters from '../../build-parameters';
import { Cli } from '../../cli/cli';
import CloudRunnerOptions from '../options/cloud-runner-options';
export class RemoteClient {
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
static async setupRemoteClient() {
CloudRunnerLogger.log(`bootstrap game ci cloud runner...`);
if (!(await RemoteClient.handleRetainedWorkspace())) {
await RemoteClient.bootstrapRepository();
}
await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
await RemoteClient.runCustomHookFiles(`before-build`);
}
@CliFunction('remote-cli-log-stream', `log stream from standard input`)
public static async remoteClientLogStream() {
const logFile = Cli.options!['logFile'];
process.stdin.resume();
process.stdin.setEncoding('utf8');
let lingeringLine = '';
process.stdin.on('data', (chunk) => {
const lines = chunk.toString().split('\n');
lines[0] = lingeringLine + lines[0];
lingeringLine = lines.pop() || '';
for (const element of lines) {
if (CloudRunnerOptions.providerStrategy !== 'k8s') {
CloudRunnerLogger.log(element);
} else {
fs.appendFileSync(logFile, element);
CloudRunnerLogger.log(element);
}
}
});
process.stdin.on('end', () => {
if (CloudRunnerOptions.providerStrategy !== 'k8s') {
CloudRunnerLogger.log(lingeringLine);
} else {
fs.appendFileSync(logFile, lingeringLine);
CloudRunnerLogger.log(lingeringLine);
}
});
}
@CliFunction(`remote-cli-post-build`, `runs a cloud runner build`)
public static async remoteClientPostBuild(): Promise<string> {
RemoteClientLogger.log(`Running POST build tasks`);
await Caching.PushToCache(
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/Library`),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
`lib-${CloudRunner.buildParameters.buildGuid}`,
);
await Caching.PushToCache(
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/build`),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.projectBuildFolderAbsolute),
`build-${CloudRunner.buildParameters.buildGuid}`,
);
if (!BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)) {
await CloudRunnerSystem.Run(
`rm -r ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
);
}
await RemoteClient.runCustomHookFiles(`after-build`);
// WIP - need to give the pod permissions to create config map
await RemoteClientLogger.handleLogManagementPostJob();
return new Promise((result) => result(``));
}
static async runCustomHookFiles(hookLifecycle: string) {
RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
const gameCiCustomHooksPath = path.join(CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`);
try {
const files = fs.readdirSync(gameCiCustomHooksPath);
for (const file of files) {
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
const fileContentsObject = YAML.parse(fileContents.toString());
if (fileContentsObject.hook === hookLifecycle) {
RemoteClientLogger.log(`Active Hook File ${file} \n \n file contents: \n ${fileContents}`);
await CloudRunnerSystem.Run(fileContentsObject.commands);
}
}
} catch (error) {
RemoteClientLogger.log(JSON.stringify(error, undefined, 4));
}
}
public static async bootstrapRepository() {
await CloudRunnerSystem.Run(
`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
);
await CloudRunnerSystem.Run(
`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForCacheKeyFull)}`,
);
await RemoteClient.cloneRepoWithoutLFSFiles();
await RemoteClient.sizeOfFolder(
'repo before lfs cache pull',
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute),
);
const lfsHashes = await LfsHashing.createLFSHashFiles();
if (fs.existsSync(CloudRunnerFolders.libraryFolderAbsolute)) {
RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`);
}
await Caching.PullFromCache(
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
`${lfsHashes.lfsGuidSum}`,
);
await RemoteClient.sizeOfFolder('repo after lfs cache pull', CloudRunnerFolders.repoPathAbsolute);
await RemoteClient.pullLatestLFS();
await RemoteClient.sizeOfFolder('repo before lfs git pull', CloudRunnerFolders.repoPathAbsolute);
await Caching.PushToCache(
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
`${lfsHashes.lfsGuidSum}`,
);
await Caching.PullFromCache(
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryCacheFolderFull),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
);
await RemoteClient.sizeOfFolder('repo after library cache pull', CloudRunnerFolders.repoPathAbsolute);
await Caching.handleCachePurging();
}
private static async sizeOfFolder(message: string, folder: string) {
if (CloudRunner.buildParameters.cloudRunnerDebug) {
CloudRunnerLogger.log(`Size of ${message}`);
await CloudRunnerSystem.Run(`du -sh ${folder}`);
}
}
private static async cloneRepoWithoutLFSFiles() {
process.chdir(`${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
if (
fs.existsSync(CloudRunnerFolders.repoPathAbsolute) &&
!fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))
) {
await CloudRunnerSystem.Run(`rm -r ${CloudRunnerFolders.repoPathAbsolute}`);
CloudRunnerLogger.log(`${CloudRunnerFolders.repoPathAbsolute} repo exists, but no git folder, cleaning up`);
}
if (
BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters) &&
fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))
) {
process.chdir(CloudRunnerFolders.repoPathAbsolute);
RemoteClientLogger.log(
`${
CloudRunnerFolders.repoPathAbsolute
} repo exists - skipping clone - retained workspace mode ${BuildParameters.shouldUseRetainedWorkspaceMode(
CloudRunner.buildParameters,
)}`,
);
await CloudRunnerSystem.Run(`git fetch && git reset --hard ${CloudRunner.buildParameters.gitSha}`);
return;
}
RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
RemoteClientLogger.log(`Cloning the repository being built:`);
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f"`);
await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process --skip"`);
try {
await CloudRunnerSystem.Run(
`git clone ${CloudRunnerFolders.targetBuildRepoUrl} ${path.basename(CloudRunnerFolders.repoPathAbsolute)}`,
);
} catch (error: any) {
throw error;
}
process.chdir(CloudRunnerFolders.repoPathAbsolute);
await CloudRunnerSystem.Run(`git lfs install`);
assert(fs.existsSync(`.git`), 'git folder exists');
RemoteClientLogger.log(`${CloudRunner.buildParameters.branch}`);
if (CloudRunner.buildParameters.gitSha !== undefined) {
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.gitSha}`);
} else {
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.branch}`);
RemoteClientLogger.log(`buildParameter Git Sha is empty`);
}
assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
RemoteClientLogger.log(`Checked out ${CloudRunner.buildParameters.branch}`);
}
static async replaceLargePackageReferencesWithSharedReferences() {
CloudRunnerLogger.log(`Use Shared Pkgs ${CloudRunner.buildParameters.useLargePackages}`);
GitHub.updateGitHubCheck(`Use Shared Pkgs ${CloudRunner.buildParameters.useLargePackages}`, ``);
if (CloudRunner.buildParameters.useLargePackages) {
const filePath = path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`);
let manifest = fs.readFileSync(filePath, 'utf8');
manifest = manifest.replace(/LargeContent/g, '../../../LargeContent');
fs.writeFileSync(filePath, manifest);
CloudRunnerLogger.log(`Package Manifest \n ${manifest}`);
GitHub.updateGitHubCheck(`Package Manifest \n ${manifest}`, ``);
}
}
private static async pullLatestLFS() {
process.chdir(CloudRunnerFolders.repoPathAbsolute);
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge -- %f"`);
await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process"`);
if (!CloudRunner.buildParameters.skipLfs) {
await CloudRunnerSystem.Run(`git lfs pull`);
RemoteClientLogger.log(`pulled latest LFS files`);
assert(fs.existsSync(CloudRunnerFolders.lfsFolderAbsolute));
}
}
static async handleRetainedWorkspace() {
RemoteClientLogger.log(
`Retained Workspace: ${BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)}`,
);
if (
BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters) &&
fs.existsSync(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)) &&
fs.existsSync(CloudRunnerFolders.ToLinuxFolder(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`)))
) {
CloudRunnerLogger.log(`Retained Workspace Already Exists!`);
process.chdir(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute));
await CloudRunnerSystem.Run(`git fetch`);
await CloudRunnerSystem.Run(`git lfs pull`);
await CloudRunnerSystem.Run(`git reset --hard "${CloudRunner.buildParameters.gitSha}"`);
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.gitSha}`);
return true;
}
return false;
}
}
@@ -0,0 +1,82 @@
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import fs from 'node:fs';
import path from 'node:path';
import CloudRunner from '../cloud-runner';
import CloudRunnerOptions from '../options/cloud-runner-options';
export class RemoteClientLogger {
private static get LogFilePath() {
return path.join(`/home`, `job-log.txt`);
}
public static log(message: string) {
const finalMessage = `[Client] ${message}`;
this.appendToFile(finalMessage);
CloudRunnerLogger.log(finalMessage);
}
public static logCliError(message: string) {
CloudRunnerLogger.log(`[Client][Error] ${message}`);
}
public static logCliDiagnostic(message: string) {
CloudRunnerLogger.log(`[Client][Diagnostic] ${message}`);
}
public static logWarning(message: string) {
CloudRunnerLogger.logWarning(message);
}
public static appendToFile(message: string) {
if (CloudRunner.isCloudRunnerEnvironment) {
fs.appendFileSync(RemoteClientLogger.LogFilePath, `${message}\n`);
}
}
public static async handleLogManagementPostJob() {
if (CloudRunnerOptions.providerStrategy !== 'k8s') {
return;
}
CloudRunnerLogger.log(`Collected Logs`);
// check for log file not existing
if (!fs.existsSync(RemoteClientLogger.LogFilePath)) {
CloudRunnerLogger.log(`Log file does not exist`);
// check if CloudRunner.isCloudRunnerEnvironment is true, log
if (!CloudRunner.isCloudRunnerEnvironment) {
CloudRunnerLogger.log(`Cloud Runner is not running in a cloud environment, not collecting logs`);
}
return;
}
CloudRunnerLogger.log(`Log file exist`);
await new Promise((resolve) => setTimeout(resolve, 1));
// let hashedLogs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
//
// hashedLogs = md5(hashedLogs);
//
// for (let index = 0; index < 3; index++) {
// CloudRunnerLogger.log(`LOGHASH: ${hashedLogs}`);
// const logs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
// CloudRunnerLogger.log(`LOGS: ${Buffer.from(logs).toString('base64')}`);
// CloudRunnerLogger.log(
// `Game CI's "Cloud Runner System" will cancel the log when it has successfully received the log data to verify all logs have been received.`,
// );
//
// // wait for 15 seconds to allow the log to be sent
// await new Promise((resolve) => setTimeout(resolve, 15000));
// }
}
public static HandleLog(message: string): boolean {
if (RemoteClientLogger.value !== '') {
RemoteClientLogger.value += `\n`;
}
RemoteClientLogger.value += message;
return false;
}
static value: string = '';
}
@@ -0,0 +1,47 @@
import * as core from '@actions/core';
class CloudRunnerLogger {
private static timestamp: number;
private static globalTimestamp: number;
public static setup() {
this.timestamp = this.createTimestamp();
this.globalTimestamp = this.timestamp;
}
public static log(message: string) {
core.info(message);
}
public static logWarning(message: string) {
core.warning(message);
}
public static logLine(message: string) {
core.info(`${message}\n`);
}
public static error(message: string) {
core.error(message);
}
public static logWithTime(message: string) {
const newTimestamp = this.createTimestamp();
core.info(
`${message} (Since previous: ${this.calculateTimeDiff(
newTimestamp,
this.timestamp,
)}, Total time: ${this.calculateTimeDiff(newTimestamp, this.globalTimestamp)})`,
);
this.timestamp = newTimestamp;
}
private static calculateTimeDiff(x: number, y: number) {
return Math.floor((x - y) / 1000);
}
private static createTimestamp() {
return Date.now();
}
}
export default CloudRunnerLogger;
@@ -0,0 +1,24 @@
import BuildParameters from '../../../build-parameters';
class CloudRunnerResult {
public BuildParameters: BuildParameters;
public BuildResults: string;
public BuildSucceeded: boolean;
public BuildFinished: boolean;
public LibraryCacheUsed: boolean;
public constructor(
buildParameters: BuildParameters,
buildResults: string,
buildSucceeded: boolean,
buildFinished: boolean,
libraryCacheUsed: boolean,
) {
this.BuildParameters = buildParameters;
this.BuildResults = buildResults;
this.BuildSucceeded = buildSucceeded;
this.BuildFinished = buildFinished;
this.LibraryCacheUsed = libraryCacheUsed;
}
}
export default CloudRunnerResult;
@@ -0,0 +1,69 @@
import { exec } from 'child_process';
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
export class CloudRunnerSystem {
public static async RunAndReadLines(command: string): Promise<string[]> {
const result = await CloudRunnerSystem.Run(command, false, true);
return result
.split(`\n`)
.map((x) => x.replace(`\r`, ``))
.filter((x) => x !== ``)
.map((x) => {
const lineValues = x.split(` `);
return lineValues[lineValues.length - 1];
});
}
public static async Run(
command: string,
suppressError = false,
suppressLogs = false,
// eslint-disable-next-line no-unused-vars
outputCallback?: (output: string) => void,
) {
for (const element of command.split(`\n`)) {
if (!suppressLogs) {
RemoteClientLogger.log(element);
}
}
return await new Promise<string>((promise, throwError) => {
let output = '';
const child = exec(command, { maxBuffer: 1024 * 10000 }, (error, stdout, stderr) => {
if (!suppressError && error) {
RemoteClientLogger.log(error.toString());
throwError(error);
}
if (stderr) {
const diagnosticOutput = `${stderr.toString()}`;
if (!suppressLogs) {
RemoteClientLogger.logCliDiagnostic(diagnosticOutput);
}
output += diagnosticOutput;
}
const outputChunk = `${stdout}`;
if (outputCallback) {
outputCallback(outputChunk);
}
output += outputChunk;
});
child.on('close', (code) => {
if (!suppressLogs) {
RemoteClientLogger.log(`[${code}]`);
}
if (code !== 0 && !suppressError) {
throwError(output);
}
const outputLines = output.split(`\n`);
for (const element of outputLines) {
if (!suppressLogs) {
RemoteClientLogger.log(element);
}
}
promise(output);
});
});
}
}
@@ -0,0 +1,57 @@
import GitHub from '../../../github';
import CloudRunner from '../../cloud-runner';
import { CloudRunnerStatics } from '../../options/cloud-runner-statics';
import CloudRunnerLogger from './cloud-runner-logger';
import * as core from '@actions/core';
export class FollowLogStreamService {
static Reset() {
FollowLogStreamService.DidReceiveEndOfTransmission = false;
}
static errors = ``;
public static DidReceiveEndOfTransmission = false;
public static handleIteration(message: string, shouldReadLogs: boolean, shouldCleanup: boolean, output: string) {
if (message.includes(`---${CloudRunner.buildParameters.logId}`)) {
CloudRunnerLogger.log('End of log transmission received');
FollowLogStreamService.DidReceiveEndOfTransmission = true;
shouldReadLogs = false;
} else if (message.includes('Rebuilding Library because the asset database could not be found!')) {
GitHub.updateGitHubCheck(`Library was not found, importing new Library`, ``);
core.warning('LIBRARY NOT FOUND!');
core.setOutput('library-found', 'false');
} else if (message.includes('Build succeeded')) {
GitHub.updateGitHubCheck(`Build succeeded`, `Build succeeded`);
core.setOutput('build-result', 'success');
} else if (message.includes('Build fail')) {
GitHub.updateGitHubCheck(
`Build failed\n${FollowLogStreamService.errors}`,
`Build failed`,
`failure`,
`completed`,
);
core.setOutput('build-result', 'failed');
core.setFailed('unity build failed');
core.error('BUILD FAILED!');
} else if (message.toLowerCase().includes('error ')) {
core.error(message);
FollowLogStreamService.errors += `\n${message}`;
} else if (message.toLowerCase().includes('error: ')) {
core.error(message);
FollowLogStreamService.errors += `\n${message}`;
} else if (message.toLowerCase().includes('command failed: ')) {
FollowLogStreamService.errors += `\n${message}`;
} else if (message.toLowerCase().includes('invalid ')) {
FollowLogStreamService.errors += `\n${message}`;
} else if (message.toLowerCase().includes('incompatible ')) {
FollowLogStreamService.errors += `\n${message}`;
} else if (message.toLowerCase().includes('cannot be found')) {
FollowLogStreamService.errors += `\n${message}`;
}
if (CloudRunner.buildParameters.cloudRunnerDebug) {
output += `${message}\n`;
}
CloudRunnerLogger.log(`[${CloudRunnerStatics.logPrefix}] ${message}`);
return { shouldReadLogs, shouldCleanup, output };
}
}
@@ -0,0 +1,300 @@
import { CloudRunnerSystem } from './cloud-runner-system';
import fs from 'node:fs';
import CloudRunnerLogger from './cloud-runner-logger';
import BuildParameters from '../../../build-parameters';
import CloudRunner from '../../cloud-runner';
export class SharedWorkspaceLocking {
public static get workspaceBucketRoot() {
return `s3://${CloudRunner.buildParameters.awsStackName}/`;
}
public static get workspaceRoot() {
return `${SharedWorkspaceLocking.workspaceBucketRoot}locks/`;
}
public static async GetAllWorkspaces(buildParametersContext: BuildParameters): Promise<string[]> {
if (!(await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParametersContext))) {
return [];
}
return (
await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
)
)
.map((x) => x.replace(`/`, ``))
.filter((x) => x.endsWith(`_workspace`))
.map((x) => x.split(`_`)[1]);
}
public static async DoesCacheKeyTopLevelExist(buildParametersContext: BuildParameters) {
try {
const rootLines = await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceBucketRoot}`,
);
const lockFolderExists = rootLines.map((x) => x.replace(`/`, ``)).includes(`locks`);
if (lockFolderExists) {
const lines = await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`);
return lines.map((x) => x.replace(`/`, ``)).includes(buildParametersContext.cacheKey);
} else {
return false;
}
} catch {
return false;
}
}
public static NewWorkspaceName() {
return `${CloudRunner.retainedWorkspacePrefix}-${CloudRunner.buildParameters.buildGuid}`;
}
public static async GetAllLocksForWorkspace(
workspace: string,
buildParametersContext: BuildParameters,
): Promise<string[]> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
return [];
}
return (
await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
)
)
.map((x) => x.replace(`/`, ``))
.filter((x) => x.includes(workspace) && x.endsWith(`_lock`));
}
public static async GetLockedWorkspace(workspace: string, runId: string, buildParametersContext: BuildParameters) {
if (buildParametersContext.maxRetainedWorkspaces === 0) {
return false;
}
if (await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParametersContext)) {
const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
CloudRunnerLogger.log(`run agent ${runId} is trying to access a workspace, free: ${JSON.stringify(workspaces)}`);
for (const element of workspaces) {
const lockResult = await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext);
CloudRunnerLogger.log(
`run agent: ${runId} try lock workspace: ${element} locking attempt result: ${lockResult}`,
);
if (lockResult) {
return true;
}
}
}
if (await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext)) {
workspace = SharedWorkspaceLocking.NewWorkspaceName();
CloudRunner.lockedWorkspace = workspace;
}
const createResult = await SharedWorkspaceLocking.CreateWorkspace(workspace, buildParametersContext);
const lockResult = await SharedWorkspaceLocking.LockWorkspace(workspace, runId, buildParametersContext);
CloudRunnerLogger.log(
`run agent ${runId} didn't find a free workspace so created: ${workspace} createWorkspaceSuccess: ${createResult} Lock:${lockResult}`,
);
return createResult && lockResult;
}
public static async DoesWorkspaceExist(workspace: string, buildParametersContext: BuildParameters) {
return (
(await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext)).filter((x) => x.includes(workspace))
.length > 0
);
}
public static async HasWorkspaceLock(
workspace: string,
runId: string,
buildParametersContext: BuildParameters,
): Promise<boolean> {
const locks = (await SharedWorkspaceLocking.GetAllLocksForWorkspace(workspace, buildParametersContext))
.map((x) => {
return {
name: x,
timestamp: Number(x.split(`_`)[0]),
};
})
.sort((x) => x.timestamp);
const lockMatches = locks.filter((x) => x.name.includes(runId));
const includesRunLock = lockMatches.length > 0 && locks.indexOf(lockMatches[0]) === 0;
CloudRunnerLogger.log(
`Checking has workspace lock, runId: ${runId}, workspace: ${workspace}, success: ${includesRunLock} \n- Num of locks created by Run Agent: ${
lockMatches.length
} Num of Locks: ${locks.length}, Time ordered index for Run Agent: ${locks.indexOf(lockMatches[0])} \n \n`,
);
return includesRunLock;
}
public static async GetFreeWorkspaces(buildParametersContext: BuildParameters): Promise<string[]> {
const result: string[] = [];
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
for (const element of workspaces) {
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext);
const isBelowMax = await SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParametersContext);
CloudRunnerLogger.log(`workspace ${element} locked:${isLocked} below max:${isBelowMax}`);
if (!isLocked && isBelowMax) {
result.push(element);
}
}
return result;
}
public static async IsWorkspaceBelowMax(
workspace: string,
buildParametersContext: BuildParameters,
): Promise<boolean> {
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
if (workspace === ``) {
return (
workspaces.length < buildParametersContext.maxRetainedWorkspaces ||
buildParametersContext.maxRetainedWorkspaces === 0
);
}
const ordered: any[] = [];
for (const ws of workspaces) {
ordered.push({
name: ws,
timestamp: await SharedWorkspaceLocking.GetWorkspaceTimestamp(ws, buildParametersContext),
});
}
ordered.sort((x) => x.timestamp);
const matches = ordered.filter((x) => x.name.includes(workspace));
const isWorkspaceBelowMax =
matches.length > 0 &&
(ordered.indexOf(matches[0]) < buildParametersContext.maxRetainedWorkspaces ||
buildParametersContext.maxRetainedWorkspaces === 0);
return isWorkspaceBelowMax;
}
public static async GetWorkspaceTimestamp(
workspace: string,
buildParametersContext: BuildParameters,
): Promise<Number> {
if (workspace.split(`_`).length > 0) {
return Number(workspace.split(`_`)[1]);
}
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
throw new Error("Workspace doesn't exist, can't call get all locks");
}
return (
await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
)
)
.map((x) => x.replace(`/`, ``))
.filter((x) => x.includes(workspace) && x.endsWith(`_workspace`))
.map((x) => Number(x))[0];
}
public static async IsWorkspaceLocked(workspace: string, buildParametersContext: BuildParameters): Promise<boolean> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
throw new Error(`workspace doesn't exist ${workspace}`);
}
const files = await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
);
const lockFilesExist =
files.filter((x) => {
return x.includes(workspace) && x.endsWith(`_lock`);
}).length > 0;
return lockFilesExist;
}
public static async CreateWorkspace(workspace: string, buildParametersContext: BuildParameters): Promise<boolean> {
if (await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext)) {
throw new Error(`${workspace} already exists`);
}
const timestamp = Date.now();
const file = `${timestamp}_${workspace}_workspace`;
fs.writeFileSync(file, '');
await CloudRunnerSystem.Run(
`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
false,
true,
);
fs.rmSync(file);
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
CloudRunnerLogger.log(`All workspaces ${workspaces}`);
if (!(await SharedWorkspaceLocking.IsWorkspaceBelowMax(workspace, buildParametersContext))) {
CloudRunnerLogger.log(`Workspace is above max ${workspaces} ${buildParametersContext.maxRetainedWorkspaces}`);
await SharedWorkspaceLocking.CleanupWorkspace(workspace, buildParametersContext);
return false;
}
return true;
}
public static async LockWorkspace(
workspace: string,
runId: string,
buildParametersContext: BuildParameters,
): Promise<boolean> {
const existingWorkspace = workspace.endsWith(`_workspace`);
const ending = existingWorkspace ? workspace : `${workspace}_workspace`;
const file = `${Date.now()}_${runId}_${ending}_lock`;
fs.writeFileSync(file, '');
await CloudRunnerSystem.Run(
`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
false,
true,
);
fs.rmSync(file);
const hasLock = await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
if (hasLock) {
CloudRunner.lockedWorkspace = workspace;
} else {
await CloudRunnerSystem.Run(
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
false,
true,
);
}
return hasLock;
}
public static async ReleaseWorkspace(
workspace: string,
runId: string,
buildParametersContext: BuildParameters,
): Promise<boolean> {
const files = await SharedWorkspaceLocking.GetAllLocksForWorkspace(workspace, buildParametersContext);
const file = files.find((x) => x.includes(workspace) && x.endsWith(`_lock`) && x.includes(runId));
CloudRunnerLogger.log(`All Locks ${files} ${workspace} ${runId}`);
CloudRunnerLogger.log(`Deleting lock ${workspace}/${file}`);
CloudRunnerLogger.log(`rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`);
await CloudRunnerSystem.Run(
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
false,
true,
);
return !(await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext));
}
public static async CleanupWorkspace(workspace: string, buildParametersContext: BuildParameters) {
await CloudRunnerSystem.Run(
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey} --exclude "*" --include "*_${workspace}_*"`,
false,
true,
);
}
public static async ReadLines(command: string): Promise<string[]> {
return CloudRunnerSystem.RunAndReadLines(command);
}
}
export default SharedWorkspaceLocking;
@@ -0,0 +1,175 @@
import BuildParameters from '../../../build-parameters';
import Input from '../../../input';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerOptionsReader from '../../options/cloud-runner-options-reader';
import CloudRunnerQueryOverride from '../../options/cloud-runner-query-override';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import { CommandHookService } from '../hooks/command-hook-service';
export class TaskParameterSerializer {
static readonly blockedParameterNames: Set<string> = new Set([
'0',
'length',
'prototype',
'',
'unityVersion',
'CACHE_UNITY_INSTALLATION_ON_MAC',
'RUNNER_TEMP_PATH',
'NAME',
'CUSTOM_JOB',
]);
public static createCloudRunnerEnvironmentVariables(
buildParameters: BuildParameters,
): CloudRunnerEnvironmentVariable[] {
const result: CloudRunnerEnvironmentVariable[] = this.uniqBy(
[
...[
{ name: 'BUILD_TARGET', value: buildParameters.targetPlatform },
{ name: 'UNITY_VERSION', value: buildParameters.editorVersion },
{ name: 'GITHUB_TOKEN', value: process.env.GITHUB_TOKEN },
],
...TaskParameterSerializer.serializeFromObject(buildParameters),
...TaskParameterSerializer.serializeInput(),
...TaskParameterSerializer.serializeCloudRunnerOptions(),
...CommandHookService.getSecrets(CommandHookService.getHooks(buildParameters.commandHooks)),
]
.filter(
(x) =>
!TaskParameterSerializer.blockedParameterNames.has(x.name) &&
x.value !== '' &&
x.value !== undefined &&
x.value !== `undefined`,
)
.map((x) => {
x.name = `${TaskParameterSerializer.ToEnvVarFormat(x.name)}`;
x.value = `${x.value}`;
return x;
}),
(item: CloudRunnerEnvironmentVariable) => item.name,
);
return result;
}
// eslint-disable-next-line no-unused-vars
static uniqBy(a: CloudRunnerEnvironmentVariable[], key: (parameters: CloudRunnerEnvironmentVariable) => string) {
const seen: { [key: string]: boolean } = {};
return a.filter(function (item) {
const k = key(item);
return seen.hasOwnProperty(k) ? false : (seen[k] = true);
});
}
public static readBuildParameterFromEnvironment(): BuildParameters {
const buildParameters = new BuildParameters();
const keys = [
...new Set(
Object.getOwnPropertyNames(process.env)
.filter((x) => !this.blockedParameterNames.has(x) && x.startsWith(''))
.map((x) => TaskParameterSerializer.UndoEnvVarFormat(x)),
),
];
for (const element of keys) {
if (element !== `customJob`) {
buildParameters[element] = process.env[`${TaskParameterSerializer.ToEnvVarFormat(element)}`];
}
}
return buildParameters;
}
private static serializeInput() {
return TaskParameterSerializer.serializeFromType(Input);
}
private static serializeCloudRunnerOptions() {
return TaskParameterSerializer.serializeFromType(CloudRunnerOptions);
}
public static ToEnvVarFormat(input: string): string {
return CloudRunnerOptions.ToEnvVarFormat(input);
}
public static UndoEnvVarFormat(element: string): string {
return this.camelize(element.toLowerCase().replace(/_+/g, ' '));
}
private static camelize(string: string) {
return TaskParameterSerializer.uncapitalizeFirstLetter(
string
.replace(/(^\w)|([A-Z])|(\b\w)/g, function (word: string, index: number) {
return index === 0 ? word.toLowerCase() : word.toUpperCase();
})
.replace(/\s+/g, ''),
);
}
private static uncapitalizeFirstLetter(string: string) {
return string.charAt(0).toLowerCase() + string.slice(1);
}
private static serializeFromObject(buildParameters: any) {
const array: any[] = [];
const keys = Object.getOwnPropertyNames(buildParameters).filter((x) => !this.blockedParameterNames.has(x));
for (const element of keys) {
array.push({
name: TaskParameterSerializer.ToEnvVarFormat(element),
value: buildParameters[element],
});
}
return array;
}
private static serializeFromType(type: any) {
const array: any[] = [];
const input = CloudRunnerOptionsReader.GetProperties();
for (const element of input) {
if (typeof type[element] !== 'function' && array.filter((x) => x.name === element).length === 0) {
array.push({
name: element,
value: `${type[element]}`,
});
}
}
return array;
}
public static readDefaultSecrets(): CloudRunnerSecret[] {
let array = new Array();
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_SERIAL');
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_EMAIL');
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_PASSWORD');
// array = TaskParameterSerializer.tryAddInput(array, 'UNITY_LICENSE');
array = TaskParameterSerializer.tryAddInput(array, 'GIT_PRIVATE_TOKEN');
return array;
}
private static getValue(key: string) {
return CloudRunnerQueryOverride.queryOverrides !== undefined &&
CloudRunnerQueryOverride.queryOverrides[key] !== undefined
? CloudRunnerQueryOverride.queryOverrides[key]
: process.env[key];
}
private static tryAddInput(array: CloudRunnerSecret[], key: string): CloudRunnerSecret[] {
const value = TaskParameterSerializer.getValue(key);
if (value !== undefined && value !== '' && value !== 'null') {
array.push({
ParameterKey: key,
EnvironmentVariable: key,
ParameterValue: value,
});
}
return array;
}
}
@@ -0,0 +1,118 @@
import { BuildParameters, Input } from '../../..';
import YAML from 'yaml';
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
import path from 'node:path';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import * as fs from 'node:fs';
import CloudRunnerLogger from '../core/cloud-runner-logger';
import { CommandHook } from './command-hook';
// import CloudRunnerLogger from './cloud-runner-logger';
export class CommandHookService {
public static ApplyHooksToCommands(commands: string, buildParameters: BuildParameters): string {
const hooks = CommandHookService.getHooks(buildParameters.commandHooks);
CloudRunnerLogger.log(`Applying hooks ${hooks.length}`);
return `echo "---"
echo "start cloud runner init"
${CloudRunnerOptions.cloudRunnerDebug ? `printenv` : `#`}
echo "start of cloud runner job"
${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
${commands}
${hooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
echo "end of cloud runner job"
echo "---${buildParameters.logId}"`;
}
public static getHooks(customCommandHooks: string): CommandHook[] {
const experimentHooks = customCommandHooks;
let output = new Array<CommandHook>();
if (experimentHooks && experimentHooks !== '') {
try {
output = YAML.parse(experimentHooks);
} catch (error) {
throw error;
}
}
return [
...output.filter((x) => x.hook !== undefined && x.hook.length > 0),
...CommandHookService.GetCustomHooksFromFiles(`before`),
...CommandHookService.GetCustomHooksFromFiles(`after`),
];
}
static GetCustomHooksFromFiles(hookLifecycle: string): CommandHook[] {
const results: CommandHook[] = [];
// RemoteClientLogger.log(`GetCustomHookFiles: ${hookLifecycle}`);
try {
const gameCiCustomHooksPath = path.join(process.cwd(), `game-ci`, `command-hooks`);
const files = fs.readdirSync(gameCiCustomHooksPath);
for (const file of files) {
if (!CloudRunnerOptions.commandHookFiles.includes(file.replace(`.yaml`, ``))) {
continue;
}
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
const fileContentsObject = CommandHookService.ParseHooks(fileContents)[0];
if (fileContentsObject.hook.includes(hookLifecycle)) {
results.push(fileContentsObject);
}
}
} catch (error) {
RemoteClientLogger.log(`Failed Getting: ${hookLifecycle} \n ${JSON.stringify(error, undefined, 4)}`);
}
// RemoteClientLogger.log(`Active Steps From Hooks: \n ${JSON.stringify(results, undefined, 4)}`);
return results;
}
private static ConvertYamlSecrets(object: CommandHook) {
if (object.secrets === undefined) {
object.secrets = [];
return;
}
object.secrets = object.secrets.map((x: any) => {
return {
ParameterKey: x.name,
EnvironmentVariable: Input.ToEnvVarFormat(x.name),
ParameterValue: x.value,
};
});
}
public static ParseHooks(hooks: string): CommandHook[] {
if (hooks === '') {
return [];
}
// if (CloudRunner.buildParameters?.cloudRunnerIntegrationTests) {
// CloudRunnerLogger.log(`Parsing build hooks: ${steps}`);
// }
const isArray = hooks.replace(/\s/g, ``)[0] === `-`;
const object: CommandHook[] = isArray ? YAML.parse(hooks) : [YAML.parse(hooks)];
for (const hook of object) {
CommandHookService.ConvertYamlSecrets(hook);
if (hook.secrets === undefined) {
hook.secrets = [];
}
}
if (object === undefined) {
throw new Error(`Failed to parse ${hooks}`);
}
return object;
}
public static getSecrets(hooks: any) {
const secrets = hooks.map((x: any) => x.secrets).filter((x: any) => x !== undefined && x.length > 0);
// eslint-disable-next-line unicorn/no-array-reduce
return secrets.length > 0 ? secrets.reduce((x: any, y: any) => [...x, ...y]) : [];
}
}
@@ -0,0 +1,9 @@
import CloudRunnerSecret from '../../options/cloud-runner-secret';
export class CommandHook {
public commands: string[] = new Array<string>();
public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
public name!: string;
public hook!: string[];
public step!: string[];
}
@@ -0,0 +1,265 @@
import YAML from 'yaml';
import CloudRunner from '../../cloud-runner';
import { CustomWorkflow } from '../../workflows/custom-workflow';
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
import path from 'node:path';
import fs from 'node:fs';
import Input from '../../../input';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import { ContainerHook as ContainerHook } from './container-hook';
import { CloudRunnerStepParameters } from '../../options/cloud-runner-step-parameters';
export class ContainerHookService {
static GetContainerHooksFromFiles(hookLifecycle: string): ContainerHook[] {
const results: ContainerHook[] = [];
try {
const gameCiCustomStepsPath = path.join(process.cwd(), `game-ci`, `container-hooks`);
const files = fs.readdirSync(gameCiCustomStepsPath);
for (const file of files) {
if (!CloudRunnerOptions.containerHookFiles.includes(file.replace(`.yaml`, ``))) {
// RemoteClientLogger.log(`Skipping CustomStepFile: ${file}`);
continue;
}
const fileContents = fs.readFileSync(path.join(gameCiCustomStepsPath, file), `utf8`);
const fileContentsObject = ContainerHookService.ParseContainerHooks(fileContents)[0];
if (fileContentsObject.hook === hookLifecycle) {
results.push(fileContentsObject);
}
}
} catch (error) {
RemoteClientLogger.log(`Failed Getting: ${hookLifecycle} \n ${JSON.stringify(error, undefined, 4)}`);
}
// RemoteClientLogger.log(`Active Steps From Files: \n ${JSON.stringify(results, undefined, 4)}`);
const builtInContainerHooks: ContainerHook[] = ContainerHookService.ParseContainerHooks(
`- name: aws-s3-upload-build
image: amazon/aws-cli
hook: after
commands: |
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 cp /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
} s3://${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}
rm /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}
secrets:
- name: awsAccessKeyId
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
- name: awsSecretAccessKey
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
- name: awsDefaultRegion
value: ${process.env.AWS_REGION || ``}
- name: aws-s3-pull-build
image: amazon/aws-cli
commands: |
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/ || true
aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/build || true
mkdir -p /data/cache/$CACHE_KEY/build/
aws s3 cp s3://${
CloudRunner.buildParameters.awsStackName
}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
} /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}
secrets:
- name: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
- name: AWS_DEFAULT_REGION
- name: BUILD_GUID_TARGET
- name: steam-deploy-client
image: steamcmd/steamcmd
commands: |
apt-get update
apt-get install -y curl tar coreutils git tree > /dev/null
curl -s https://gist.githubusercontent.com/frostebite/1d56f5505b36b403b64193b7a6e54cdc/raw/fa6639ed4ef750c4268ea319d63aa80f52712ffb/deploy-client-steam.sh | bash
secrets:
- name: STEAM_USERNAME
- name: STEAM_PASSWORD
- name: STEAM_APPID
- name: STEAM_SSFN_FILE_NAME
- name: STEAM_SSFN_FILE_CONTENTS
- name: STEAM_CONFIG_VDF_1
- name: STEAM_CONFIG_VDF_2
- name: STEAM_CONFIG_VDF_3
- name: STEAM_CONFIG_VDF_4
- name: BUILD_GUID_TARGET
- name: RELEASE_BRANCH
- name: steam-deploy-project
image: steamcmd/steamcmd
commands: |
apt-get update
apt-get install -y curl tar coreutils git tree > /dev/null
curl -s https://gist.githubusercontent.com/frostebite/969da6a41002a0e901174124b643709f/raw/02403e53fb292026cba81ddcf4ff35fc1eba111d/steam-deploy-project.sh | bash
secrets:
- name: STEAM_USERNAME
- name: STEAM_PASSWORD
- name: STEAM_APPID
- name: STEAM_SSFN_FILE_NAME
- name: STEAM_SSFN_FILE_CONTENTS
- name: STEAM_CONFIG_VDF_1
- name: STEAM_CONFIG_VDF_2
- name: STEAM_CONFIG_VDF_3
- name: STEAM_CONFIG_VDF_4
- name: BUILD_GUID_2
- name: RELEASE_BRANCH
- name: aws-s3-upload-cache
image: amazon/aws-cli
hook: after
commands: |
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://${
CloudRunner.buildParameters.awsStackName
}/cloud-runner-cache/$CACHE_KEY/lfs
rm -r /data/cache/$CACHE_KEY/lfs
aws s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://${
CloudRunner.buildParameters.awsStackName
}/cloud-runner-cache/$CACHE_KEY/Library
rm -r /data/cache/$CACHE_KEY/Library
secrets:
- name: AWS_ACCESS_KEY_ID
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
- name: AWS_SECRET_ACCESS_KEY
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
- name: AWS_DEFAULT_REGION
value: ${process.env.AWS_REGION || ``}
- name: aws-s3-pull-cache
image: amazon/aws-cli
hook: before
commands: |
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
mkdir -p /data/cache/$CACHE_KEY/Library/
mkdir -p /data/cache/$CACHE_KEY/lfs/
aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/ || true
aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/ || true
BUCKET1="${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/Library/"
aws s3 ls $BUCKET1 || true
OBJECT1="$(aws s3 ls $BUCKET1 | sort | tail -n 1 | awk '{print $4}' || '')"
aws s3 cp s3://$BUCKET1$OBJECT1 /data/cache/$CACHE_KEY/Library/ || true
BUCKET2="${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/lfs/"
aws s3 ls $BUCKET2 || true
OBJECT2="$(aws s3 ls $BUCKET2 | sort | tail -n 1 | awk '{print $4}' || '')"
aws s3 cp s3://$BUCKET2$OBJECT2 /data/cache/$CACHE_KEY/lfs/ || true
secrets:
- name: AWS_ACCESS_KEY_ID
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
- name: AWS_SECRET_ACCESS_KEY
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
- name: AWS_DEFAULT_REGION
value: ${process.env.AWS_REGION || ``}
- name: debug-cache
image: ubuntu
hook: after
commands: |
apt-get update > /dev/null
${CloudRunnerOptions.cloudRunnerDebug ? `apt-get install -y tree > /dev/null` : `#`}
${CloudRunnerOptions.cloudRunnerDebug ? `tree -L 3 /data/cache` : `#`}
secrets:
- name: awsAccessKeyId
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
- name: awsSecretAccessKey
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
- name: awsDefaultRegion
value: ${process.env.AWS_REGION || ``}`,
).filter((x) => CloudRunnerOptions.containerHookFiles.includes(x.name) && x.hook === hookLifecycle);
if (builtInContainerHooks.length > 0) {
results.push(...builtInContainerHooks);
}
return results;
}
private static ConvertYamlSecrets(object: ContainerHook) {
if (object.secrets === undefined) {
object.secrets = [];
return;
}
object.secrets = object.secrets.map((x: { [key: string]: any }) => {
return {
ParameterKey: x.name,
EnvironmentVariable: Input.ToEnvVarFormat(x.name),
ParameterValue: x.value,
};
});
}
public static ParseContainerHooks(steps: string): ContainerHook[] {
if (steps === '') {
return [];
}
const isArray = steps.replace(/\s/g, ``)[0] === `-`;
const object: ContainerHook[] = isArray ? YAML.parse(steps) : [YAML.parse(steps)];
for (const step of object) {
ContainerHookService.ConvertYamlSecrets(step);
if (step.secrets === undefined) {
step.secrets = [];
} else {
for (const secret of step.secrets) {
if (secret.ParameterValue === undefined && process.env[secret.EnvironmentVariable] !== undefined) {
if (CloudRunner.buildParameters?.cloudRunnerDebug) {
// CloudRunnerLogger.log(`Injecting custom step ${step.name} from env var ${secret.ParameterKey}`);
}
secret.ParameterValue = process.env[secret.ParameterKey] || ``;
}
}
}
if (step.image === undefined) {
step.image = `ubuntu`;
}
}
if (object === undefined) {
throw new Error(`Failed to parse ${steps}`);
}
return object;
}
static async RunPostBuildSteps(cloudRunnerStepState: CloudRunnerStepParameters) {
let output = ``;
const steps: ContainerHook[] = [
...ContainerHookService.ParseContainerHooks(CloudRunner.buildParameters.postBuildContainerHooks),
...ContainerHookService.GetContainerHooksFromFiles(`after`),
];
if (steps.length > 0) {
output += await CustomWorkflow.runContainerJob(
steps,
cloudRunnerStepState.environment,
cloudRunnerStepState.secrets,
);
}
return output;
}
static async RunPreBuildSteps(cloudRunnerStepState: CloudRunnerStepParameters) {
let output = ``;
const steps: ContainerHook[] = [
...ContainerHookService.ParseContainerHooks(CloudRunner.buildParameters.preBuildContainerHooks),
...ContainerHookService.GetContainerHooksFromFiles(`before`),
];
if (steps.length > 0) {
output += await CustomWorkflow.runContainerJob(
steps,
cloudRunnerStepState.environment,
cloudRunnerStepState.secrets,
);
}
return output;
}
}
@@ -0,0 +1,9 @@
import CloudRunnerSecret from '../../options/cloud-runner-secret';
export class ContainerHook {
public commands!: string;
public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
public name!: string;
public image: string = `ubuntu`;
public hook!: string;
}

Some files were not shown because too many files have changed in this diff Show More