mirror of
https://github.com/game-ci/unity-builder.git
synced 2026-06-01 06:16:14 -07:00
Compare commits
22 Commits
v4.6.3
...
feature/us
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4233b08bae | ||
|
|
2badde1790 | ||
|
|
eba50f7627 | ||
|
|
f8b20890d9 | ||
|
|
b57598a959 | ||
|
|
7b2ec07fc1 | ||
|
|
3d3a018c23 | ||
|
|
a12e3e829e | ||
|
|
2321712bb4 | ||
|
|
ef0555fb0d | ||
|
|
4a7fc08e63 | ||
|
|
ce7ce7a416 | ||
|
|
9d475434d3 | ||
|
|
f3849ee1c9 | ||
|
|
0c82a58873 | ||
|
|
1d4ee0697f | ||
|
|
3a2abf9037 | ||
|
|
cfdebb67c1 | ||
|
|
ab64768ceb | ||
|
|
00fa0d3772 | ||
|
|
d587557287 | ||
|
|
6e0bf17345 |
@@ -2,3 +2,4 @@ dist/
|
|||||||
lib/
|
lib/
|
||||||
node_modules/
|
node_modules/
|
||||||
jest.config.js
|
jest.config.js
|
||||||
|
src/types/
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
{
|
{
|
||||||
|
"root": true,
|
||||||
"plugins": ["jest", "@typescript-eslint", "prettier", "unicorn"],
|
"plugins": ["jest", "@typescript-eslint", "prettier", "unicorn"],
|
||||||
"extends": ["plugin:unicorn/recommended", "plugin:github/recommended", "plugin:prettier/recommended"],
|
"extends": ["plugin:unicorn/recommended", "plugin:github/recommended", "plugin:prettier/recommended"],
|
||||||
"parser": "@typescript-eslint/parser",
|
"parser": "@typescript-eslint/parser",
|
||||||
@@ -14,7 +15,8 @@
|
|||||||
"env": {
|
"env": {
|
||||||
"node": true,
|
"node": true,
|
||||||
"es6": true,
|
"es6": true,
|
||||||
"jest/globals": true
|
"jest/globals": true,
|
||||||
|
"es2020": true
|
||||||
},
|
},
|
||||||
"rules": {
|
"rules": {
|
||||||
// Error out for code formatting errors
|
// Error out for code formatting errors
|
||||||
@@ -77,5 +79,13 @@
|
|||||||
"unicorn/prefer-spread": "off",
|
"unicorn/prefer-spread": "off",
|
||||||
// Temp disable to prevent mixing changes with other PRs
|
// Temp disable to prevent mixing changes with other PRs
|
||||||
"i18n-text/no-en": "off"
|
"i18n-text/no-en": "off"
|
||||||
}
|
},
|
||||||
|
"overrides": [
|
||||||
|
{
|
||||||
|
"files": ["jest.setup.js"],
|
||||||
|
"rules": {
|
||||||
|
"import/no-commonjs": "off"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
6
.github/workflows/build-tests-mac.yml
vendored
6
.github/workflows/build-tests-mac.yml
vendored
@@ -18,9 +18,9 @@ jobs:
|
|||||||
projectPath:
|
projectPath:
|
||||||
- test-project
|
- test-project
|
||||||
unityVersion:
|
unityVersion:
|
||||||
- 2021.3.45f1
|
- 2021.3.45f2
|
||||||
- 2022.3.13f1
|
- 2022.3.62f3
|
||||||
- 2023.2.2f1
|
- 2023.2.22f1
|
||||||
targetPlatform:
|
targetPlatform:
|
||||||
- StandaloneOSX # Build a MacOS executable
|
- StandaloneOSX # Build a MacOS executable
|
||||||
- iOS # Build an iOS executable
|
- iOS # Build an iOS executable
|
||||||
|
|||||||
6
.github/workflows/build-tests-ubuntu.yml
vendored
6
.github/workflows/build-tests-ubuntu.yml
vendored
@@ -48,9 +48,9 @@ jobs:
|
|||||||
projectPath:
|
projectPath:
|
||||||
- test-project
|
- test-project
|
||||||
unityVersion:
|
unityVersion:
|
||||||
- 2021.3.32f1
|
- 2021.3.45f2
|
||||||
- 2022.3.13f1
|
- 2022.3.62f3
|
||||||
- 2023.2.2f1
|
- 2023.2.22f1
|
||||||
targetPlatform:
|
targetPlatform:
|
||||||
- StandaloneOSX # Build a macOS standalone (Intel 64-bit) with mono backend.
|
- StandaloneOSX # Build a macOS standalone (Intel 64-bit) with mono backend.
|
||||||
- StandaloneWindows64 # Build a Windows 64-bit standalone with mono backend.
|
- StandaloneWindows64 # Build a Windows 64-bit standalone with mono backend.
|
||||||
|
|||||||
40
.github/workflows/build-tests-windows.yml
vendored
40
.github/workflows/build-tests-windows.yml
vendored
@@ -18,9 +18,9 @@ jobs:
|
|||||||
projectPath:
|
projectPath:
|
||||||
- test-project
|
- test-project
|
||||||
unityVersion:
|
unityVersion:
|
||||||
- 2021.3.32f1
|
- 2021.3.45f2
|
||||||
- 2022.3.13f1
|
- 2022.3.62f3
|
||||||
- 2023.2.2f1
|
- 2023.2.22f1
|
||||||
targetPlatform:
|
targetPlatform:
|
||||||
- Android # Build an Android apk.
|
- Android # Build an Android apk.
|
||||||
- StandaloneWindows64 # Build a Windows 64-bit standalone.
|
- StandaloneWindows64 # Build a Windows 64-bit standalone.
|
||||||
@@ -39,7 +39,7 @@ jobs:
|
|||||||
- unityVersion: 6000.0.36f1
|
- unityVersion: 6000.0.36f1
|
||||||
targetPlatform: StandaloneWindows64
|
targetPlatform: StandaloneWindows64
|
||||||
buildProfile: 'Assets/Settings/Build Profiles/Sample Windows Build Profile.asset'
|
buildProfile: 'Assets/Settings/Build Profiles/Sample Windows Build Profile.asset'
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
###########################
|
###########################
|
||||||
# Checkout #
|
# Checkout #
|
||||||
@@ -66,6 +66,34 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
Move-Item -Path "./test-project/ProjectSettings/ProjectSettingsIl2cpp.asset" -Destination "./test-project/ProjectSettings/ProjectSettings.asset" -Force
|
Move-Item -Path "./test-project/ProjectSettings/ProjectSettingsIl2cpp.asset" -Destination "./test-project/ProjectSettings/ProjectSettings.asset" -Force
|
||||||
|
|
||||||
|
###########################
|
||||||
|
# Docker Readiness #
|
||||||
|
###########################
|
||||||
|
- name: Ensure Docker daemon is ready
|
||||||
|
timeout-minutes: 2
|
||||||
|
shell: powershell
|
||||||
|
run: |
|
||||||
|
$maxRetries = 10
|
||||||
|
$retryDelay = 6
|
||||||
|
for ($i = 0; $i -lt $maxRetries; $i++) {
|
||||||
|
$svc = Get-Service docker -ErrorAction SilentlyContinue
|
||||||
|
if ($svc -and $svc.Status -eq 'Running') {
|
||||||
|
docker version 2>$null
|
||||||
|
if ($LASTEXITCODE -eq 0) {
|
||||||
|
Write-Host "Docker is ready."
|
||||||
|
exit 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($svc -and $svc.Status -eq 'Stopped') {
|
||||||
|
Write-Host "Docker service stopped, attempting to start..."
|
||||||
|
Start-Service docker -ErrorAction SilentlyContinue
|
||||||
|
}
|
||||||
|
Write-Host "Waiting for Docker daemon (attempt $($i+1)/$maxRetries)..."
|
||||||
|
Start-Sleep -Seconds $retryDelay
|
||||||
|
}
|
||||||
|
Write-Error "Docker daemon did not start within $($maxRetries * $retryDelay) seconds"
|
||||||
|
exit 1
|
||||||
|
|
||||||
###########################
|
###########################
|
||||||
# Build #
|
# Build #
|
||||||
###########################
|
###########################
|
||||||
@@ -146,6 +174,8 @@ jobs:
|
|||||||
###########################
|
###########################
|
||||||
- uses: actions/upload-artifact@v4
|
- uses: actions/upload-artifact@v4
|
||||||
with:
|
with:
|
||||||
name: Build ${{ matrix.targetPlatform }} on Windows (${{ matrix.unityVersion }})${{ matrix.enableGpu && ' With GPU' || '' }}${{ matrix.buildProfile && ' With Build Profile' || '' }}
|
name:
|
||||||
|
Build ${{ matrix.targetPlatform }} on Windows (${{ matrix.unityVersion }})${{ matrix.enableGpu && ' With
|
||||||
|
GPU' || '' }}${{ matrix.buildProfile && ' With Build Profile' || '' }}
|
||||||
path: build
|
path: build
|
||||||
retention-days: 14
|
retention-days: 14
|
||||||
|
|||||||
37
.github/workflows/cleanup.yml
vendored
37
.github/workflows/cleanup.yml
vendored
@@ -1,37 +0,0 @@
|
|||||||
name: Cleanup (cron)
|
|
||||||
on:
|
|
||||||
schedule:
|
|
||||||
- cron: '30 10 * * SUN' # every sunday at 10:30
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
deleteArtifacts:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Delete old artifacts
|
|
||||||
uses: kolpav/purge-artifacts-action@v1
|
|
||||||
with:
|
|
||||||
token: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
expire-in: 21 days
|
|
||||||
cleanupCloudRunner:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
if: github.event.event_type != 'pull_request_target'
|
|
||||||
with:
|
|
||||||
lfs: true
|
|
||||||
- uses: actions/setup-node@v4
|
|
||||||
with:
|
|
||||||
node-version: '18'
|
|
||||||
- run: yarn
|
|
||||||
- run: yarn run cli --help
|
|
||||||
env:
|
|
||||||
AWS_REGION: eu-west-2
|
|
||||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
||||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
||||||
AWS_DEFAULT_REGION: eu-west-2
|
|
||||||
- run: yarn run cli -m list-resources
|
|
||||||
env:
|
|
||||||
AWS_REGION: eu-west-2
|
|
||||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
||||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
||||||
AWS_DEFAULT_REGION: eu-west-2
|
|
||||||
59
.github/workflows/cloud-runner-async-checks.yml
vendored
59
.github/workflows/cloud-runner-async-checks.yml
vendored
@@ -1,59 +0,0 @@
|
|||||||
name: Async Checks API
|
|
||||||
|
|
||||||
on:
|
|
||||||
workflow_dispatch:
|
|
||||||
inputs:
|
|
||||||
checksObject:
|
|
||||||
description: ''
|
|
||||||
required: false
|
|
||||||
default: ''
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
checks: write
|
|
||||||
|
|
||||||
env:
|
|
||||||
GKE_ZONE: 'us-central1'
|
|
||||||
GKE_REGION: 'us-central1'
|
|
||||||
GKE_PROJECT: 'unitykubernetesbuilder'
|
|
||||||
GKE_CLUSTER: 'game-ci-github-pipelines'
|
|
||||||
GCP_LOGGING: true
|
|
||||||
GCP_PROJECT: unitykubernetesbuilder
|
|
||||||
GCP_LOG_FILE: ${{ github.workspace }}/cloud-runner-logs.txt
|
|
||||||
AWS_REGION: eu-west-2
|
|
||||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
||||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
||||||
AWS_DEFAULT_REGION: eu-west-2
|
|
||||||
AWS_STACK_NAME: game-ci-github-pipelines
|
|
||||||
CLOUD_RUNNER_BRANCH: ${{ github.ref }}
|
|
||||||
CLOUD_RUNNER_DEBUG: true
|
|
||||||
CLOUD_RUNNER_DEBUG_TREE: true
|
|
||||||
DEBUG: true
|
|
||||||
UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
|
|
||||||
PROJECT_PATH: test-project
|
|
||||||
UNITY_VERSION: 2019.3.15f1
|
|
||||||
USE_IL2CPP: false
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
asyncChecks:
|
|
||||||
name: Async Checks
|
|
||||||
if: github.event.event_type != 'pull_request_target'
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- timeout-minutes: 180
|
|
||||||
env:
|
|
||||||
UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
|
|
||||||
PROJECT_PATH: test-project
|
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
TARGET_PLATFORM: StandaloneWindows64
|
|
||||||
cloudRunnerTests: true
|
|
||||||
versioning: None
|
|
||||||
CLOUD_RUNNER_CLUSTER: local-docker
|
|
||||||
AWS_STACK_NAME: game-ci-github-pipelines
|
|
||||||
CHECKS_UPDATE: ${{ github.event.inputs.checksObject }}
|
|
||||||
run: |
|
|
||||||
git clone -b cloud-runner-develop https://github.com/game-ci/unity-builder
|
|
||||||
cd unity-builder
|
|
||||||
yarn
|
|
||||||
ls
|
|
||||||
yarn run cli -m checks-update
|
|
||||||
231
.github/workflows/cloud-runner-ci-pipeline.yml
vendored
231
.github/workflows/cloud-runner-ci-pipeline.yml
vendored
@@ -1,231 +0,0 @@
|
|||||||
name: Cloud Runner CI Pipeline
|
|
||||||
|
|
||||||
on:
|
|
||||||
push: { branches: [cloud-runner-develop, cloud-runner-preview, main] }
|
|
||||||
workflow_dispatch:
|
|
||||||
inputs:
|
|
||||||
runGithubIntegrationTests:
|
|
||||||
description: 'Run GitHub Checks integration tests'
|
|
||||||
required: false
|
|
||||||
default: 'false'
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
checks: write
|
|
||||||
contents: read
|
|
||||||
actions: write
|
|
||||||
|
|
||||||
env:
|
|
||||||
GKE_ZONE: 'us-central1'
|
|
||||||
GKE_REGION: 'us-central1'
|
|
||||||
GKE_PROJECT: 'unitykubernetesbuilder'
|
|
||||||
GKE_CLUSTER: 'game-ci-github-pipelines'
|
|
||||||
GCP_LOGGING: true
|
|
||||||
GCP_PROJECT: unitykubernetesbuilder
|
|
||||||
GCP_LOG_FILE: ${{ github.workspace }}/cloud-runner-logs.txt
|
|
||||||
AWS_REGION: eu-west-2
|
|
||||||
AWS_DEFAULT_REGION: eu-west-2
|
|
||||||
AWS_STACK_NAME: game-ci-team-pipelines
|
|
||||||
CLOUD_RUNNER_BRANCH: ${{ github.ref }}
|
|
||||||
DEBUG: true
|
|
||||||
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
|
|
||||||
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
|
|
||||||
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
|
|
||||||
PROJECT_PATH: test-project
|
|
||||||
UNITY_VERSION: 2019.3.15f1
|
|
||||||
USE_IL2CPP: false
|
|
||||||
USE_GKE_GCLOUD_AUTH_PLUGIN: true
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
tests:
|
|
||||||
name: Tests
|
|
||||||
if: github.event.event_type != 'pull_request_target'
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
strategy:
|
|
||||||
fail-fast: false
|
|
||||||
matrix:
|
|
||||||
test:
|
|
||||||
- 'cloud-runner-end2end-locking'
|
|
||||||
- 'cloud-runner-end2end-caching'
|
|
||||||
- 'cloud-runner-end2end-retaining'
|
|
||||||
- 'cloud-runner-caching'
|
|
||||||
- 'cloud-runner-environment'
|
|
||||||
- 'cloud-runner-image'
|
|
||||||
- 'cloud-runner-hooks'
|
|
||||||
- 'cloud-runner-local-persistence'
|
|
||||||
- 'cloud-runner-locking-core'
|
|
||||||
- 'cloud-runner-locking-get-locked'
|
|
||||||
steps:
|
|
||||||
- name: Checkout (default)
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
lfs: false
|
|
||||||
- name: Configure AWS Credentials
|
|
||||||
uses: aws-actions/configure-aws-credentials@v1
|
|
||||||
with:
|
|
||||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
||||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
||||||
aws-region: eu-west-2
|
|
||||||
- run: yarn
|
|
||||||
- run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
|
|
||||||
timeout-minutes: 60
|
|
||||||
env:
|
|
||||||
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
|
|
||||||
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
|
|
||||||
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
|
|
||||||
PROJECT_PATH: test-project
|
|
||||||
TARGET_PLATFORM: StandaloneWindows64
|
|
||||||
cloudRunnerTests: true
|
|
||||||
versioning: None
|
|
||||||
KUBE_STORAGE_CLASS: local-path
|
|
||||||
PROVIDER_STRATEGY: local-docker
|
|
||||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
||||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
||||||
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
k8sTests:
|
|
||||||
name: K8s Tests
|
|
||||||
if: github.event.event_type != 'pull_request_target'
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
strategy:
|
|
||||||
fail-fast: false
|
|
||||||
matrix:
|
|
||||||
test:
|
|
||||||
# - 'cloud-runner-async-workflow'
|
|
||||||
- 'cloud-runner-end2end-locking'
|
|
||||||
- 'cloud-runner-end2end-caching'
|
|
||||||
- 'cloud-runner-end2end-retaining'
|
|
||||||
- 'cloud-runner-kubernetes'
|
|
||||||
- 'cloud-runner-environment'
|
|
||||||
- 'cloud-runner-github-checks'
|
|
||||||
steps:
|
|
||||||
- name: Checkout (default)
|
|
||||||
uses: actions/checkout@v2
|
|
||||||
with:
|
|
||||||
lfs: false
|
|
||||||
- run: yarn
|
|
||||||
- name: actions-k3s
|
|
||||||
uses: debianmaster/actions-k3s@v1.0.5
|
|
||||||
with:
|
|
||||||
version: 'latest'
|
|
||||||
- run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
|
|
||||||
timeout-minutes: 60
|
|
||||||
env:
|
|
||||||
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
|
|
||||||
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
|
|
||||||
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
|
|
||||||
PROJECT_PATH: test-project
|
|
||||||
TARGET_PLATFORM: StandaloneWindows64
|
|
||||||
cloudRunnerTests: true
|
|
||||||
versioning: None
|
|
||||||
KUBE_STORAGE_CLASS: local-path
|
|
||||||
PROVIDER_STRATEGY: k8s
|
|
||||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
||||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
||||||
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
awsTests:
|
|
||||||
name: AWS Tests
|
|
||||||
if: github.event.event_type != 'pull_request_target'
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
strategy:
|
|
||||||
fail-fast: false
|
|
||||||
matrix:
|
|
||||||
test:
|
|
||||||
- 'cloud-runner-end2end-locking'
|
|
||||||
- 'cloud-runner-end2end-caching'
|
|
||||||
- 'cloud-runner-end2end-retaining'
|
|
||||||
- 'cloud-runner-environment'
|
|
||||||
- 'cloud-runner-s3-steps'
|
|
||||||
steps:
|
|
||||||
- name: Checkout (default)
|
|
||||||
uses: actions/checkout@v2
|
|
||||||
with:
|
|
||||||
lfs: false
|
|
||||||
- name: Configure AWS Credentials
|
|
||||||
uses: aws-actions/configure-aws-credentials@v1
|
|
||||||
with:
|
|
||||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
||||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
||||||
aws-region: eu-west-2
|
|
||||||
- run: yarn
|
|
||||||
- run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
|
|
||||||
timeout-minutes: 60
|
|
||||||
env:
|
|
||||||
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
|
|
||||||
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
|
|
||||||
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
|
|
||||||
PROJECT_PATH: test-project
|
|
||||||
TARGET_PLATFORM: StandaloneWindows64
|
|
||||||
cloudRunnerTests: true
|
|
||||||
versioning: None
|
|
||||||
KUBE_STORAGE_CLASS: local-path
|
|
||||||
PROVIDER_STRATEGY: aws
|
|
||||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
||||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
||||||
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
|
|
||||||
buildTargetTests:
|
|
||||||
name: Local Build Target Tests
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
strategy:
|
|
||||||
fail-fast: false
|
|
||||||
matrix:
|
|
||||||
providerStrategy:
|
|
||||||
#- aws
|
|
||||||
- local-docker
|
|
||||||
#- k8s
|
|
||||||
targetPlatform:
|
|
||||||
- StandaloneOSX # Build a macOS standalone (Intel 64-bit).
|
|
||||||
- StandaloneWindows64 # Build a Windows 64-bit standalone.
|
|
||||||
- StandaloneLinux64 # Build a Linux 64-bit standalone.
|
|
||||||
- WebGL # WebGL.
|
|
||||||
- iOS # Build an iOS player.
|
|
||||||
# - Android # Build an Android .apk.
|
|
||||||
steps:
|
|
||||||
- name: Checkout (default)
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
lfs: false
|
|
||||||
- run: yarn
|
|
||||||
- uses: ./
|
|
||||||
id: unity-build
|
|
||||||
timeout-minutes: 30
|
|
||||||
env:
|
|
||||||
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
|
|
||||||
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
|
|
||||||
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
|
|
||||||
|
|
||||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
||||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
||||||
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
with:
|
|
||||||
cloudRunnerTests: true
|
|
||||||
versioning: None
|
|
||||||
targetPlatform: ${{ matrix.targetPlatform }}
|
|
||||||
providerStrategy: ${{ matrix.providerStrategy }}
|
|
||||||
- run: |
|
|
||||||
cp ./cloud-runner-cache/cache/${{ steps.unity-build.outputs.CACHE_KEY }}/build/${{ steps.unity-build.outputs.BUILD_ARTIFACT }} ${{ steps.unity-build.outputs.BUILD_ARTIFACT }}
|
|
||||||
- uses: actions/upload-artifact@v4
|
|
||||||
with:
|
|
||||||
name: ${{ matrix.providerStrategy }} Build (${{ matrix.targetPlatform }})
|
|
||||||
path: ${{ steps.unity-build.outputs.BUILD_ARTIFACT }}
|
|
||||||
retention-days: 14
|
|
||||||
|
|
||||||
githubChecksIntegration:
|
|
||||||
name: GitHub Checks Integration
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.runGithubIntegrationTests == 'true'
|
|
||||||
env:
|
|
||||||
RUN_GITHUB_INTEGRATION_TESTS: true
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
- uses: actions/setup-node@v4
|
|
||||||
with:
|
|
||||||
node-version: 20
|
|
||||||
cache: 'yarn'
|
|
||||||
- run: yarn install --frozen-lockfile
|
|
||||||
- run: yarn test cloud-runner-github-checks-integration-test --detectOpenHandles --forceExit --runInBand
|
|
||||||
env:
|
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
19
.github/workflows/integrity-check.yml
vendored
19
.github/workflows/integrity-check.yml
vendored
@@ -2,7 +2,13 @@ name: Integrity
|
|||||||
|
|
||||||
on:
|
on:
|
||||||
push: { branches: [main] }
|
push: { branches: [main] }
|
||||||
pull_request: {}
|
pull_request:
|
||||||
|
types: [opened, synchronize, reopened, labeled]
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
checks: write
|
||||||
|
statuses: write
|
||||||
|
|
||||||
env:
|
env:
|
||||||
CODECOV_TOKEN: '2f2eb890-30e2-4724-83eb-7633832cf0de'
|
CODECOV_TOKEN: '2f2eb890-30e2-4724-83eb-7633832cf0de'
|
||||||
@@ -22,7 +28,14 @@ jobs:
|
|||||||
node-version: '18'
|
node-version: '18'
|
||||||
- run: yarn
|
- run: yarn
|
||||||
- run: yarn lint
|
- run: yarn lint
|
||||||
- run: yarn test --coverage
|
- run: yarn test:ci --coverage
|
||||||
- run: bash <(curl -s https://codecov.io/bash)
|
- run: bash <(curl -s https://codecov.io/bash)
|
||||||
- run: yarn build || { echo "build command should always succeed" ; exit 61; }
|
- run: yarn build || { echo "build command should always succeed" ; exit 61; }
|
||||||
# - run: yarn build --quiet && git diff --quiet dist || { echo "dist should be auto generated" ; git diff dist ; exit 62; }
|
# - run: yarn build --quiet && git diff --quiet dist || { echo "dist should be auto generated" ; git diff dist ; exit 62; }
|
||||||
|
|
||||||
|
orchestrator-integration:
|
||||||
|
name: Orchestrator Integration
|
||||||
|
if: >-
|
||||||
|
github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'run-integration')
|
||||||
|
uses: ./.github/workflows/validate-orchestrator-integration.yml
|
||||||
|
secrets: inherit
|
||||||
|
|||||||
91
.github/workflows/sync-secrets.yml
vendored
Normal file
91
.github/workflows/sync-secrets.yml
vendored
Normal file
@@ -0,0 +1,91 @@
|
|||||||
|
name: Sync Secrets to Repositories
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
target_repo:
|
||||||
|
description: 'Target repository (org/repo format)'
|
||||||
|
required: true
|
||||||
|
default: 'game-ci/orchestrator'
|
||||||
|
type: choice
|
||||||
|
options:
|
||||||
|
- game-ci/orchestrator
|
||||||
|
- game-ci/cli
|
||||||
|
dry_run:
|
||||||
|
description: 'Dry run (list secrets to sync without writing)'
|
||||||
|
required: false
|
||||||
|
default: false
|
||||||
|
type: boolean
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
sync-secrets:
|
||||||
|
name: Sync secrets to ${{ inputs.target_repo }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Sync secrets
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
||||||
|
TARGET_REPO: ${{ inputs.target_repo }}
|
||||||
|
DRY_RUN: ${{ inputs.dry_run }}
|
||||||
|
# Secrets to sync — values come from repo + org secrets available here
|
||||||
|
SECRET_UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
|
||||||
|
SECRET_UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
|
||||||
|
SECRET_UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
|
||||||
|
SECRET_GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
|
||||||
|
SECRET_GOOGLE_SERVICE_ACCOUNT_EMAIL: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_EMAIL }}
|
||||||
|
SECRET_GOOGLE_SERVICE_ACCOUNT_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
|
||||||
|
SECRET_CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||||
|
SECRET_UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
|
||||||
|
SECRET_NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||||
|
run: |
|
||||||
|
SECRETS=(
|
||||||
|
"UNITY_EMAIL:SECRET_UNITY_EMAIL"
|
||||||
|
"UNITY_PASSWORD:SECRET_UNITY_PASSWORD"
|
||||||
|
"UNITY_SERIAL:SECRET_UNITY_SERIAL"
|
||||||
|
"UNITY_LICENSE:SECRET_UNITY_LICENSE"
|
||||||
|
"GIT_PRIVATE_TOKEN:SECRET_GIT_PRIVATE_TOKEN"
|
||||||
|
"GOOGLE_SERVICE_ACCOUNT_EMAIL:SECRET_GOOGLE_SERVICE_ACCOUNT_EMAIL"
|
||||||
|
"GOOGLE_SERVICE_ACCOUNT_KEY:SECRET_GOOGLE_SERVICE_ACCOUNT_KEY"
|
||||||
|
"CODECOV_TOKEN:SECRET_CODECOV_TOKEN"
|
||||||
|
"NPM_TOKEN:SECRET_NPM_TOKEN"
|
||||||
|
)
|
||||||
|
|
||||||
|
synced=0
|
||||||
|
skipped=0
|
||||||
|
|
||||||
|
for entry in "${SECRETS[@]}"; do
|
||||||
|
name="${entry%%:*}"
|
||||||
|
env_var="${entry##*:}"
|
||||||
|
value="${!env_var}"
|
||||||
|
|
||||||
|
if [ -z "$value" ]; then
|
||||||
|
echo "⏭ SKIP: $name (not available in this repo's context)"
|
||||||
|
skipped=$((skipped + 1))
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$DRY_RUN" = "true" ]; then
|
||||||
|
echo "🔍 DRY RUN: would sync $name → $TARGET_REPO"
|
||||||
|
else
|
||||||
|
if echo "$value" | gh secret set "$name" -R "$TARGET_REPO" --body - 2>/dev/null; then
|
||||||
|
echo "✅ SYNCED: $name → $TARGET_REPO"
|
||||||
|
else
|
||||||
|
echo "⚠️ FAILED: $name → $TARGET_REPO (continuing)"
|
||||||
|
skipped=$((skipped + 1))
|
||||||
|
synced=$((synced - 1))
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
synced=$((synced + 1))
|
||||||
|
done
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "=== Summary ==="
|
||||||
|
echo "Synced: $synced"
|
||||||
|
echo "Skipped (not available): $skipped"
|
||||||
|
echo "Target: $TARGET_REPO"
|
||||||
|
if [ "$DRY_RUN" = "true" ]; then
|
||||||
|
echo "Mode: DRY RUN (no secrets were written)"
|
||||||
|
fi
|
||||||
203
.github/workflows/validate-community-plugins.yml
vendored
Normal file
203
.github/workflows/validate-community-plugins.yml
vendored
Normal file
@@ -0,0 +1,203 @@
|
|||||||
|
name: Validate Community Plugins
|
||||||
|
|
||||||
|
on:
|
||||||
|
schedule:
|
||||||
|
# Run weekly on Sunday at 02:00 UTC
|
||||||
|
- cron: '0 2 * * 0'
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
plugin_filter:
|
||||||
|
description: 'Filter plugins by name (regex pattern, empty = all)'
|
||||||
|
required: false
|
||||||
|
default: ''
|
||||||
|
unity_version:
|
||||||
|
description: 'Override Unity version (empty = use plugin default)'
|
||||||
|
required: false
|
||||||
|
default: ''
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
issues: write
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
load-plugins:
|
||||||
|
name: Load Plugin Registry
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
outputs:
|
||||||
|
matrix: ${{ steps.parse.outputs.matrix }}
|
||||||
|
plugin_count: ${{ steps.parse.outputs.count }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Parse plugin registry
|
||||||
|
id: parse
|
||||||
|
uses: actions/github-script@v7
|
||||||
|
with:
|
||||||
|
script: |
|
||||||
|
const fs = require('fs');
|
||||||
|
const yaml = require('js-yaml');
|
||||||
|
|
||||||
|
const registry = yaml.load(fs.readFileSync('community-plugins.yml', 'utf8'));
|
||||||
|
let plugins = registry.plugins || [];
|
||||||
|
|
||||||
|
// Apply name filter if provided
|
||||||
|
const filter = '${{ github.event.inputs.plugin_filter }}';
|
||||||
|
if (filter) {
|
||||||
|
const regex = new RegExp(filter, 'i');
|
||||||
|
plugins = plugins.filter(p => regex.test(p.name));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Expand platform matrix
|
||||||
|
const matrix = [];
|
||||||
|
for (const plugin of plugins) {
|
||||||
|
const platforms = plugin.platforms || ['StandaloneLinux64'];
|
||||||
|
for (const platform of platforms) {
|
||||||
|
matrix.push({
|
||||||
|
name: plugin.name,
|
||||||
|
package: plugin.package,
|
||||||
|
source: plugin.source || 'git',
|
||||||
|
unity: '${{ github.event.inputs.unity_version }}' || plugin.unity || '2021.3',
|
||||||
|
platform: platform,
|
||||||
|
timeout: plugin.timeout || 30
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
core.setOutput('matrix', JSON.stringify({ include: matrix }));
|
||||||
|
core.setOutput('count', matrix.length);
|
||||||
|
console.log(`Found ${matrix.length} plugin-platform combinations to validate`);
|
||||||
|
|
||||||
|
validate:
|
||||||
|
name: '${{ matrix.name }} (${{ matrix.platform }})'
|
||||||
|
needs: load-plugins
|
||||||
|
if: needs.load-plugins.outputs.plugin_count > 0
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: ${{ fromJson(matrix.timeout) }}
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix: ${{ fromJson(needs.load-plugins.outputs.matrix) }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Create test project
|
||||||
|
run: |
|
||||||
|
mkdir -p test-project/Assets
|
||||||
|
mkdir -p test-project/Packages
|
||||||
|
mkdir -p test-project/ProjectSettings
|
||||||
|
|
||||||
|
# Create minimal manifest.json
|
||||||
|
if [ "${{ matrix.source }}" = "git" ]; then
|
||||||
|
cat > test-project/Packages/manifest.json << 'MANIFEST'
|
||||||
|
{
|
||||||
|
"dependencies": {
|
||||||
|
"com.unity.modules.imgui": "1.0.0",
|
||||||
|
"com.unity.modules.jsonserialize": "1.0.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
MANIFEST
|
||||||
|
|
||||||
|
# Add git package via manifest
|
||||||
|
cd test-project
|
||||||
|
cat Packages/manifest.json | python3 -c "
|
||||||
|
import sys, json
|
||||||
|
manifest = json.load(sys.stdin)
|
||||||
|
manifest['dependencies']['${{ matrix.name }}'] = '${{ matrix.package }}'
|
||||||
|
json.dump(manifest, sys.stdout, indent=2)
|
||||||
|
" > Packages/manifest.tmp && mv Packages/manifest.tmp Packages/manifest.json
|
||||||
|
cd ..
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Create minimal ProjectSettings
|
||||||
|
cat > test-project/ProjectSettings/ProjectVersion.txt << EOF
|
||||||
|
m_EditorVersion: ${{ matrix.unity }}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
- name: Build with unity-builder
|
||||||
|
uses: ./
|
||||||
|
id: build
|
||||||
|
with:
|
||||||
|
projectPath: test-project
|
||||||
|
targetPlatform: ${{ matrix.platform }}
|
||||||
|
unityVersion: ${{ matrix.unity }}
|
||||||
|
continue-on-error: true
|
||||||
|
|
||||||
|
- name: Record result
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
STATUS="${{ steps.build.outcome }}"
|
||||||
|
echo "## ${{ matrix.name }} — ${{ matrix.platform }}" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "" >> $GITHUB_STEP_SUMMARY
|
||||||
|
if [ "$STATUS" = "success" ]; then
|
||||||
|
echo "✅ **PASSED** — Compiled and built successfully" >> $GITHUB_STEP_SUMMARY
|
||||||
|
else
|
||||||
|
echo "❌ **FAILED** — Build or compilation failed" >> $GITHUB_STEP_SUMMARY
|
||||||
|
fi
|
||||||
|
echo "" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "- Unity: ${{ matrix.unity }}" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "- Platform: ${{ matrix.platform }}" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "- Source: ${{ matrix.source }}" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "- Package: \`${{ matrix.package }}\`" >> $GITHUB_STEP_SUMMARY
|
||||||
|
|
||||||
|
report:
|
||||||
|
name: Validation Report
|
||||||
|
needs: [load-plugins, validate]
|
||||||
|
if: always()
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Generate summary
|
||||||
|
uses: actions/github-script@v7
|
||||||
|
with:
|
||||||
|
script: |
|
||||||
|
const { data: run } = await github.rest.actions.listJobsForWorkflowRun({
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
run_id: context.runId
|
||||||
|
});
|
||||||
|
|
||||||
|
const validateJobs = run.jobs.filter(j => j.name.startsWith('validate'));
|
||||||
|
const passed = validateJobs.filter(j => j.conclusion === 'success').length;
|
||||||
|
const failed = validateJobs.filter(j => j.conclusion === 'failure').length;
|
||||||
|
const total = validateJobs.length;
|
||||||
|
|
||||||
|
let summary = `# Community Plugin Validation Report\n\n`;
|
||||||
|
summary += `**${passed}/${total} passed** | ${failed} failed\n\n`;
|
||||||
|
summary += `| Plugin | Platform | Status |\n|--------|----------|--------|\n`;
|
||||||
|
|
||||||
|
for (const job of validateJobs) {
|
||||||
|
const icon = job.conclusion === 'success' ? '✅' : '❌';
|
||||||
|
summary += `| ${job.name} | | ${icon} ${job.conclusion} |\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
await core.summary.addRaw(summary).write();
|
||||||
|
|
||||||
|
// Create or update issue if there are failures
|
||||||
|
if (failed > 0) {
|
||||||
|
const title = `Community Plugin Validation: ${failed} failure(s) — ${new Date().toISOString().split('T')[0]}`;
|
||||||
|
const body = summary + `\n\n[Workflow Run](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})`;
|
||||||
|
|
||||||
|
const { data: issues } = await github.rest.issues.listForRepo({
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
state: 'open',
|
||||||
|
labels: 'community-plugin-validation'
|
||||||
|
});
|
||||||
|
|
||||||
|
if (issues.length > 0) {
|
||||||
|
await github.rest.issues.createComment({
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
issue_number: issues[0].number,
|
||||||
|
body: body
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
await github.rest.issues.create({
|
||||||
|
owner: context.repo.owner,
|
||||||
|
repo: context.repo.repo,
|
||||||
|
title: title,
|
||||||
|
body: body,
|
||||||
|
labels: ['community-plugin-validation']
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
1172
.github/workflows/validate-orchestrator-integration.yml
vendored
Normal file
1172
.github/workflows/validate-orchestrator-integration.yml
vendored
Normal file
File diff suppressed because it is too large
Load Diff
237
.github/workflows/validate-orchestrator.yml
vendored
Normal file
237
.github/workflows/validate-orchestrator.yml
vendored
Normal file
@@ -0,0 +1,237 @@
|
|||||||
|
name: Validate Orchestrator Compatibility
|
||||||
|
|
||||||
|
# ==============================================================================
|
||||||
|
# Essential plugin health checks — runs on every PR and push.
|
||||||
|
# Fast (~5 min): compilation, unit tests, plugin interface, type declarations.
|
||||||
|
#
|
||||||
|
# For exhaustive integration tests (k8s, AWS, local-docker, rclone) see
|
||||||
|
# validate-orchestrator-integration.yml which runs on a daily cron.
|
||||||
|
# ==============================================================================
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
push:
|
||||||
|
branches: [main, 'release/**', 'feature/**', 'refactor/**']
|
||||||
|
paths:
|
||||||
|
- 'src/model/orchestrator-plugin.ts'
|
||||||
|
- 'src/model/build-parameters.ts'
|
||||||
|
- 'src/model/input.ts'
|
||||||
|
- 'src/model/github.ts'
|
||||||
|
- 'src/model/cli/cli.ts'
|
||||||
|
- 'src/model/input-readers/**'
|
||||||
|
- 'src/index.ts'
|
||||||
|
- 'src/types/game-ci-orchestrator.d.ts'
|
||||||
|
- 'action.yml'
|
||||||
|
- 'package.json'
|
||||||
|
- 'yarn.lock'
|
||||||
|
- '.github/workflows/validate-orchestrator.yml'
|
||||||
|
pull_request:
|
||||||
|
branches: [main, 'release/**']
|
||||||
|
paths:
|
||||||
|
- 'src/model/orchestrator-plugin.ts'
|
||||||
|
- 'src/model/build-parameters.ts'
|
||||||
|
- 'src/model/input.ts'
|
||||||
|
- 'src/model/github.ts'
|
||||||
|
- 'src/model/cli/cli.ts'
|
||||||
|
- 'src/model/input-readers/**'
|
||||||
|
- 'src/index.ts'
|
||||||
|
- 'src/types/game-ci-orchestrator.d.ts'
|
||||||
|
- 'action.yml'
|
||||||
|
- 'package.json'
|
||||||
|
- 'yarn.lock'
|
||||||
|
- '.github/workflows/validate-orchestrator.yml'
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
packages: read
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
# ============================================================================
|
||||||
|
# PLUGIN ARCHITECTURE HEALTH CHECK
|
||||||
|
# ============================================================================
|
||||||
|
# Validates that:
|
||||||
|
# 1. unity-builder compiles and its unit tests pass
|
||||||
|
# 2. Plugin loader degrades gracefully without orchestrator
|
||||||
|
# 3. Orchestrator compiles and its unit tests pass
|
||||||
|
# 4. Plugin loader loads all services when orchestrator is installed
|
||||||
|
# 5. Type declarations match actual exports
|
||||||
|
# ============================================================================
|
||||||
|
plugin-health:
|
||||||
|
name: Plugin Architecture Health
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout unity-builder
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Checkout orchestrator
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
repository: game-ci/orchestrator
|
||||||
|
ref: ${{ github.head_ref || github.ref_name }}
|
||||||
|
path: orchestrator-standalone
|
||||||
|
continue-on-error: true
|
||||||
|
id: orchestrator-branch
|
||||||
|
|
||||||
|
- name: Fallback to orchestrator main branch
|
||||||
|
if: steps.orchestrator-branch.outcome == 'failure'
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
repository: game-ci/orchestrator
|
||||||
|
path: orchestrator-standalone
|
||||||
|
|
||||||
|
- name: Setup Node.js
|
||||||
|
uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: 20
|
||||||
|
cache: yarn
|
||||||
|
|
||||||
|
# --- unity-builder compilation and tests ---
|
||||||
|
- name: Install unity-builder dependencies
|
||||||
|
run: yarn install --frozen-lockfile
|
||||||
|
|
||||||
|
- name: Build unity-builder
|
||||||
|
run: |
|
||||||
|
echo "Building unity-builder TypeScript..."
|
||||||
|
npx tsc
|
||||||
|
echo "✓ unity-builder compiles successfully"
|
||||||
|
|
||||||
|
- name: Run orchestrator-plugin unit tests
|
||||||
|
run: |
|
||||||
|
echo "Running orchestrator-plugin unit tests..."
|
||||||
|
npx jest orchestrator-plugin --verbose --detectOpenHandles --forceExit
|
||||||
|
|
||||||
|
# --- Plugin loader without orchestrator ---
|
||||||
|
- name: Verify plugin loader returns undefined without orchestrator
|
||||||
|
run: |
|
||||||
|
echo "Checking plugin loader handles missing @game-ci/orchestrator..."
|
||||||
|
node -e "
|
||||||
|
const { loadOrchestratorPlugin } = require('./lib/model/orchestrator-plugin');
|
||||||
|
(async () => {
|
||||||
|
const plugin = await loadOrchestratorPlugin();
|
||||||
|
if (plugin !== undefined) {
|
||||||
|
console.error('ERROR: loadOrchestratorPlugin should return undefined when package not installed');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
console.log('✓ loadOrchestratorPlugin() returns undefined when package not installed');
|
||||||
|
})();
|
||||||
|
"
|
||||||
|
|
||||||
|
- name: Verify orchestrator type declarations exist
|
||||||
|
run: |
|
||||||
|
if [ -f "src/types/game-ci-orchestrator.d.ts" ]; then
|
||||||
|
echo "✓ Type declarations for @game-ci/orchestrator exist"
|
||||||
|
else
|
||||||
|
echo "::error::Missing type declarations: src/types/game-ci-orchestrator.d.ts"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- Orchestrator compilation and tests ---
|
||||||
|
- name: Build and pack orchestrator
|
||||||
|
working-directory: orchestrator-standalone
|
||||||
|
run: |
|
||||||
|
yarn install --frozen-lockfile
|
||||||
|
echo "Building orchestrator..."
|
||||||
|
npx tsc
|
||||||
|
echo "✓ orchestrator compiles successfully"
|
||||||
|
echo "Packing orchestrator as tarball..."
|
||||||
|
npm pack
|
||||||
|
|
||||||
|
- name: Run orchestrator unit tests
|
||||||
|
working-directory: orchestrator-standalone
|
||||||
|
run: |
|
||||||
|
echo "Running orchestrator unit tests..."
|
||||||
|
npx jest --no-cache 2>&1 | tail -20
|
||||||
|
|
||||||
|
# --- Plugin loader with orchestrator installed ---
|
||||||
|
- name: Install orchestrator into unity-builder
|
||||||
|
run: |
|
||||||
|
echo "Installing orchestrator into unity-builder workspace..."
|
||||||
|
npm install ./orchestrator-standalone/game-ci-orchestrator-*.tgz --no-save --legacy-peer-deps
|
||||||
|
|
||||||
|
- name: Verify plugin loader returns exports with orchestrator installed
|
||||||
|
run: |
|
||||||
|
echo "Checking plugin loader returns defined exports..."
|
||||||
|
node -e "
|
||||||
|
const { loadOrchestratorPlugin } = require('./lib/model/orchestrator-plugin');
|
||||||
|
(async () => {
|
||||||
|
const plugin = await loadOrchestratorPlugin();
|
||||||
|
if (plugin === undefined) {
|
||||||
|
console.error('ERROR: loadOrchestratorPlugin should return defined plugin when package is installed');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
const lifecycleMethods = [
|
||||||
|
'initialize', 'canHandleBuild', 'handleBuild',
|
||||||
|
'beforeLocalBuild', 'afterLocalBuild', 'handlePostBuild',
|
||||||
|
];
|
||||||
|
for (const method of lifecycleMethods) {
|
||||||
|
if (typeof plugin[method] !== 'function') {
|
||||||
|
console.error('ERROR: plugin.' + method + ' should be a function, got ' + typeof plugin[method]);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.log('✓ loadOrchestratorPlugin() returns plugin with all ' + lifecycleMethods.length + ' lifecycle methods');
|
||||||
|
})();
|
||||||
|
"
|
||||||
|
|
||||||
|
- name: Verify type declarations match orchestrator exports
|
||||||
|
run: |
|
||||||
|
echo "Checking type declarations align with orchestrator exports..."
|
||||||
|
node -e "
|
||||||
|
const orch = require('@game-ci/orchestrator');
|
||||||
|
const expectedExports = [
|
||||||
|
'Orchestrator', 'BuildReliabilityService', 'TestWorkflowService',
|
||||||
|
'HotRunnerService', 'OutputService', 'OutputTypeRegistry',
|
||||||
|
'ArtifactUploadHandler', 'IncrementalSyncService',
|
||||||
|
'ChildWorkspaceService', 'LocalCacheService', 'SubmoduleProfileService',
|
||||||
|
'LfsAgentService', 'GitHooksService',
|
||||||
|
];
|
||||||
|
const missing = expectedExports.filter(e => orch[e] === undefined);
|
||||||
|
if (missing.length > 0) {
|
||||||
|
console.error('ERROR: Missing exports from @game-ci/orchestrator:', missing.join(', '));
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
console.log('✓ All ' + expectedExports.length + ' declared exports present in orchestrator package');
|
||||||
|
"
|
||||||
|
|
||||||
|
- name: Smoke test orchestrator build wiring
|
||||||
|
run: |
|
||||||
|
echo "Verifying orchestrator build wiring end-to-end..."
|
||||||
|
node -e "
|
||||||
|
const { loadOrchestratorPlugin } = require('./lib/model/orchestrator-plugin');
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
// Verify plugin loads successfully with orchestrator installed
|
||||||
|
const plugin = await loadOrchestratorPlugin();
|
||||||
|
if (plugin === undefined) {
|
||||||
|
console.error('ERROR: plugin should be defined when orchestrator is installed');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify all lifecycle methods are callable
|
||||||
|
const lifecycleMethods = [
|
||||||
|
'initialize', 'canHandleBuild', 'handleBuild',
|
||||||
|
'beforeLocalBuild', 'afterLocalBuild', 'handlePostBuild',
|
||||||
|
];
|
||||||
|
for (const m of lifecycleMethods) {
|
||||||
|
if (typeof plugin[m] !== 'function') {
|
||||||
|
console.error('ERROR: plugin.' + m + ' should be a function, got ' + typeof plugin[m]);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.log('✓ Plugin has all ' + lifecycleMethods.length + ' lifecycle methods');
|
||||||
|
|
||||||
|
// Verify canHandleBuild returns a boolean
|
||||||
|
const canHandle = plugin.canHandleBuild();
|
||||||
|
if (typeof canHandle !== 'boolean') {
|
||||||
|
console.error('ERROR: canHandleBuild() should return a boolean, got ' + typeof canHandle);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
console.log('✓ canHandleBuild() returns boolean');
|
||||||
|
|
||||||
|
console.log('✓ Plugin architecture wiring verified');
|
||||||
|
})();
|
||||||
|
"
|
||||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -5,3 +5,5 @@ lib/
|
|||||||
.vsconfig
|
.vsconfig
|
||||||
yarn-error.log
|
yarn-error.log
|
||||||
.orig
|
.orig
|
||||||
|
$LOG_FILE
|
||||||
|
temp/
|
||||||
|
|||||||
107
action.yml
107
action.yml
@@ -47,6 +47,10 @@ inputs:
|
|||||||
required: false
|
required: false
|
||||||
default: ''
|
default: ''
|
||||||
description: 'Custom parameters to configure the build.'
|
description: 'Custom parameters to configure the build.'
|
||||||
|
useHostNetwork:
|
||||||
|
required: false
|
||||||
|
default: false
|
||||||
|
description: 'Initialises Docker using the host network. (Linux only)'
|
||||||
versioning:
|
versioning:
|
||||||
required: false
|
required: false
|
||||||
default: 'Semantic'
|
default: 'Semantic'
|
||||||
@@ -104,11 +108,13 @@ inputs:
|
|||||||
gitPrivateToken:
|
gitPrivateToken:
|
||||||
required: false
|
required: false
|
||||||
default: ''
|
default: ''
|
||||||
description: '[CloudRunner] Github private token to pull from github'
|
description: 'Github private token to pull from github'
|
||||||
githubOwner:
|
providerStrategy:
|
||||||
|
default: 'local'
|
||||||
required: false
|
required: false
|
||||||
default: ''
|
description:
|
||||||
description: '[CloudRunner] GitHub owner name or organization/team name'
|
'Build execution strategy. Use "local" for local Docker/Mac builds. For remote builds (aws, k8s, etc.), install
|
||||||
|
@game-ci/orchestrator and use the game-ci/orchestrator action which declares its own inputs.'
|
||||||
runAsHostUser:
|
runAsHostUser:
|
||||||
required: false
|
required: false
|
||||||
default: 'false'
|
default: 'false'
|
||||||
@@ -149,98 +155,7 @@ inputs:
|
|||||||
allowDirtyBuild:
|
allowDirtyBuild:
|
||||||
required: false
|
required: false
|
||||||
default: ''
|
default: ''
|
||||||
description: '[CloudRunner] Allows the branch of the build to be dirty, and still generate the build.'
|
description: 'Allows the branch of the build to be dirty, and still generate the build.'
|
||||||
postBuildSteps:
|
|
||||||
required: false
|
|
||||||
default: ''
|
|
||||||
description:
|
|
||||||
'[CloudRunner] run a post build job in yaml format with the keys image, secrets (name, value object array),
|
|
||||||
command string'
|
|
||||||
preBuildSteps:
|
|
||||||
required: false
|
|
||||||
default: ''
|
|
||||||
description:
|
|
||||||
'[CloudRunner] Run a pre build job after the repository setup but before the build job (in yaml format with the
|
|
||||||
keys image, secrets (name, value object array), command line string)'
|
|
||||||
containerHookFiles:
|
|
||||||
required: false
|
|
||||||
default: ''
|
|
||||||
description:
|
|
||||||
'[CloudRunner] Specify the names (by file name) of custom steps to run before or after cloud runner jobs, must
|
|
||||||
match a yaml step file inside your repo in the folder .game-ci/steps/'
|
|
||||||
customHookFiles:
|
|
||||||
required: false
|
|
||||||
default: ''
|
|
||||||
description:
|
|
||||||
'[CloudRunner] Specify the names (by file name) of custom hooks to run before or after cloud runner jobs, must
|
|
||||||
match a yaml step file inside your repo in the folder .game-ci/hooks/'
|
|
||||||
customCommandHooks:
|
|
||||||
required: false
|
|
||||||
default: ''
|
|
||||||
description: '[CloudRunner] Specify custom commands and trigger hooks (injects commands into jobs)'
|
|
||||||
customJob:
|
|
||||||
required: false
|
|
||||||
default: ''
|
|
||||||
description:
|
|
||||||
'[CloudRunner] Run a custom job instead of the standard build automation for cloud runner (in yaml format with the
|
|
||||||
keys image, secrets (name, value object array), command line string)'
|
|
||||||
awsStackName:
|
|
||||||
default: 'game-ci'
|
|
||||||
required: false
|
|
||||||
description: '[CloudRunner] The Cloud Formation stack name that must be setup before using this option.'
|
|
||||||
providerStrategy:
|
|
||||||
default: 'local'
|
|
||||||
required: false
|
|
||||||
description:
|
|
||||||
'[CloudRunner] Either local, k8s or aws can be used to run builds on a remote cluster. Additional parameters must
|
|
||||||
be configured.'
|
|
||||||
containerCpu:
|
|
||||||
default: ''
|
|
||||||
required: false
|
|
||||||
description: '[CloudRunner] Amount of CPU time to assign the remote build container'
|
|
||||||
containerMemory:
|
|
||||||
default: ''
|
|
||||||
required: false
|
|
||||||
description: '[CloudRunner] Amount of memory to assign the remote build container'
|
|
||||||
readInputFromOverrideList:
|
|
||||||
default: ''
|
|
||||||
required: false
|
|
||||||
description: '[CloudRunner] Comma separated list of input value names to read from "input override command"'
|
|
||||||
readInputOverrideCommand:
|
|
||||||
default: ''
|
|
||||||
required: false
|
|
||||||
description:
|
|
||||||
'[CloudRunner] Extend game ci by specifying a command to execute to pull input from external source e.g cloud
|
|
||||||
provider secret managers'
|
|
||||||
kubeConfig:
|
|
||||||
default: ''
|
|
||||||
required: false
|
|
||||||
description:
|
|
||||||
'[CloudRunner] Supply a base64 encoded kubernetes config to run builds on kubernetes and stream logs until
|
|
||||||
completion.'
|
|
||||||
kubeVolume:
|
|
||||||
default: ''
|
|
||||||
required: false
|
|
||||||
description: '[CloudRunner] Supply a Persistent Volume Claim name to use for the Unity build.'
|
|
||||||
kubeStorageClass:
|
|
||||||
default: ''
|
|
||||||
required: false
|
|
||||||
description:
|
|
||||||
'[CloudRunner] Kubernetes storage class to use for cloud runner jobs, leave empty to install rook cluster.'
|
|
||||||
kubeVolumeSize:
|
|
||||||
default: '5Gi'
|
|
||||||
required: false
|
|
||||||
description: '[CloudRunner] Amount of disc space to assign the Kubernetes Persistent Volume'
|
|
||||||
cacheKey:
|
|
||||||
default: ''
|
|
||||||
required: false
|
|
||||||
description: '[CloudRunner] Cache key to indicate bucket for cache'
|
|
||||||
watchToEnd:
|
|
||||||
default: 'true'
|
|
||||||
required: false
|
|
||||||
description:
|
|
||||||
'[CloudRunner] Whether or not to watch the build to the end. Can be used for especially long running jobs e.g
|
|
||||||
imports or self-hosted ephemeral runners.'
|
|
||||||
cacheUnityInstallationOnMac:
|
cacheUnityInstallationOnMac:
|
||||||
default: 'false'
|
default: 'false'
|
||||||
required: false
|
required: false
|
||||||
|
|||||||
27
community-plugins.yml
Normal file
27
community-plugins.yml
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
# Community Plugin Validation Registry
|
||||||
|
# Packages listed here are automatically tested on a schedule
|
||||||
|
# to ensure compatibility with unity-builder.
|
||||||
|
#
|
||||||
|
# Format:
|
||||||
|
# - name: Human-readable name
|
||||||
|
# package: UPM package name or git URL
|
||||||
|
# source: upm | git | asset-store
|
||||||
|
# unity: Minimum Unity version (optional, defaults to 2021.3)
|
||||||
|
# platforms: List of platforms to test (optional, defaults to [StandaloneLinux64])
|
||||||
|
# timeout: Build timeout in minutes (optional, defaults to 30)
|
||||||
|
|
||||||
|
plugins:
|
||||||
|
# Example entries — community members can submit PRs to add their packages
|
||||||
|
- name: UniTask
|
||||||
|
package: https://github.com/Cysharp/UniTask.git?path=src/UniTask/Assets/Plugins/UniTask
|
||||||
|
source: git
|
||||||
|
platforms: [StandaloneLinux64, StandaloneWindows64]
|
||||||
|
|
||||||
|
- name: NaughtyAttributes
|
||||||
|
package: https://github.com/dbrizov/NaughtyAttributes.git?path=Assets/NaughtyAttributes
|
||||||
|
source: git
|
||||||
|
|
||||||
|
- name: Unity Atoms
|
||||||
|
package: https://github.com/unity-atoms/unity-atoms.git
|
||||||
|
source: git
|
||||||
|
platforms: [StandaloneLinux64]
|
||||||
@@ -66,10 +66,6 @@ namespace UnityBuilderAction
|
|||||||
var buildProfile = AssetDatabase.LoadAssetAtPath<BuildProfile>(buildProfilePath)
|
var buildProfile = AssetDatabase.LoadAssetAtPath<BuildProfile>(buildProfilePath)
|
||||||
?? throw new Exception("Build profile file not found at path: " + buildProfilePath);
|
?? throw new Exception("Build profile file not found at path: " + buildProfilePath);
|
||||||
|
|
||||||
#if !BUILD_PROFILE_LOADED
|
|
||||||
throw new Exception("Build profile's define symbol not present before script execution; shouldn't happen");
|
|
||||||
#endif // BUILD_PROFILE_LOADED
|
|
||||||
|
|
||||||
// no need to set active profile, as already set by `-activeBuildProfile` CLI argument
|
// no need to set active profile, as already set by `-activeBuildProfile` CLI argument
|
||||||
// BuildProfile.SetActiveBuildProfile(buildProfile);
|
// BuildProfile.SetActiveBuildProfile(buildProfile);
|
||||||
Debug.Log($"build profile: {buildProfile.name}");
|
Debug.Log($"build profile: {buildProfile.name}");
|
||||||
|
|||||||
@@ -115,6 +115,7 @@ namespace UnityBuilderAction.Input
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if UNITY_6000_0_OR_NEWER
|
||||||
private static void SetDebugSymbols(string enumValueName)
|
private static void SetDebugSymbols(string enumValueName)
|
||||||
{
|
{
|
||||||
// UnityEditor.Android.UserBuildSettings and Unity.Android.Types.DebugSymbolLevel are part of the Unity Android module.
|
// UnityEditor.Android.UserBuildSettings and Unity.Android.Types.DebugSymbolLevel are part of the Unity Android module.
|
||||||
@@ -144,5 +145,6 @@ namespace UnityBuilderAction.Input
|
|||||||
}
|
}
|
||||||
levelProp.SetValue(null, enumValue);
|
levelProp.SetValue(null, enumValue);
|
||||||
}
|
}
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
268273
dist/index.js
generated
vendored
268273
dist/index.js
generated
vendored
File diff suppressed because one or more lines are too long
2
dist/index.js.map
generated
vendored
2
dist/index.js.map
generated
vendored
File diff suppressed because one or more lines are too long
18584
dist/licenses.txt
generated
vendored
18584
dist/licenses.txt
generated
vendored
File diff suppressed because it is too large
Load Diff
74
dist/platforms/mac/steps/activate.sh
vendored
74
dist/platforms/mac/steps/activate.sh
vendored
@@ -4,21 +4,69 @@
|
|||||||
echo "Changing to \"$ACTIVATE_LICENSE_PATH\" directory."
|
echo "Changing to \"$ACTIVATE_LICENSE_PATH\" directory."
|
||||||
pushd "$ACTIVATE_LICENSE_PATH"
|
pushd "$ACTIVATE_LICENSE_PATH"
|
||||||
|
|
||||||
echo "Requesting activation"
|
if [[ -n "$UNITY_SERIAL" && -n "$UNITY_EMAIL" && -n "$UNITY_PASSWORD" ]]; then
|
||||||
|
#
|
||||||
|
# SERIAL LICENSE MODE
|
||||||
|
#
|
||||||
|
# This will activate unity, using the serial activation process.
|
||||||
|
#
|
||||||
|
|
||||||
# Activate license
|
echo "Requesting activation"
|
||||||
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/MacOS/Unity \
|
|
||||||
-logFile - \
|
|
||||||
-batchmode \
|
|
||||||
-nographics \
|
|
||||||
-quit \
|
|
||||||
-serial "$UNITY_SERIAL" \
|
|
||||||
-username "$UNITY_EMAIL" \
|
|
||||||
-password "$UNITY_PASSWORD" \
|
|
||||||
-projectPath "$ACTIVATE_LICENSE_PATH"
|
|
||||||
|
|
||||||
# Store the exit code from the verify command
|
# Activate license
|
||||||
UNITY_EXIT_CODE=$?
|
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/MacOS/Unity \
|
||||||
|
-logFile - \
|
||||||
|
-batchmode \
|
||||||
|
-nographics \
|
||||||
|
-quit \
|
||||||
|
-serial "$UNITY_SERIAL" \
|
||||||
|
-username "$UNITY_EMAIL" \
|
||||||
|
-password "$UNITY_PASSWORD" \
|
||||||
|
-projectPath "$ACTIVATE_LICENSE_PATH"
|
||||||
|
|
||||||
|
# Store the exit code from the verify command
|
||||||
|
UNITY_EXIT_CODE=$?
|
||||||
|
|
||||||
|
elif [[ -n "$UNITY_LICENSING_SERVER" ]]; then
|
||||||
|
#
|
||||||
|
# Custom Unity License Server
|
||||||
|
#
|
||||||
|
echo "Adding licensing server config"
|
||||||
|
mkdir -p "$UNITY_LICENSE_PATH/config/"
|
||||||
|
cp "$ACTION_FOLDER/unity-config/services-config.json" "$UNITY_LICENSE_PATH/config/services-config.json"
|
||||||
|
|
||||||
|
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/Frameworks/UnityLicensingClient.app/Contents/MacOS/Unity.Licensing.Client \
|
||||||
|
--acquire-floating > license.txt
|
||||||
|
|
||||||
|
# Store the exit code from the verify command
|
||||||
|
UNITY_EXIT_CODE=$?
|
||||||
|
|
||||||
|
if [ $UNITY_EXIT_CODE -eq 0 ]; then
|
||||||
|
PARSEDFILE=$(grep -oE '\"[^"]*\"' < license.txt | tr -d '"')
|
||||||
|
export FLOATING_LICENSE
|
||||||
|
FLOATING_LICENSE=$(sed -n 2p <<< "$PARSEDFILE")
|
||||||
|
FLOATING_LICENSE_TIMEOUT=$(sed -n 4p <<< "$PARSEDFILE")
|
||||||
|
|
||||||
|
echo "Acquired floating license: \"$FLOATING_LICENSE\" with timeout $FLOATING_LICENSE_TIMEOUT"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
#
|
||||||
|
# NO LICENSE ACTIVATION STRATEGY MATCHED
|
||||||
|
#
|
||||||
|
# This will exit since no activation strategies could be matched.
|
||||||
|
#
|
||||||
|
echo "License activation strategy could not be determined."
|
||||||
|
echo ""
|
||||||
|
echo "Visit https://game.ci/docs/github/activation for more"
|
||||||
|
echo "details on how to set up one of the possible activation strategies."
|
||||||
|
|
||||||
|
echo "::error ::No valid license activation strategy could be determined. Make sure to provide UNITY_EMAIL, UNITY_PASSWORD, and either a UNITY_SERIAL \
|
||||||
|
or UNITY_LICENSE. Otherwise please use UNITY_LICENSING_SERVER. See more info at https://game.ci/docs/github/activation"
|
||||||
|
|
||||||
|
# Immediately exit as no UNITY_EXIT_CODE can be derived.
|
||||||
|
exit 1;
|
||||||
|
|
||||||
|
fi
|
||||||
|
|
||||||
#
|
#
|
||||||
# Display information about the result
|
# Display information about the result
|
||||||
|
|||||||
2
dist/platforms/mac/steps/build.sh
vendored
2
dist/platforms/mac/steps/build.sh
vendored
@@ -149,8 +149,6 @@ echo ""
|
|||||||
$( [ "${MANUAL_EXIT}" == "true" ] || echo "-quit" ) \
|
$( [ "${MANUAL_EXIT}" == "true" ] || echo "-quit" ) \
|
||||||
-batchmode \
|
-batchmode \
|
||||||
$( [ "${ENABLE_GPU}" == "true" ] || echo "-nographics" ) \
|
$( [ "${ENABLE_GPU}" == "true" ] || echo "-nographics" ) \
|
||||||
-username "$UNITY_EMAIL" \
|
|
||||||
-password "$UNITY_PASSWORD" \
|
|
||||||
-customBuildName "$BUILD_NAME" \
|
-customBuildName "$BUILD_NAME" \
|
||||||
-projectPath "$UNITY_PROJECT_PATH" \
|
-projectPath "$UNITY_PROJECT_PATH" \
|
||||||
$( [ -z "$BUILD_PROFILE" ] && echo "-buildTarget $BUILD_TARGET") \
|
$( [ -z "$BUILD_PROFILE" ] && echo "-buildTarget $BUILD_TARGET") \
|
||||||
|
|||||||
32
dist/platforms/mac/steps/return_license.sh
vendored
32
dist/platforms/mac/steps/return_license.sh
vendored
@@ -4,15 +4,29 @@
|
|||||||
echo "Changing to \"$ACTIVATE_LICENSE_PATH\" directory."
|
echo "Changing to \"$ACTIVATE_LICENSE_PATH\" directory."
|
||||||
pushd "$ACTIVATE_LICENSE_PATH"
|
pushd "$ACTIVATE_LICENSE_PATH"
|
||||||
|
|
||||||
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/MacOS/Unity \
|
if [[ -n "$UNITY_LICENSING_SERVER" ]]; then
|
||||||
-logFile - \
|
#
|
||||||
-batchmode \
|
# Return any floating license used.
|
||||||
-nographics \
|
#
|
||||||
-quit \
|
echo "Returning floating license: \"$FLOATING_LICENSE\""
|
||||||
-username "$UNITY_EMAIL" \
|
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/Frameworks/UnityLicensingClient.app/Contents/MacOS/Unity.Licensing.Client \
|
||||||
-password "$UNITY_PASSWORD" \
|
--return-floating "$FLOATING_LICENSE"
|
||||||
-returnlicense \
|
elif [[ -n "$UNITY_SERIAL" ]]; then
|
||||||
-projectPath "$ACTIVATE_LICENSE_PATH"
|
#
|
||||||
|
# SERIAL LICENSE MODE
|
||||||
|
#
|
||||||
|
# This will return the license that is currently in use.
|
||||||
|
#
|
||||||
|
/Applications/Unity/Hub/Editor/$UNITY_VERSION/Unity.app/Contents/MacOS/Unity \
|
||||||
|
-logFile - \
|
||||||
|
-batchmode \
|
||||||
|
-nographics \
|
||||||
|
-quit \
|
||||||
|
-username "$UNITY_EMAIL" \
|
||||||
|
-password "$UNITY_PASSWORD" \
|
||||||
|
-returnlicense \
|
||||||
|
-projectPath "$ACTIVATE_LICENSE_PATH"
|
||||||
|
fi
|
||||||
|
|
||||||
# Return to previous working directory
|
# Return to previous working directory
|
||||||
popd
|
popd
|
||||||
|
|||||||
14
dist/platforms/ubuntu/steps/activate.sh
vendored
14
dist/platforms/ubuntu/steps/activate.sh
vendored
@@ -68,14 +68,18 @@ elif [[ -n "$UNITY_LICENSING_SERVER" ]]; then
|
|||||||
echo "Adding licensing server config"
|
echo "Adding licensing server config"
|
||||||
|
|
||||||
/opt/unity/Editor/Data/Resources/Licensing/Client/Unity.Licensing.Client --acquire-floating > license.txt #is this accessible in a env variable?
|
/opt/unity/Editor/Data/Resources/Licensing/Client/Unity.Licensing.Client --acquire-floating > license.txt #is this accessible in a env variable?
|
||||||
PARSEDFILE=$(grep -oP '\".*?\"' < license.txt | tr -d '"')
|
|
||||||
export FLOATING_LICENSE
|
|
||||||
FLOATING_LICENSE=$(sed -n 2p <<< "$PARSEDFILE")
|
|
||||||
FLOATING_LICENSE_TIMEOUT=$(sed -n 4p <<< "$PARSEDFILE")
|
|
||||||
|
|
||||||
echo "Acquired floating license: \"$FLOATING_LICENSE\" with timeout $FLOATING_LICENSE_TIMEOUT"
|
|
||||||
# Store the exit code from the verify command
|
# Store the exit code from the verify command
|
||||||
UNITY_EXIT_CODE=$?
|
UNITY_EXIT_CODE=$?
|
||||||
|
|
||||||
|
if [ $UNITY_EXIT_CODE -eq 0 ]; then
|
||||||
|
PARSEDFILE=$(grep -oP '\".*?\"' < license.txt | tr -d '"')
|
||||||
|
export FLOATING_LICENSE
|
||||||
|
FLOATING_LICENSE=$(sed -n 2p <<< "$PARSEDFILE")
|
||||||
|
FLOATING_LICENSE_TIMEOUT=$(sed -n 4p <<< "$PARSEDFILE")
|
||||||
|
|
||||||
|
echo "Acquired floating license: \"$FLOATING_LICENSE\" with timeout $FLOATING_LICENSE_TIMEOUT"
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
#
|
#
|
||||||
# NO LICENSE ACTIVATION STRATEGY MATCHED
|
# NO LICENSE ACTIVATION STRATEGY MATCHED
|
||||||
|
|||||||
3
dist/platforms/windows/entrypoint.ps1
vendored
3
dist/platforms/windows/entrypoint.ps1
vendored
@@ -18,6 +18,9 @@ regsvr32 C:\ProgramData\Microsoft\VisualStudio\Setup\x64\Microsoft.VisualStudio.
|
|||||||
# Kill the regsvr process
|
# Kill the regsvr process
|
||||||
Get-Process -Name regsvr32 | ForEach-Object { Stop-Process -Id $_.Id -Force }
|
Get-Process -Name regsvr32 | ForEach-Object { Stop-Process -Id $_.Id -Force }
|
||||||
|
|
||||||
|
# Install Visual C++ 2013 Redistributables
|
||||||
|
. "c:\steps\install_vcredist13.ps1"
|
||||||
|
|
||||||
# Setup Git Credentials
|
# Setup Git Credentials
|
||||||
. "c:\steps\set_gitcredential.ps1"
|
. "c:\steps\set_gitcredential.ps1"
|
||||||
|
|
||||||
|
|||||||
11
dist/platforms/windows/install_vcredist13.ps1
vendored
Normal file
11
dist/platforms/windows/install_vcredist13.ps1
vendored
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
# For some reason, Unity is failing in github actions windows runners
|
||||||
|
# due to missing Visual C++ 2013 redistributables.
|
||||||
|
# This script downloads and installs the required redistributables.
|
||||||
|
Write-Output ""
|
||||||
|
Write-Output "#########################################################"
|
||||||
|
Write-Output "# Installing Visual C++ Redistributables (2013) #"
|
||||||
|
Write-Output "#########################################################"
|
||||||
|
Write-Output ""
|
||||||
|
|
||||||
|
|
||||||
|
choco install vcredist2013 -y --no-progress
|
||||||
11
jest.ci.config.js
Normal file
11
jest.ci.config.js
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
const base = require('./jest.config.js');
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
...base,
|
||||||
|
forceExit: true,
|
||||||
|
detectOpenHandles: true,
|
||||||
|
testTimeout: 120000,
|
||||||
|
maxWorkers: 1,
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
@@ -25,8 +25,6 @@ module.exports = {
|
|||||||
// An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
|
// An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
|
||||||
modulePathIgnorePatterns: ['<rootDir>/lib/', '<rootDir>/dist/'],
|
modulePathIgnorePatterns: ['<rootDir>/lib/', '<rootDir>/dist/'],
|
||||||
|
|
||||||
// Files that will be run before Jest is loaded to set globals like fetch
|
// Use jest.setup.js to polyfill fetch for all tests
|
||||||
setupFiles: ['<rootDir>/src/jest.globals.ts'],
|
setupFiles: ['<rootDir>/jest.setup.js'],
|
||||||
// A list of paths to modules that run some code to configure or set up the testing framework after the environment is ready
|
|
||||||
setupFilesAfterEnv: ['<rootDir>/src/jest.setup.ts'],
|
|
||||||
};
|
};
|
||||||
|
|||||||
2
jest.setup.js
Normal file
2
jest.setup.js
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
const fetch = require('node-fetch');
|
||||||
|
global.fetch = fetch;
|
||||||
32
package.json
32
package.json
@@ -11,18 +11,8 @@
|
|||||||
"build": "yarn && tsc && ncc build lib --source-map --license licenses.txt",
|
"build": "yarn && tsc && ncc build lib --source-map --license licenses.txt",
|
||||||
"lint": "prettier --check \"src/**/*.{js,ts}\" && eslint src/**/*.ts",
|
"lint": "prettier --check \"src/**/*.{js,ts}\" && eslint src/**/*.ts",
|
||||||
"format": "prettier --write \"src/**/*.{js,ts}\"",
|
"format": "prettier --write \"src/**/*.{js,ts}\"",
|
||||||
"cli": "yarn ts-node src/index.ts -m cli",
|
|
||||||
"gcp-secrets-tests": "cross-env providerStrategy=aws cloudRunnerTests=true inputPullCommand=\"gcp-secret-manager\" populateOverride=true pullInputList=UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD yarn test -i -t \"cloud runner\"",
|
|
||||||
"gcp-secrets-cli": "cross-env cloudRunnerTests=true USE_IL2CPP=false inputPullCommand=\"gcp-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
|
|
||||||
"aws-secrets-cli": "cross-env cloudRunnerTests=true inputPullCommand=\"aws-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
|
|
||||||
"cli-aws": "cross-env providerStrategy=aws yarn run test-cli",
|
|
||||||
"cli-k8s": "cross-env providerStrategy=k8s yarn run test-cli",
|
|
||||||
"test-cli": "cross-env cloudRunnerTests=true yarn ts-node src/index.ts -m cli --projectPath test-project",
|
|
||||||
"test": "jest",
|
"test": "jest",
|
||||||
"test-i": "cross-env cloudRunnerTests=true yarn test -i -t \"cloud runner\"",
|
"test:ci": "jest --config=jest.ci.config.js --runInBand"
|
||||||
"test-i-*": "yarn run test-i-aws && yarn run test-i-k8s",
|
|
||||||
"test-i-aws": "cross-env cloudRunnerTests=true providerStrategy=aws yarn test -i -t \"cloud runner\"",
|
|
||||||
"test-i-k8s": "cross-env cloudRunnerTests=true providerStrategy=k8s yarn test -i -t \"cloud runner\""
|
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=18.x"
|
"node": ">=18.x"
|
||||||
@@ -32,35 +22,20 @@
|
|||||||
"@actions/core": "^1.11.1",
|
"@actions/core": "^1.11.1",
|
||||||
"@actions/exec": "^1.1.1",
|
"@actions/exec": "^1.1.1",
|
||||||
"@actions/github": "^6.0.0",
|
"@actions/github": "^6.0.0",
|
||||||
"@aws-sdk/client-cloudformation": "^3.777.0",
|
|
||||||
"@aws-sdk/client-cloudwatch-logs": "^3.777.0",
|
|
||||||
"@aws-sdk/client-ecs": "^3.778.0",
|
|
||||||
"@aws-sdk/client-kinesis": "^3.777.0",
|
|
||||||
"@aws-sdk/client-s3": "^3.779.0",
|
|
||||||
"@kubernetes/client-node": "^0.16.3",
|
|
||||||
"@octokit/core": "^5.1.0",
|
|
||||||
"async-wait-until": "^2.0.12",
|
|
||||||
"aws-sdk": "^2.1081.0",
|
|
||||||
"base-64": "^1.0.0",
|
|
||||||
"commander": "^9.0.0",
|
"commander": "^9.0.0",
|
||||||
"commander-ts": "^0.2.0",
|
"commander-ts": "^0.2.0",
|
||||||
"graphql": "^16.11.0",
|
|
||||||
"kubernetes-client": "^9.0.0",
|
|
||||||
"md5": "^2.3.0",
|
"md5": "^2.3.0",
|
||||||
"nanoid": "^3.3.1",
|
|
||||||
"reflect-metadata": "^0.1.13",
|
"reflect-metadata": "^0.1.13",
|
||||||
|
"nanoid": "^3.3.1",
|
||||||
"semver": "^7.5.2",
|
"semver": "^7.5.2",
|
||||||
"ts-md5": "^1.3.1",
|
"ts-md5": "^1.3.1",
|
||||||
"unity-changeset": "^3.0.0",
|
"unity-changeset": "^3.1.0",
|
||||||
"uuid": "^9.0.0",
|
|
||||||
"yaml": "^2.2.2"
|
"yaml": "^2.2.2"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/base-64": "^1.0.0",
|
|
||||||
"@types/jest": "^27.4.1",
|
"@types/jest": "^27.4.1",
|
||||||
"@types/node": "^17.0.23",
|
"@types/node": "^17.0.23",
|
||||||
"@types/semver": "^7.3.9",
|
"@types/semver": "^7.3.9",
|
||||||
"@types/uuid": "^9.0.0",
|
|
||||||
"@typescript-eslint/parser": "4.8.1",
|
"@typescript-eslint/parser": "4.8.1",
|
||||||
"@vercel/ncc": "^0.36.1",
|
"@vercel/ncc": "^0.36.1",
|
||||||
"cross-env": "^7.0.3",
|
"cross-env": "^7.0.3",
|
||||||
@@ -75,6 +50,7 @@
|
|||||||
"jest-fail-on-console": "^3.0.2",
|
"jest-fail-on-console": "^3.0.2",
|
||||||
"js-yaml": "^4.1.0",
|
"js-yaml": "^4.1.0",
|
||||||
"lefthook": "^1.6.1",
|
"lefthook": "^1.6.1",
|
||||||
|
"node-fetch": "2",
|
||||||
"prettier": "^2.5.1",
|
"prettier": "^2.5.1",
|
||||||
"ts-jest": "^27.1.3",
|
"ts-jest": "^27.1.3",
|
||||||
"ts-node": "10.8.1",
|
"ts-node": "10.8.1",
|
||||||
|
|||||||
262
src/index-plugin-features.test.ts
Normal file
262
src/index-plugin-features.test.ts
Normal file
@@ -0,0 +1,262 @@
|
|||||||
|
/**
|
||||||
|
* Integration wiring tests for the plugin lifecycle in index.ts
|
||||||
|
*
|
||||||
|
* These tests verify that:
|
||||||
|
* - The plugin lifecycle hooks are called in the correct order
|
||||||
|
* - Plugin canHandleBuild() controls the execution path
|
||||||
|
* - fallbackToLocal is handled correctly
|
||||||
|
* - When no plugin is installed, local builds still work
|
||||||
|
* - When providerStrategy is non-local without a plugin, an error is thrown
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { BuildParameters } from './model';
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Mock plugin
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const mockPlugin = {
|
||||||
|
initialize: jest.fn().mockResolvedValue(undefined),
|
||||||
|
canHandleBuild: jest.fn().mockReturnValue(false),
|
||||||
|
handleBuild: jest.fn().mockResolvedValue({ exitCode: 0 }),
|
||||||
|
beforeLocalBuild: jest.fn().mockResolvedValue(undefined),
|
||||||
|
afterLocalBuild: jest.fn().mockResolvedValue(undefined),
|
||||||
|
handlePostBuild: jest.fn().mockResolvedValue(undefined),
|
||||||
|
};
|
||||||
|
|
||||||
|
const mockLoadOrchestratorPlugin = jest.fn().mockResolvedValue(mockPlugin);
|
||||||
|
|
||||||
|
jest.mock('./model/orchestrator-plugin', () => ({
|
||||||
|
loadOrchestratorPlugin: mockLoadOrchestratorPlugin,
|
||||||
|
}));
|
||||||
|
|
||||||
|
jest.mock('@actions/core');
|
||||||
|
jest.mock('./model', () => ({
|
||||||
|
Action: {
|
||||||
|
checkCompatibility: jest.fn(),
|
||||||
|
workspace: '/workspace',
|
||||||
|
actionFolder: '/action',
|
||||||
|
},
|
||||||
|
BuildParameters: {
|
||||||
|
create: jest.fn(),
|
||||||
|
},
|
||||||
|
Cache: {
|
||||||
|
verify: jest.fn(),
|
||||||
|
},
|
||||||
|
Docker: {
|
||||||
|
run: jest.fn().mockResolvedValue(0),
|
||||||
|
},
|
||||||
|
ImageTag: jest.fn().mockImplementation(() => ({
|
||||||
|
toString: () => 'mock-image:latest',
|
||||||
|
})),
|
||||||
|
Output: {
|
||||||
|
setBuildVersion: jest.fn().mockResolvedValue(''),
|
||||||
|
setAndroidVersionCode: jest.fn().mockResolvedValue(''),
|
||||||
|
setEngineExitCode: jest.fn().mockResolvedValue(''),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
jest.mock('./model/cli/cli', () => ({
|
||||||
|
Cli: {
|
||||||
|
InitCliMode: jest.fn().mockReturnValue(false),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
jest.mock('./model/mac-builder', () => ({
|
||||||
|
__esModule: true,
|
||||||
|
default: {
|
||||||
|
run: jest.fn().mockResolvedValue(0),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
jest.mock('./model/platform-setup', () => ({
|
||||||
|
__esModule: true,
|
||||||
|
default: {
|
||||||
|
setup: jest.fn().mockResolvedValue(''),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
const mockedBuildParametersCreate = BuildParameters.create as jest.Mock;
|
||||||
|
|
||||||
|
function createMockBuildParameters(overrides: Record<string, any> = {}) {
|
||||||
|
return {
|
||||||
|
providerStrategy: 'local',
|
||||||
|
targetPlatform: 'StandaloneLinux64',
|
||||||
|
editorVersion: '2021.3.1f1',
|
||||||
|
buildVersion: '1.0.0',
|
||||||
|
androidVersionCode: '1',
|
||||||
|
projectPath: '.',
|
||||||
|
branch: 'main',
|
||||||
|
runnerTempPath: '/tmp',
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runIndex(overrides: Record<string, any> = {}): Promise<void> {
|
||||||
|
mockedBuildParametersCreate.mockResolvedValue(createMockBuildParameters(overrides));
|
||||||
|
|
||||||
|
return new Promise<void>((resolve) => {
|
||||||
|
jest.isolateModules(() => {
|
||||||
|
require('./index');
|
||||||
|
});
|
||||||
|
setTimeout(resolve, 100);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Tests
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
describe('index.ts plugin lifecycle wiring', () => {
|
||||||
|
const originalPlatform = process.platform;
|
||||||
|
const originalEnvironment = { ...process.env };
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
jest.clearAllMocks();
|
||||||
|
process.env.GITHUB_WORKSPACE = '/workspace';
|
||||||
|
Object.defineProperty(process, 'platform', { value: 'linux' });
|
||||||
|
|
||||||
|
// Reset plugin to default behavior
|
||||||
|
mockPlugin.canHandleBuild.mockReturnValue(false);
|
||||||
|
mockPlugin.handleBuild.mockResolvedValue({ exitCode: 0 });
|
||||||
|
mockLoadOrchestratorPlugin.mockResolvedValue(mockPlugin);
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
Object.defineProperty(process, 'platform', { value: originalPlatform });
|
||||||
|
process.env = { ...originalEnvironment };
|
||||||
|
});
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Local build with plugin
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
describe('local build with plugin installed', () => {
|
||||||
|
it('should call lifecycle hooks in order: initialize → beforeLocalBuild → [build] → afterLocalBuild → handlePostBuild', async () => {
|
||||||
|
const callOrder: string[] = [];
|
||||||
|
mockPlugin.initialize.mockImplementation(async () => callOrder.push('initialize'));
|
||||||
|
mockPlugin.beforeLocalBuild.mockImplementation(async () => callOrder.push('beforeLocalBuild'));
|
||||||
|
mockPlugin.afterLocalBuild.mockImplementation(async () => callOrder.push('afterLocalBuild'));
|
||||||
|
mockPlugin.handlePostBuild.mockImplementation(async () => callOrder.push('handlePostBuild'));
|
||||||
|
|
||||||
|
await runIndex();
|
||||||
|
|
||||||
|
expect(callOrder).toEqual(['initialize', 'beforeLocalBuild', 'afterLocalBuild', 'handlePostBuild']);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should pass buildParameters and workspace to initialize', async () => {
|
||||||
|
await runIndex({ targetPlatform: 'WebGL' });
|
||||||
|
|
||||||
|
expect(mockPlugin.initialize).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({ targetPlatform: 'WebGL' }),
|
||||||
|
'/workspace',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should pass workspace to beforeLocalBuild', async () => {
|
||||||
|
await runIndex();
|
||||||
|
|
||||||
|
expect(mockPlugin.beforeLocalBuild).toHaveBeenCalledWith('/workspace');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should pass workspace and exit code to afterLocalBuild', async () => {
|
||||||
|
await runIndex();
|
||||||
|
|
||||||
|
expect(mockPlugin.afterLocalBuild).toHaveBeenCalledWith('/workspace', 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should pass exit code to handlePostBuild', async () => {
|
||||||
|
await runIndex();
|
||||||
|
|
||||||
|
expect(mockPlugin.handlePostBuild).toHaveBeenCalledWith(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Plugin handles build entirely
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
describe('plugin handles build (canHandleBuild = true)', () => {
|
||||||
|
it('should call handleBuild instead of Docker.run', async () => {
|
||||||
|
const { Docker } = require('./model');
|
||||||
|
mockPlugin.canHandleBuild.mockReturnValue(true);
|
||||||
|
mockPlugin.handleBuild.mockResolvedValue({ exitCode: 0 });
|
||||||
|
|
||||||
|
await runIndex();
|
||||||
|
|
||||||
|
expect(mockPlugin.handleBuild).toHaveBeenCalledWith('mock-image:latest');
|
||||||
|
expect(Docker.run).not.toHaveBeenCalled();
|
||||||
|
expect(mockPlugin.beforeLocalBuild).not.toHaveBeenCalled();
|
||||||
|
expect(mockPlugin.afterLocalBuild).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should still call handlePostBuild after handleBuild', async () => {
|
||||||
|
mockPlugin.canHandleBuild.mockReturnValue(true);
|
||||||
|
mockPlugin.handleBuild.mockResolvedValue({ exitCode: 0 });
|
||||||
|
|
||||||
|
await runIndex();
|
||||||
|
|
||||||
|
expect(mockPlugin.handlePostBuild).toHaveBeenCalledWith(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Fallback to local
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
describe('fallback to local build', () => {
|
||||||
|
it('should do a local build when handleBuild returns fallbackToLocal', async () => {
|
||||||
|
const { Docker } = require('./model');
|
||||||
|
mockPlugin.canHandleBuild.mockReturnValue(true);
|
||||||
|
mockPlugin.handleBuild.mockResolvedValue({ exitCode: -1, fallbackToLocal: true });
|
||||||
|
|
||||||
|
await runIndex();
|
||||||
|
|
||||||
|
expect(mockPlugin.handleBuild).toHaveBeenCalled();
|
||||||
|
expect(mockPlugin.beforeLocalBuild).toHaveBeenCalled();
|
||||||
|
expect(Docker.run).toHaveBeenCalled();
|
||||||
|
expect(mockPlugin.afterLocalBuild).toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// No plugin installed
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
describe('no plugin installed', () => {
|
||||||
|
it('should build locally without errors when providerStrategy is local', async () => {
|
||||||
|
const { Docker } = require('./model');
|
||||||
|
mockLoadOrchestratorPlugin.mockResolvedValue(undefined);
|
||||||
|
|
||||||
|
await runIndex({ providerStrategy: 'local' });
|
||||||
|
|
||||||
|
expect(Docker.run).toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should error when providerStrategy is non-local and no plugin', async () => {
|
||||||
|
const core = require('@actions/core');
|
||||||
|
mockLoadOrchestratorPlugin.mockResolvedValue(undefined);
|
||||||
|
|
||||||
|
await runIndex({ providerStrategy: 'aws' });
|
||||||
|
|
||||||
|
expect(core.setFailed).toHaveBeenCalledWith(expect.stringContaining('requires @game-ci/orchestrator'));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// canHandleBuild = false with non-local provider
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
describe('plugin installed but canHandleBuild returns false with non-local provider', () => {
|
||||||
|
it('should error when providerStrategy is non-local', async () => {
|
||||||
|
const core = require('@actions/core');
|
||||||
|
mockPlugin.canHandleBuild.mockReturnValue(false);
|
||||||
|
|
||||||
|
await runIndex({ providerStrategy: 'aws' });
|
||||||
|
|
||||||
|
// The plugin is initialized but says it can't handle the build,
|
||||||
|
// and providerStrategy is not local, so it falls to the error case
|
||||||
|
expect(core.setFailed).toHaveBeenCalledWith(expect.stringContaining('requires @game-ci/orchestrator'));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
63
src/index.ts
63
src/index.ts
@@ -1,8 +1,9 @@
|
|||||||
import * as core from '@actions/core';
|
import * as core from '@actions/core';
|
||||||
import { Action, BuildParameters, Cache, CloudRunner, Docker, ImageTag, Output } from './model';
|
import { Action, BuildParameters, Cache, Docker, ImageTag, Output } from './model';
|
||||||
import { Cli } from './model/cli/cli';
|
import { Cli } from './model/cli/cli';
|
||||||
import MacBuilder from './model/mac-builder';
|
import MacBuilder from './model/mac-builder';
|
||||||
import PlatformSetup from './model/platform-setup';
|
import PlatformSetup from './model/platform-setup';
|
||||||
|
import { loadOrchestratorPlugin, OrchestratorPlugin } from './model/orchestrator-plugin';
|
||||||
|
|
||||||
async function runMain() {
|
async function runMain() {
|
||||||
try {
|
try {
|
||||||
@@ -15,33 +16,39 @@ async function runMain() {
|
|||||||
Cache.verify();
|
Cache.verify();
|
||||||
|
|
||||||
const { workspace, actionFolder } = Action;
|
const { workspace, actionFolder } = Action;
|
||||||
|
|
||||||
const buildParameters = await BuildParameters.create();
|
const buildParameters = await BuildParameters.create();
|
||||||
const baseImage = new ImageTag(buildParameters);
|
const baseImage = new ImageTag(buildParameters);
|
||||||
|
|
||||||
|
// Load orchestrator plugin (optional — only needed for remote builds and plugin features)
|
||||||
|
const plugin = await loadOrchestratorPlugin();
|
||||||
|
await plugin?.initialize(buildParameters, workspace);
|
||||||
|
|
||||||
let exitCode = -1;
|
let exitCode = -1;
|
||||||
|
|
||||||
if (buildParameters.providerStrategy === 'local') {
|
if (plugin?.canHandleBuild()) {
|
||||||
core.info('Building locally');
|
// Plugin handles the build entirely (remote providers, hot runner, test workflows)
|
||||||
await PlatformSetup.setup(buildParameters, actionFolder);
|
const result = await plugin.handleBuild(baseImage.toString());
|
||||||
exitCode =
|
|
||||||
process.platform === 'darwin'
|
exitCode = result.fallbackToLocal
|
||||||
? await MacBuilder.run(actionFolder)
|
? await runLocalBuild(buildParameters, baseImage, workspace, actionFolder, plugin)
|
||||||
: await Docker.run(baseImage.toString(), {
|
: result.exitCode;
|
||||||
workspace,
|
} else if (buildParameters.providerStrategy === 'local') {
|
||||||
actionFolder,
|
exitCode = await runLocalBuild(buildParameters, baseImage, workspace, actionFolder, plugin);
|
||||||
...buildParameters,
|
|
||||||
});
|
|
||||||
} else {
|
} else {
|
||||||
await CloudRunner.run(buildParameters, baseImage.toString());
|
throw new Error(
|
||||||
exitCode = 0;
|
`Provider strategy "${buildParameters.providerStrategy}" requires @game-ci/orchestrator. ` +
|
||||||
|
'Install it via the game-ci/orchestrator action, or use providerStrategy=local.',
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set output
|
// Set core outputs
|
||||||
await Output.setBuildVersion(buildParameters.buildVersion);
|
await Output.setBuildVersion(buildParameters.buildVersion);
|
||||||
await Output.setAndroidVersionCode(buildParameters.androidVersionCode);
|
await Output.setAndroidVersionCode(buildParameters.androidVersionCode);
|
||||||
await Output.setEngineExitCode(exitCode);
|
await Output.setEngineExitCode(exitCode);
|
||||||
|
|
||||||
|
// Plugin handles post-build (artifacts, archiving, retention)
|
||||||
|
await plugin?.handlePostBuild(exitCode);
|
||||||
|
|
||||||
if (exitCode !== 0) {
|
if (exitCode !== 0) {
|
||||||
core.setFailed(`Build failed with exit code ${exitCode}`);
|
core.setFailed(`Build failed with exit code ${exitCode}`);
|
||||||
}
|
}
|
||||||
@@ -50,4 +57,28 @@ async function runMain() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function runLocalBuild(
|
||||||
|
buildParameters: BuildParameters,
|
||||||
|
baseImage: ImageTag,
|
||||||
|
workspace: string,
|
||||||
|
actionFolder: string,
|
||||||
|
plugin?: OrchestratorPlugin,
|
||||||
|
): Promise<number> {
|
||||||
|
await plugin?.beforeLocalBuild(workspace);
|
||||||
|
|
||||||
|
await PlatformSetup.setup(buildParameters, actionFolder);
|
||||||
|
const exitCode =
|
||||||
|
process.platform === 'darwin'
|
||||||
|
? await MacBuilder.run(actionFolder)
|
||||||
|
: await Docker.run(baseImage.toString(), {
|
||||||
|
workspace,
|
||||||
|
actionFolder,
|
||||||
|
...buildParameters,
|
||||||
|
});
|
||||||
|
|
||||||
|
await plugin?.afterLocalBuild(workspace, exitCode);
|
||||||
|
|
||||||
|
return exitCode;
|
||||||
|
}
|
||||||
|
|
||||||
runMain();
|
runMain();
|
||||||
|
|||||||
@@ -1,29 +0,0 @@
|
|||||||
// Integration test for exercising real GitHub check creation and updates.
|
|
||||||
import CloudRunner from '../model/cloud-runner/cloud-runner';
|
|
||||||
import UnityVersioning from '../model/unity-versioning';
|
|
||||||
import GitHub from '../model/github';
|
|
||||||
import { TIMEOUT_INFINITE, createParameters } from '../test-utils/cloud-runner-test-helpers';
|
|
||||||
|
|
||||||
const runIntegration = process.env.RUN_GITHUB_INTEGRATION_TESTS === 'true';
|
|
||||||
const describeOrSkip = runIntegration ? describe : describe.skip;
|
|
||||||
|
|
||||||
describeOrSkip('Cloud Runner Github Checks Integration', () => {
|
|
||||||
it(
|
|
||||||
'creates and updates a real GitHub check',
|
|
||||||
async () => {
|
|
||||||
const buildParameter = await createParameters({
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.read('test-project'),
|
|
||||||
asyncCloudRunner: `true`,
|
|
||||||
githubChecks: `true`,
|
|
||||||
});
|
|
||||||
await CloudRunner.setup(buildParameter);
|
|
||||||
const checkId = await GitHub.createGitHubCheck(`integration create`);
|
|
||||||
expect(checkId).not.toEqual('');
|
|
||||||
await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `integration`);
|
|
||||||
await GitHub.updateGitHubCheck(`2 ${new Date().toISOString()}`, `integration`, `success`, `completed`);
|
|
||||||
},
|
|
||||||
TIMEOUT_INFINITE,
|
|
||||||
);
|
|
||||||
});
|
|
||||||
@@ -219,5 +219,10 @@ describe('BuildParameters', () => {
|
|||||||
jest.spyOn(Input, 'customParameters', 'get').mockReturnValue(mockValue);
|
jest.spyOn(Input, 'customParameters', 'get').mockReturnValue(mockValue);
|
||||||
await expect(BuildParameters.create()).resolves.toEqual(expect.objectContaining({ customParameters: mockValue }));
|
await expect(BuildParameters.create()).resolves.toEqual(expect.objectContaining({ customParameters: mockValue }));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it.each([true, false])('returns the flag for useHostNetwork when %s', async (mockValue) => {
|
||||||
|
jest.spyOn(Input, 'useHostNetwork', 'get').mockReturnValue(mockValue);
|
||||||
|
await expect(BuildParameters.create()).resolves.toEqual(expect.objectContaining({ useHostNetwork: mockValue }));
|
||||||
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,7 +1,5 @@
|
|||||||
import { customAlphabet } from 'nanoid';
|
import { customAlphabet } from 'nanoid';
|
||||||
import AndroidVersioning from './android-versioning';
|
import AndroidVersioning from './android-versioning';
|
||||||
import CloudRunnerConstants from './cloud-runner/options/cloud-runner-constants';
|
|
||||||
import CloudRunnerBuildGuid from './cloud-runner/options/cloud-runner-guid';
|
|
||||||
import Input from './input';
|
import Input from './input';
|
||||||
import Platform from './platform';
|
import Platform from './platform';
|
||||||
import UnityVersioning from './unity-versioning';
|
import UnityVersioning from './unity-versioning';
|
||||||
@@ -10,8 +8,6 @@ import { GitRepoReader } from './input-readers/git-repo';
|
|||||||
import { GithubCliReader } from './input-readers/github-cli';
|
import { GithubCliReader } from './input-readers/github-cli';
|
||||||
import { Cli } from './cli/cli';
|
import { Cli } from './cli/cli';
|
||||||
import GitHub from './github';
|
import GitHub from './github';
|
||||||
import CloudRunnerOptions from './cloud-runner/options/cloud-runner-options';
|
|
||||||
import CloudRunner from './cloud-runner/cloud-runner';
|
|
||||||
import * as core from '@actions/core';
|
import * as core from '@actions/core';
|
||||||
|
|
||||||
class BuildParameters {
|
class BuildParameters {
|
||||||
@@ -51,55 +47,27 @@ class BuildParameters {
|
|||||||
public containerRegistryImageVersion!: string;
|
public containerRegistryImageVersion!: string;
|
||||||
|
|
||||||
public customParameters!: string;
|
public customParameters!: string;
|
||||||
|
public useHostNetwork!: boolean;
|
||||||
public sshAgent!: string;
|
public sshAgent!: string;
|
||||||
public sshPublicKeysDirectoryPath!: string;
|
public sshPublicKeysDirectoryPath!: string;
|
||||||
public providerStrategy!: string;
|
public providerStrategy!: string;
|
||||||
public gitPrivateToken!: string;
|
public gitPrivateToken!: string;
|
||||||
public awsStackName!: string;
|
|
||||||
public kubeConfig!: string;
|
|
||||||
public containerMemory!: string;
|
|
||||||
public containerCpu!: string;
|
|
||||||
public kubeVolumeSize!: string;
|
|
||||||
public kubeVolume!: string;
|
|
||||||
public kubeStorageClass!: string;
|
|
||||||
public runAsHostUser!: string;
|
public runAsHostUser!: string;
|
||||||
public chownFilesTo!: string;
|
public chownFilesTo!: string;
|
||||||
public commandHooks!: string;
|
|
||||||
public pullInputList!: string[];
|
|
||||||
public inputPullCommand!: string;
|
|
||||||
public cacheKey!: string;
|
|
||||||
|
|
||||||
public postBuildContainerHooks!: string;
|
|
||||||
public preBuildContainerHooks!: string;
|
|
||||||
public customJob!: string;
|
|
||||||
public runNumber!: string;
|
public runNumber!: string;
|
||||||
public branch!: string;
|
public branch!: string;
|
||||||
public githubRepo!: string;
|
public githubRepo!: string;
|
||||||
public gitSha!: string;
|
public gitSha!: string;
|
||||||
public logId!: string;
|
public logId!: string;
|
||||||
public buildGuid!: string;
|
public buildGuid!: string;
|
||||||
public cloudRunnerBranch!: string;
|
|
||||||
public cloudRunnerDebug!: boolean | undefined;
|
|
||||||
public buildPlatform!: string | undefined;
|
public buildPlatform!: string | undefined;
|
||||||
public isCliMode!: boolean;
|
public isCliMode!: boolean;
|
||||||
public maxRetainedWorkspaces!: number;
|
|
||||||
public useLargePackages!: boolean;
|
|
||||||
public useCompressionStrategy!: boolean;
|
|
||||||
public garbageMaxAge!: number;
|
|
||||||
public githubChecks!: boolean;
|
|
||||||
public asyncWorkflow!: boolean;
|
|
||||||
public githubCheckId!: string;
|
|
||||||
public finalHooks!: string[];
|
|
||||||
public skipLfs!: boolean;
|
|
||||||
public skipCache!: boolean;
|
|
||||||
public cacheUnityInstallationOnMac!: boolean;
|
public cacheUnityInstallationOnMac!: boolean;
|
||||||
public unityHubVersionOnMac!: string;
|
public unityHubVersionOnMac!: string;
|
||||||
public dockerWorkspacePath!: string;
|
public dockerWorkspacePath!: string;
|
||||||
|
|
||||||
public static shouldUseRetainedWorkspaceMode(buildParameters: BuildParameters) {
|
|
||||||
return buildParameters.maxRetainedWorkspaces > 0 && CloudRunner.lockedWorkspace !== ``;
|
|
||||||
}
|
|
||||||
|
|
||||||
static async create(): Promise<BuildParameters> {
|
static async create(): Promise<BuildParameters> {
|
||||||
const buildFile = this.parseBuildFile(Input.buildName, Input.targetPlatform, Input.androidExportType);
|
const buildFile = this.parseBuildFile(Input.buildName, Input.targetPlatform, Input.androidExportType);
|
||||||
const editorVersion = UnityVersioning.determineUnityVersion(Input.projectPath, Input.unityVersion);
|
const editorVersion = UnityVersioning.determineUnityVersion(Input.projectPath, Input.unityVersion);
|
||||||
@@ -144,6 +112,8 @@ class BuildParameters {
|
|||||||
core.setSecret(`${unitySerial.slice(0, -4)}XXXX`);
|
core.setSecret(`${unitySerial.slice(0, -4)}XXXX`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const providerStrategy = Input.getInput('providerStrategy') || (Cli.isCliMode ? 'aws' : 'local');
|
||||||
|
|
||||||
return {
|
return {
|
||||||
editorVersion,
|
editorVersion,
|
||||||
customImage: Input.customImage,
|
customImage: Input.customImage,
|
||||||
@@ -172,6 +142,7 @@ class BuildParameters {
|
|||||||
androidExportType: Input.androidExportType,
|
androidExportType: Input.androidExportType,
|
||||||
androidSymbolType: androidSymbolExportType,
|
androidSymbolType: androidSymbolExportType,
|
||||||
customParameters: Input.customParameters,
|
customParameters: Input.customParameters,
|
||||||
|
useHostNetwork: Input.useHostNetwork,
|
||||||
sshAgent: Input.sshAgent,
|
sshAgent: Input.sshAgent,
|
||||||
sshPublicKeysDirectoryPath: Input.sshPublicKeysDirectoryPath,
|
sshPublicKeysDirectoryPath: Input.sshPublicKeysDirectoryPath,
|
||||||
gitPrivateToken: Input.gitPrivateToken ?? (await GithubCliReader.GetGitHubAuthToken()),
|
gitPrivateToken: Input.gitPrivateToken ?? (await GithubCliReader.GetGitHubAuthToken()),
|
||||||
@@ -182,41 +153,18 @@ class BuildParameters {
|
|||||||
dockerIsolationMode: Input.dockerIsolationMode,
|
dockerIsolationMode: Input.dockerIsolationMode,
|
||||||
containerRegistryRepository: Input.containerRegistryRepository,
|
containerRegistryRepository: Input.containerRegistryRepository,
|
||||||
containerRegistryImageVersion: Input.containerRegistryImageVersion,
|
containerRegistryImageVersion: Input.containerRegistryImageVersion,
|
||||||
providerStrategy: CloudRunnerOptions.providerStrategy,
|
providerStrategy,
|
||||||
buildPlatform: CloudRunnerOptions.buildPlatform,
|
buildPlatform: providerStrategy !== 'local' ? 'linux' : process.platform,
|
||||||
kubeConfig: CloudRunnerOptions.kubeConfig,
|
|
||||||
containerMemory: CloudRunnerOptions.containerMemory,
|
|
||||||
containerCpu: CloudRunnerOptions.containerCpu,
|
|
||||||
kubeVolumeSize: CloudRunnerOptions.kubeVolumeSize,
|
|
||||||
kubeVolume: CloudRunnerOptions.kubeVolume,
|
|
||||||
postBuildContainerHooks: CloudRunnerOptions.postBuildContainerHooks,
|
|
||||||
preBuildContainerHooks: CloudRunnerOptions.preBuildContainerHooks,
|
|
||||||
customJob: CloudRunnerOptions.customJob,
|
|
||||||
runNumber: Input.runNumber,
|
runNumber: Input.runNumber,
|
||||||
branch: Input.branch.replace('/head', '') || (await GitRepoReader.GetBranch()),
|
branch: Input.branch.replace('/head', '') || (await GitRepoReader.GetBranch()),
|
||||||
cloudRunnerBranch: CloudRunnerOptions.cloudRunnerBranch.split('/').reverse()[0],
|
|
||||||
cloudRunnerDebug: CloudRunnerOptions.cloudRunnerDebug,
|
|
||||||
githubRepo: (Input.githubRepo ?? (await GitRepoReader.GetRemote())) || 'game-ci/unity-builder',
|
githubRepo: (Input.githubRepo ?? (await GitRepoReader.GetRemote())) || 'game-ci/unity-builder',
|
||||||
isCliMode: Cli.isCliMode,
|
|
||||||
awsStackName: CloudRunnerOptions.awsStackName,
|
|
||||||
gitSha: Input.gitSha,
|
gitSha: Input.gitSha,
|
||||||
logId: customAlphabet(CloudRunnerConstants.alphabet, 9)(),
|
logId: customAlphabet('0123456789abcdefghijklmnopqrstuvwxyz', 9)(),
|
||||||
buildGuid: CloudRunnerBuildGuid.generateGuid(Input.runNumber, Input.targetPlatform),
|
buildGuid: `${Input.runNumber}-${Input.targetPlatform.toLowerCase().replace('standalone', '')}-${customAlphabet(
|
||||||
commandHooks: CloudRunnerOptions.commandHooks,
|
'0123456789abcdefghijklmnopqrstuvwxyz',
|
||||||
inputPullCommand: CloudRunnerOptions.inputPullCommand,
|
4,
|
||||||
pullInputList: CloudRunnerOptions.pullInputList,
|
)()}`,
|
||||||
kubeStorageClass: CloudRunnerOptions.kubeStorageClass,
|
isCliMode: Cli.isCliMode,
|
||||||
cacheKey: CloudRunnerOptions.cacheKey,
|
|
||||||
maxRetainedWorkspaces: Number.parseInt(CloudRunnerOptions.maxRetainedWorkspaces),
|
|
||||||
useLargePackages: CloudRunnerOptions.useLargePackages,
|
|
||||||
useCompressionStrategy: CloudRunnerOptions.useCompressionStrategy,
|
|
||||||
garbageMaxAge: CloudRunnerOptions.garbageMaxAge,
|
|
||||||
githubChecks: CloudRunnerOptions.githubChecks,
|
|
||||||
asyncWorkflow: CloudRunnerOptions.asyncCloudRunner,
|
|
||||||
githubCheckId: CloudRunnerOptions.githubCheckId,
|
|
||||||
finalHooks: CloudRunnerOptions.finalHooks,
|
|
||||||
skipLfs: CloudRunnerOptions.skipLfs,
|
|
||||||
skipCache: CloudRunnerOptions.skipCache,
|
|
||||||
cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac,
|
cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac,
|
||||||
unityHubVersionOnMac: Input.unityHubVersionOnMac,
|
unityHubVersionOnMac: Input.unityHubVersionOnMac,
|
||||||
dockerWorkspacePath: Input.dockerWorkspacePath,
|
dockerWorkspacePath: Input.dockerWorkspacePath,
|
||||||
|
|||||||
@@ -1,15 +1,8 @@
|
|||||||
import { Command } from 'commander-ts';
|
import { Command } from 'commander-ts';
|
||||||
import { BuildParameters, CloudRunner, ImageTag, Input } from '..';
|
import { Input } from '..';
|
||||||
import * as core from '@actions/core';
|
import * as core from '@actions/core';
|
||||||
import { ActionYamlReader } from '../input-readers/action-yaml';
|
import { ActionYamlReader } from '../input-readers/action-yaml';
|
||||||
import CloudRunnerLogger from '../cloud-runner/services/core/cloud-runner-logger';
|
|
||||||
import CloudRunnerQueryOverride from '../cloud-runner/options/cloud-runner-query-override';
|
|
||||||
import { CliFunction, CliFunctionsRepository } from './cli-functions-repository';
|
import { CliFunction, CliFunctionsRepository } from './cli-functions-repository';
|
||||||
import { Caching } from '../cloud-runner/remote-client/caching';
|
|
||||||
import { LfsHashing } from '../cloud-runner/services/utility/lfs-hashing';
|
|
||||||
import { RemoteClient } from '../cloud-runner/remote-client';
|
|
||||||
import CloudRunnerOptionsReader from '../cloud-runner/options/cloud-runner-options-reader';
|
|
||||||
import GitHub from '../github';
|
|
||||||
import { OptionValues } from 'commander';
|
import { OptionValues } from 'commander';
|
||||||
import { InputKey } from '../input';
|
import { InputKey } from '../input';
|
||||||
|
|
||||||
@@ -30,14 +23,13 @@ export class Cli {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public static InitCliMode() {
|
public static InitCliMode() {
|
||||||
CliFunctionsRepository.PushCliFunctionSource(RemoteClient);
|
|
||||||
CliFunctionsRepository.PushCliFunctionSource(Caching);
|
|
||||||
CliFunctionsRepository.PushCliFunctionSource(LfsHashing);
|
|
||||||
const program = new Command();
|
const program = new Command();
|
||||||
program.version('0.0.1');
|
program.version('0.0.1');
|
||||||
|
|
||||||
const properties = CloudRunnerOptionsReader.GetProperties();
|
|
||||||
const actionYamlReader: ActionYamlReader = new ActionYamlReader();
|
const actionYamlReader: ActionYamlReader = new ActionYamlReader();
|
||||||
|
const properties = Object.getOwnPropertyNames(Input).filter(
|
||||||
|
(p) => p !== 'length' && p !== 'prototype' && p !== 'name',
|
||||||
|
);
|
||||||
for (const element of properties) {
|
for (const element of properties) {
|
||||||
program.option(`--${element} <${element}>`, actionYamlReader.GetActionYamlValue(element));
|
program.option(`--${element} <${element}>`, actionYamlReader.GetActionYamlValue(element));
|
||||||
}
|
}
|
||||||
@@ -53,6 +45,11 @@ export class Cli {
|
|||||||
program.option('--artifactName <artifactName>', 'caching artifact name');
|
program.option('--artifactName <artifactName>', 'caching artifact name');
|
||||||
program.option('--select <select>', 'select a particular resource');
|
program.option('--select <select>', 'select a particular resource');
|
||||||
program.option('--logFile <logFile>', 'output to log file (log stream only)');
|
program.option('--logFile <logFile>', 'output to log file (log stream only)');
|
||||||
|
program.option('--profilePath <profilePath>', 'path to submodule profile YAML');
|
||||||
|
program.option('--variantPath <variantPath>', 'path to submodule variant YAML');
|
||||||
|
program.option('--agentPath <agentPath>', 'path to custom LFS transfer agent');
|
||||||
|
program.option('--agentArgs <agentArgs>', 'arguments for custom LFS transfer agent');
|
||||||
|
program.option('--storagePaths <storagePaths>', 'semicolon-separated storage paths for LFS agent');
|
||||||
program.parse(process.argv);
|
program.parse(process.argv);
|
||||||
Cli.options = program.opts();
|
Cli.options = program.opts();
|
||||||
|
|
||||||
@@ -60,26 +57,15 @@ export class Cli {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static async RunCli(): Promise<void> {
|
static async RunCli(): Promise<void> {
|
||||||
GitHub.githubInputEnabled = false;
|
|
||||||
if (Cli.options!['populateOverride'] === `true`) {
|
|
||||||
await CloudRunnerQueryOverride.PopulateQueryOverrideInput();
|
|
||||||
}
|
|
||||||
if (Cli.options!['logInput']) {
|
|
||||||
Cli.logInput();
|
|
||||||
}
|
|
||||||
const results = CliFunctionsRepository.GetCliFunctions(Cli.options?.mode);
|
const results = CliFunctionsRepository.GetCliFunctions(Cli.options?.mode);
|
||||||
CloudRunnerLogger.log(`Entrypoint: ${results.key}`);
|
if (!results) {
|
||||||
|
throw new Error(
|
||||||
|
`Unknown CLI mode: ${Cli.options?.mode}. Orchestrator CLI features require @game-ci/orchestrator.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
core.info(`Entrypoint: ${results.key}`);
|
||||||
Cli.options!.versioning = 'None';
|
Cli.options!.versioning = 'None';
|
||||||
|
|
||||||
CloudRunner.buildParameters = await BuildParameters.create();
|
|
||||||
CloudRunner.buildParameters.buildGuid = process.env.BUILD_GUID || ``;
|
|
||||||
CloudRunnerLogger.log(`Build Params:
|
|
||||||
${JSON.stringify(CloudRunner.buildParameters, undefined, 4)}
|
|
||||||
`);
|
|
||||||
CloudRunner.lockedWorkspace = process.env.LOCKED_WORKSPACE || ``;
|
|
||||||
CloudRunnerLogger.log(`Locked Workspace: ${CloudRunner.lockedWorkspace}`);
|
|
||||||
await CloudRunner.setup(CloudRunner.buildParameters);
|
|
||||||
|
|
||||||
return await results.target[results.propertyKey](Cli.options);
|
return await results.target[results.propertyKey](Cli.options);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -87,7 +73,9 @@ export class Cli {
|
|||||||
private static logInput() {
|
private static logInput() {
|
||||||
core.info(`\n`);
|
core.info(`\n`);
|
||||||
core.info(`INPUT:`);
|
core.info(`INPUT:`);
|
||||||
const properties = CloudRunnerOptionsReader.GetProperties();
|
const properties = Object.getOwnPropertyNames(Input).filter(
|
||||||
|
(p) => p !== 'length' && p !== 'prototype' && p !== 'name',
|
||||||
|
);
|
||||||
for (const element of properties) {
|
for (const element of properties) {
|
||||||
if (
|
if (
|
||||||
element in Input &&
|
element in Input &&
|
||||||
@@ -103,73 +91,4 @@ export class Cli {
|
|||||||
}
|
}
|
||||||
core.info(`\n`);
|
core.info(`\n`);
|
||||||
}
|
}
|
||||||
|
|
||||||
@CliFunction(`cli-build`, `runs a cloud runner build`)
|
|
||||||
public static async CLIBuild(): Promise<string> {
|
|
||||||
const buildParameter = await BuildParameters.create();
|
|
||||||
const baseImage = new ImageTag(buildParameter);
|
|
||||||
|
|
||||||
return (await CloudRunner.run(buildParameter, baseImage.toString())).BuildResults;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliFunction(`async-workflow`, `runs a cloud runner build`)
|
|
||||||
public static async asyncronousWorkflow(): Promise<string> {
|
|
||||||
const buildParameter = await BuildParameters.create();
|
|
||||||
const baseImage = new ImageTag(buildParameter);
|
|
||||||
await CloudRunner.setup(buildParameter);
|
|
||||||
|
|
||||||
return (await CloudRunner.run(buildParameter, baseImage.toString())).BuildResults;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliFunction(`checks-update`, `runs a cloud runner build`)
|
|
||||||
public static async checksUpdate() {
|
|
||||||
const buildParameter = await BuildParameters.create();
|
|
||||||
|
|
||||||
await CloudRunner.setup(buildParameter);
|
|
||||||
const input = JSON.parse(process.env.CHECKS_UPDATE || ``);
|
|
||||||
core.info(`Checks Update ${process.env.CHECKS_UPDATE}`);
|
|
||||||
if (input.mode === `create`) {
|
|
||||||
throw new Error(`Not supported: only use update`);
|
|
||||||
} else if (input.mode === `update`) {
|
|
||||||
await GitHub.updateGitHubCheckRequest(input.data);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliFunction(`garbage-collect`, `runs garbage collection`)
|
|
||||||
public static async GarbageCollect(): Promise<string> {
|
|
||||||
const buildParameter = await BuildParameters.create();
|
|
||||||
|
|
||||||
await CloudRunner.setup(buildParameter);
|
|
||||||
|
|
||||||
return await CloudRunner.Provider.garbageCollect(``, false, 0, false, false);
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliFunction(`list-resources`, `lists active resources`)
|
|
||||||
public static async ListResources(): Promise<string[]> {
|
|
||||||
const buildParameter = await BuildParameters.create();
|
|
||||||
|
|
||||||
await CloudRunner.setup(buildParameter);
|
|
||||||
const result = await CloudRunner.Provider.listResources();
|
|
||||||
CloudRunnerLogger.log(JSON.stringify(result, undefined, 4));
|
|
||||||
|
|
||||||
return result.map((x) => x.Name);
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliFunction(`list-worfklow`, `lists running workflows`)
|
|
||||||
public static async ListWorfklow(): Promise<string[]> {
|
|
||||||
const buildParameter = await BuildParameters.create();
|
|
||||||
|
|
||||||
await CloudRunner.setup(buildParameter);
|
|
||||||
|
|
||||||
return (await CloudRunner.Provider.listWorkflow()).map((x) => x.Name);
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliFunction(`watch`, `follows logs of a running workflow`)
|
|
||||||
public static async Watch(): Promise<string> {
|
|
||||||
const buildParameter = await BuildParameters.create();
|
|
||||||
|
|
||||||
await CloudRunner.setup(buildParameter);
|
|
||||||
|
|
||||||
return await CloudRunner.Provider.watchWorkflow();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,187 +0,0 @@
|
|||||||
import AwsBuildPlatform from './providers/aws';
|
|
||||||
import { BuildParameters, Input } from '..';
|
|
||||||
import Kubernetes from './providers/k8s';
|
|
||||||
import CloudRunnerLogger from './services/core/cloud-runner-logger';
|
|
||||||
import { CloudRunnerStepParameters } from './options/cloud-runner-step-parameters';
|
|
||||||
import { WorkflowCompositionRoot } from './workflows/workflow-composition-root';
|
|
||||||
import { CloudRunnerError } from './error/cloud-runner-error';
|
|
||||||
import { TaskParameterSerializer } from './services/core/task-parameter-serializer';
|
|
||||||
import * as core from '@actions/core';
|
|
||||||
import CloudRunnerSecret from './options/cloud-runner-secret';
|
|
||||||
import { ProviderInterface } from './providers/provider-interface';
|
|
||||||
import CloudRunnerEnvironmentVariable from './options/cloud-runner-environment-variable';
|
|
||||||
import TestCloudRunner from './providers/test';
|
|
||||||
import LocalCloudRunner from './providers/local';
|
|
||||||
import LocalDockerCloudRunner from './providers/docker';
|
|
||||||
import GitHub from '../github';
|
|
||||||
import SharedWorkspaceLocking from './services/core/shared-workspace-locking';
|
|
||||||
import { FollowLogStreamService } from './services/core/follow-log-stream-service';
|
|
||||||
import CloudRunnerResult from './services/core/cloud-runner-result';
|
|
||||||
|
|
||||||
class CloudRunner {
|
|
||||||
public static Provider: ProviderInterface;
|
|
||||||
public static buildParameters: BuildParameters;
|
|
||||||
private static defaultSecrets: CloudRunnerSecret[];
|
|
||||||
private static cloudRunnerEnvironmentVariables: CloudRunnerEnvironmentVariable[];
|
|
||||||
static lockedWorkspace: string = ``;
|
|
||||||
public static readonly retainedWorkspacePrefix: string = `retained-workspace`;
|
|
||||||
public static get isCloudRunnerEnvironment() {
|
|
||||||
return process.env[`GITHUB_ACTIONS`] !== `true`;
|
|
||||||
}
|
|
||||||
public static get isCloudRunnerAsyncEnvironment() {
|
|
||||||
return process.env[`ASYNC_WORKFLOW`] === `true`;
|
|
||||||
}
|
|
||||||
public static async setup(buildParameters: BuildParameters) {
|
|
||||||
CloudRunnerLogger.setup();
|
|
||||||
CloudRunnerLogger.log(`Setting up cloud runner`);
|
|
||||||
CloudRunner.buildParameters = buildParameters;
|
|
||||||
if (CloudRunner.buildParameters.githubCheckId === ``) {
|
|
||||||
CloudRunner.buildParameters.githubCheckId = await GitHub.createGitHubCheck(CloudRunner.buildParameters.buildGuid);
|
|
||||||
}
|
|
||||||
CloudRunner.setupSelectedBuildPlatform();
|
|
||||||
CloudRunner.defaultSecrets = TaskParameterSerializer.readDefaultSecrets();
|
|
||||||
CloudRunner.cloudRunnerEnvironmentVariables =
|
|
||||||
TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameters);
|
|
||||||
if (GitHub.githubInputEnabled) {
|
|
||||||
const buildParameterPropertyNames = Object.getOwnPropertyNames(buildParameters);
|
|
||||||
for (const element of CloudRunner.cloudRunnerEnvironmentVariables) {
|
|
||||||
// CloudRunnerLogger.log(`Cloud Runner output ${Input.ToEnvVarFormat(element.name)} = ${element.value}`);
|
|
||||||
core.setOutput(Input.ToEnvVarFormat(element.name), element.value);
|
|
||||||
}
|
|
||||||
for (const element of buildParameterPropertyNames) {
|
|
||||||
// CloudRunnerLogger.log(`Cloud Runner output ${Input.ToEnvVarFormat(element)} = ${buildParameters[element]}`);
|
|
||||||
core.setOutput(Input.ToEnvVarFormat(element), buildParameters[element]);
|
|
||||||
}
|
|
||||||
core.setOutput(
|
|
||||||
Input.ToEnvVarFormat(`buildArtifact`),
|
|
||||||
`build-${CloudRunner.buildParameters.buildGuid}.tar${
|
|
||||||
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
|
||||||
}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
FollowLogStreamService.Reset();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static setupSelectedBuildPlatform() {
|
|
||||||
CloudRunnerLogger.log(`Cloud Runner platform selected ${CloudRunner.buildParameters.providerStrategy}`);
|
|
||||||
switch (CloudRunner.buildParameters.providerStrategy) {
|
|
||||||
case 'k8s':
|
|
||||||
CloudRunner.Provider = new Kubernetes(CloudRunner.buildParameters);
|
|
||||||
break;
|
|
||||||
case 'aws':
|
|
||||||
CloudRunner.Provider = new AwsBuildPlatform(CloudRunner.buildParameters);
|
|
||||||
break;
|
|
||||||
case 'test':
|
|
||||||
CloudRunner.Provider = new TestCloudRunner();
|
|
||||||
break;
|
|
||||||
case 'local-docker':
|
|
||||||
CloudRunner.Provider = new LocalDockerCloudRunner();
|
|
||||||
break;
|
|
||||||
case 'local-system':
|
|
||||||
CloudRunner.Provider = new LocalCloudRunner();
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static async run(buildParameters: BuildParameters, baseImage: string) {
|
|
||||||
if (baseImage.includes(`undefined`)) {
|
|
||||||
throw new Error(`baseImage is undefined`);
|
|
||||||
}
|
|
||||||
await CloudRunner.setup(buildParameters);
|
|
||||||
await CloudRunner.Provider.setupWorkflow(
|
|
||||||
CloudRunner.buildParameters.buildGuid,
|
|
||||||
CloudRunner.buildParameters,
|
|
||||||
CloudRunner.buildParameters.branch,
|
|
||||||
CloudRunner.defaultSecrets,
|
|
||||||
);
|
|
||||||
try {
|
|
||||||
if (buildParameters.maxRetainedWorkspaces > 0) {
|
|
||||||
CloudRunner.lockedWorkspace = SharedWorkspaceLocking.NewWorkspaceName();
|
|
||||||
|
|
||||||
const result = await SharedWorkspaceLocking.GetLockedWorkspace(
|
|
||||||
CloudRunner.lockedWorkspace,
|
|
||||||
CloudRunner.buildParameters.buildGuid,
|
|
||||||
CloudRunner.buildParameters,
|
|
||||||
);
|
|
||||||
|
|
||||||
if (result) {
|
|
||||||
CloudRunnerLogger.logLine(`Using retained workspace ${CloudRunner.lockedWorkspace}`);
|
|
||||||
CloudRunner.cloudRunnerEnvironmentVariables = [
|
|
||||||
...CloudRunner.cloudRunnerEnvironmentVariables,
|
|
||||||
{ name: `LOCKED_WORKSPACE`, value: CloudRunner.lockedWorkspace },
|
|
||||||
];
|
|
||||||
} else {
|
|
||||||
CloudRunnerLogger.log(`Max retained workspaces reached ${buildParameters.maxRetainedWorkspaces}`);
|
|
||||||
buildParameters.maxRetainedWorkspaces = 0;
|
|
||||||
CloudRunner.lockedWorkspace = ``;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await CloudRunner.updateStatusWithBuildParameters();
|
|
||||||
const output = await new WorkflowCompositionRoot().run(
|
|
||||||
new CloudRunnerStepParameters(
|
|
||||||
baseImage,
|
|
||||||
CloudRunner.cloudRunnerEnvironmentVariables,
|
|
||||||
CloudRunner.defaultSecrets,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
await CloudRunner.Provider.cleanupWorkflow(
|
|
||||||
CloudRunner.buildParameters,
|
|
||||||
CloudRunner.buildParameters.branch,
|
|
||||||
CloudRunner.defaultSecrets,
|
|
||||||
);
|
|
||||||
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
|
|
||||||
if (buildParameters.asyncWorkflow && this.isCloudRunnerEnvironment && this.isCloudRunnerAsyncEnvironment) {
|
|
||||||
await GitHub.updateGitHubCheck(CloudRunner.buildParameters.buildGuid, `success`, `success`, `completed`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
|
|
||||||
const workspace = CloudRunner.lockedWorkspace || ``;
|
|
||||||
await SharedWorkspaceLocking.ReleaseWorkspace(
|
|
||||||
workspace,
|
|
||||||
CloudRunner.buildParameters.buildGuid,
|
|
||||||
CloudRunner.buildParameters,
|
|
||||||
);
|
|
||||||
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(workspace, CloudRunner.buildParameters);
|
|
||||||
if (isLocked) {
|
|
||||||
throw new Error(
|
|
||||||
`still locked after releasing ${await SharedWorkspaceLocking.GetAllLocksForWorkspace(
|
|
||||||
workspace,
|
|
||||||
buildParameters,
|
|
||||||
)}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
CloudRunner.lockedWorkspace = ``;
|
|
||||||
}
|
|
||||||
|
|
||||||
await GitHub.triggerWorkflowOnComplete(CloudRunner.buildParameters.finalHooks);
|
|
||||||
|
|
||||||
if (buildParameters.constantGarbageCollection) {
|
|
||||||
CloudRunner.Provider.garbageCollect(``, true, buildParameters.garbageMaxAge, true, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
return new CloudRunnerResult(buildParameters, output, true, true, false);
|
|
||||||
} catch (error: any) {
|
|
||||||
CloudRunnerLogger.log(JSON.stringify(error, undefined, 4));
|
|
||||||
await GitHub.updateGitHubCheck(
|
|
||||||
CloudRunner.buildParameters.buildGuid,
|
|
||||||
`Failed - Error ${error?.message || error}`,
|
|
||||||
`failure`,
|
|
||||||
`completed`,
|
|
||||||
);
|
|
||||||
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
|
|
||||||
await CloudRunnerError.handleException(error, CloudRunner.buildParameters, CloudRunner.defaultSecrets);
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static async updateStatusWithBuildParameters() {
|
|
||||||
const content = { ...CloudRunner.buildParameters };
|
|
||||||
content.gitPrivateToken = ``;
|
|
||||||
content.unitySerial = ``;
|
|
||||||
content.unityEmail = ``;
|
|
||||||
content.unityPassword = ``;
|
|
||||||
const jsonContent = JSON.stringify(content, undefined, 4);
|
|
||||||
await GitHub.updateGitHubCheck(jsonContent, CloudRunner.buildParameters.buildGuid);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
export default CloudRunner;
|
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
|
||||||
import * as core from '@actions/core';
|
|
||||||
import CloudRunner from '../cloud-runner';
|
|
||||||
import CloudRunnerSecret from '../options/cloud-runner-secret';
|
|
||||||
import BuildParameters from '../../build-parameters';
|
|
||||||
|
|
||||||
export class CloudRunnerError {
|
|
||||||
public static async handleException(error: unknown, buildParameters: BuildParameters, secrets: CloudRunnerSecret[]) {
|
|
||||||
CloudRunnerLogger.error(JSON.stringify(error, undefined, 4));
|
|
||||||
core.setFailed('Cloud Runner failed');
|
|
||||||
if (CloudRunner.Provider !== undefined) {
|
|
||||||
await CloudRunner.Provider.cleanupWorkflow(buildParameters, buildParameters.branch, secrets);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
class CloudRunnerConstants {
|
|
||||||
static alphabet = '0123456789abcdefghijklmnopqrstuvwxyz';
|
|
||||||
}
|
|
||||||
export default CloudRunnerConstants;
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
class CloudRunnerEnvironmentVariable {
|
|
||||||
public name!: string;
|
|
||||||
public value!: string;
|
|
||||||
}
|
|
||||||
export default CloudRunnerEnvironmentVariable;
|
|
||||||
@@ -1,90 +0,0 @@
|
|||||||
import path from 'node:path';
|
|
||||||
import CloudRunnerOptions from './cloud-runner-options';
|
|
||||||
import CloudRunner from '../cloud-runner';
|
|
||||||
import BuildParameters from '../../build-parameters';
|
|
||||||
|
|
||||||
export class CloudRunnerFolders {
|
|
||||||
public static readonly repositoryFolder = 'repo';
|
|
||||||
|
|
||||||
public static ToLinuxFolder(folder: string) {
|
|
||||||
return folder.replace(/\\/g, `/`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
|
|
||||||
|
|
||||||
public static get uniqueCloudRunnerJobFolderAbsolute(): string {
|
|
||||||
return CloudRunner.buildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)
|
|
||||||
? path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.lockedWorkspace)
|
|
||||||
: path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.buildParameters.buildGuid);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get cacheFolderForAllFull(): string {
|
|
||||||
return path.join('/', CloudRunnerFolders.buildVolumeFolder, CloudRunnerFolders.cacheFolder);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get cacheFolderForCacheKeyFull(): string {
|
|
||||||
return path.join(
|
|
||||||
'/',
|
|
||||||
CloudRunnerFolders.buildVolumeFolder,
|
|
||||||
CloudRunnerFolders.cacheFolder,
|
|
||||||
CloudRunner.buildParameters.cacheKey,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get builderPathAbsolute(): string {
|
|
||||||
return path.join(
|
|
||||||
CloudRunnerOptions.useSharedBuilder
|
|
||||||
? `/${CloudRunnerFolders.buildVolumeFolder}`
|
|
||||||
: CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute,
|
|
||||||
`builder`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get repoPathAbsolute(): string {
|
|
||||||
return path.join(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute, CloudRunnerFolders.repositoryFolder);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get projectPathAbsolute(): string {
|
|
||||||
return path.join(CloudRunnerFolders.repoPathAbsolute, CloudRunner.buildParameters.projectPath);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get libraryFolderAbsolute(): string {
|
|
||||||
return path.join(CloudRunnerFolders.projectPathAbsolute, `Library`);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get projectBuildFolderAbsolute(): string {
|
|
||||||
return path.join(CloudRunnerFolders.repoPathAbsolute, CloudRunner.buildParameters.buildPath);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get lfsFolderAbsolute(): string {
|
|
||||||
return path.join(CloudRunnerFolders.repoPathAbsolute, `.git`, `lfs`);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get purgeRemoteCaching(): boolean {
|
|
||||||
return process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get lfsCacheFolderFull() {
|
|
||||||
return path.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `lfs`);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get libraryCacheFolderFull() {
|
|
||||||
return path.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `Library`);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get unityBuilderRepoUrl(): string {
|
|
||||||
return `https://${CloudRunner.buildParameters.gitPrivateToken}@github.com/game-ci/unity-builder.git`;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get targetBuildRepoUrl(): string {
|
|
||||||
return `https://${CloudRunner.buildParameters.gitPrivateToken}@github.com/${CloudRunner.buildParameters.githubRepo}.git`;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get buildVolumeFolder() {
|
|
||||||
return 'data';
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get cacheFolder() {
|
|
||||||
return 'cache';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
import { customAlphabet } from 'nanoid';
|
|
||||||
import CloudRunnerConstants from './cloud-runner-constants';
|
|
||||||
|
|
||||||
class CloudRunnerNamespace {
|
|
||||||
static generateGuid(runNumber: string | number, platform: string) {
|
|
||||||
const nanoid = customAlphabet(CloudRunnerConstants.alphabet, 4);
|
|
||||||
|
|
||||||
return `${runNumber}-${platform.toLowerCase().replace('standalone', '')}-${nanoid()}`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
export default CloudRunnerNamespace;
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
import Input from '../../input';
|
|
||||||
import CloudRunnerOptions from './cloud-runner-options';
|
|
||||||
|
|
||||||
class CloudRunnerOptionsReader {
|
|
||||||
static GetProperties() {
|
|
||||||
return [...Object.getOwnPropertyNames(Input), ...Object.getOwnPropertyNames(CloudRunnerOptions)];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export default CloudRunnerOptionsReader;
|
|
||||||
@@ -1,283 +0,0 @@
|
|||||||
import { Cli } from '../../cli/cli';
|
|
||||||
import CloudRunnerQueryOverride from './cloud-runner-query-override';
|
|
||||||
import GitHub from '../../github';
|
|
||||||
import * as core from '@actions/core';
|
|
||||||
|
|
||||||
class CloudRunnerOptions {
|
|
||||||
// ### ### ###
|
|
||||||
// Input Handling
|
|
||||||
// ### ### ###
|
|
||||||
public static getInput(query: string): string | undefined {
|
|
||||||
if (GitHub.githubInputEnabled) {
|
|
||||||
const coreInput = core.getInput(query);
|
|
||||||
if (coreInput && coreInput !== '') {
|
|
||||||
return coreInput;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const alternativeQuery = CloudRunnerOptions.ToEnvVarFormat(query);
|
|
||||||
|
|
||||||
// Query input sources
|
|
||||||
if (Cli.query(query, alternativeQuery)) {
|
|
||||||
return Cli.query(query, alternativeQuery);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (CloudRunnerQueryOverride.query(query, alternativeQuery)) {
|
|
||||||
return CloudRunnerQueryOverride.query(query, alternativeQuery);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (process.env[query] !== undefined) {
|
|
||||||
return process.env[query];
|
|
||||||
}
|
|
||||||
|
|
||||||
if (alternativeQuery !== query && process.env[alternativeQuery] !== undefined) {
|
|
||||||
return process.env[alternativeQuery];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static ToEnvVarFormat(input: string): string {
|
|
||||||
if (input.toUpperCase() === input) {
|
|
||||||
return input;
|
|
||||||
}
|
|
||||||
|
|
||||||
return input
|
|
||||||
.replace(/([A-Z])/g, ' $1')
|
|
||||||
.trim()
|
|
||||||
.toUpperCase()
|
|
||||||
.replace(/ /g, '_');
|
|
||||||
}
|
|
||||||
|
|
||||||
// ### ### ###
|
|
||||||
// Provider parameters
|
|
||||||
// ### ### ###
|
|
||||||
|
|
||||||
static get region(): string {
|
|
||||||
return CloudRunnerOptions.getInput('region') || 'eu-west-2';
|
|
||||||
}
|
|
||||||
|
|
||||||
// ### ### ###
|
|
||||||
// GitHub parameters
|
|
||||||
// ### ### ###
|
|
||||||
static get githubChecks(): boolean {
|
|
||||||
const value = CloudRunnerOptions.getInput('githubChecks');
|
|
||||||
|
|
||||||
return value === `true` || false;
|
|
||||||
}
|
|
||||||
static get githubCheckId(): string {
|
|
||||||
return CloudRunnerOptions.getInput('githubCheckId') || ``;
|
|
||||||
}
|
|
||||||
|
|
||||||
static get githubOwner(): string {
|
|
||||||
return CloudRunnerOptions.getInput('githubOwner') || CloudRunnerOptions.githubRepo?.split(`/`)[0] || '';
|
|
||||||
}
|
|
||||||
|
|
||||||
static get githubRepoName(): string {
|
|
||||||
return CloudRunnerOptions.getInput('githubRepoName') || CloudRunnerOptions.githubRepo?.split(`/`)[1] || '';
|
|
||||||
}
|
|
||||||
|
|
||||||
static get finalHooks(): string[] {
|
|
||||||
return CloudRunnerOptions.getInput('finalHooks')?.split(',') || [];
|
|
||||||
}
|
|
||||||
|
|
||||||
// ### ### ###
|
|
||||||
// Git syncronization parameters
|
|
||||||
// ### ### ###
|
|
||||||
|
|
||||||
static get githubRepo(): string | undefined {
|
|
||||||
return CloudRunnerOptions.getInput('GITHUB_REPOSITORY') || CloudRunnerOptions.getInput('GITHUB_REPO') || undefined;
|
|
||||||
}
|
|
||||||
static get branch(): string {
|
|
||||||
if (CloudRunnerOptions.getInput(`GITHUB_REF`)) {
|
|
||||||
return (
|
|
||||||
CloudRunnerOptions.getInput(`GITHUB_REF`)?.replace('refs/', '').replace(`head/`, '').replace(`heads/`, '') || ``
|
|
||||||
);
|
|
||||||
} else if (CloudRunnerOptions.getInput('branch')) {
|
|
||||||
return CloudRunnerOptions.getInput('branch') || ``;
|
|
||||||
} else {
|
|
||||||
return '';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ### ### ###
|
|
||||||
// Cloud Runner parameters
|
|
||||||
// ### ### ###
|
|
||||||
|
|
||||||
static get buildPlatform(): string {
|
|
||||||
const input = CloudRunnerOptions.getInput('buildPlatform');
|
|
||||||
if (input && input !== '') {
|
|
||||||
return input;
|
|
||||||
}
|
|
||||||
if (CloudRunnerOptions.providerStrategy !== 'local') {
|
|
||||||
return 'linux';
|
|
||||||
}
|
|
||||||
|
|
||||||
return process.platform;
|
|
||||||
}
|
|
||||||
|
|
||||||
static get cloudRunnerBranch(): string {
|
|
||||||
return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'main';
|
|
||||||
}
|
|
||||||
|
|
||||||
static get providerStrategy(): string {
|
|
||||||
const provider =
|
|
||||||
CloudRunnerOptions.getInput('cloudRunnerCluster') || CloudRunnerOptions.getInput('providerStrategy');
|
|
||||||
if (Cli.isCliMode) {
|
|
||||||
return provider || 'aws';
|
|
||||||
}
|
|
||||||
|
|
||||||
return provider || 'local';
|
|
||||||
}
|
|
||||||
|
|
||||||
static get containerCpu(): string {
|
|
||||||
return CloudRunnerOptions.getInput('containerCpu') || `1024`;
|
|
||||||
}
|
|
||||||
|
|
||||||
static get containerMemory(): string {
|
|
||||||
return CloudRunnerOptions.getInput('containerMemory') || `3072`;
|
|
||||||
}
|
|
||||||
|
|
||||||
static get customJob(): string {
|
|
||||||
return CloudRunnerOptions.getInput('customJob') || '';
|
|
||||||
}
|
|
||||||
|
|
||||||
// ### ### ###
|
|
||||||
// Custom commands from files parameters
|
|
||||||
// ### ### ###
|
|
||||||
|
|
||||||
static get containerHookFiles(): string[] {
|
|
||||||
return CloudRunnerOptions.getInput('containerHookFiles')?.split(`,`) || [];
|
|
||||||
}
|
|
||||||
|
|
||||||
static get commandHookFiles(): string[] {
|
|
||||||
return CloudRunnerOptions.getInput('commandHookFiles')?.split(`,`) || [];
|
|
||||||
}
|
|
||||||
|
|
||||||
// ### ### ###
|
|
||||||
// Custom commands from yaml parameters
|
|
||||||
// ### ### ###
|
|
||||||
|
|
||||||
static get commandHooks(): string {
|
|
||||||
return CloudRunnerOptions.getInput('commandHooks') || '';
|
|
||||||
}
|
|
||||||
|
|
||||||
static get postBuildContainerHooks(): string {
|
|
||||||
return CloudRunnerOptions.getInput('postBuildContainerHooks') || '';
|
|
||||||
}
|
|
||||||
|
|
||||||
static get preBuildContainerHooks(): string {
|
|
||||||
return CloudRunnerOptions.getInput('preBuildContainerHooks') || '';
|
|
||||||
}
|
|
||||||
|
|
||||||
// ### ### ###
|
|
||||||
// Input override handling
|
|
||||||
// ### ### ###
|
|
||||||
|
|
||||||
static get pullInputList(): string[] {
|
|
||||||
return CloudRunnerOptions.getInput('pullInputList')?.split(`,`) || [];
|
|
||||||
}
|
|
||||||
|
|
||||||
static get inputPullCommand(): string {
|
|
||||||
const value = CloudRunnerOptions.getInput('inputPullCommand');
|
|
||||||
|
|
||||||
if (value === 'gcp-secret-manager') {
|
|
||||||
return 'gcloud secrets versions access 1 --secret="{0}"';
|
|
||||||
} else if (value === 'aws-secret-manager') {
|
|
||||||
return 'aws secretsmanager get-secret-value --secret-id {0}';
|
|
||||||
}
|
|
||||||
|
|
||||||
return value || '';
|
|
||||||
}
|
|
||||||
|
|
||||||
// ### ### ###
|
|
||||||
// Aws
|
|
||||||
// ### ### ###
|
|
||||||
|
|
||||||
static get awsStackName() {
|
|
||||||
return CloudRunnerOptions.getInput('awsStackName') || 'game-ci';
|
|
||||||
}
|
|
||||||
|
|
||||||
// ### ### ###
|
|
||||||
// K8s
|
|
||||||
// ### ### ###
|
|
||||||
|
|
||||||
static get kubeConfig(): string {
|
|
||||||
return CloudRunnerOptions.getInput('kubeConfig') || '';
|
|
||||||
}
|
|
||||||
|
|
||||||
static get kubeVolume(): string {
|
|
||||||
return CloudRunnerOptions.getInput('kubeVolume') || '';
|
|
||||||
}
|
|
||||||
|
|
||||||
static get kubeVolumeSize(): string {
|
|
||||||
return CloudRunnerOptions.getInput('kubeVolumeSize') || '25Gi';
|
|
||||||
}
|
|
||||||
|
|
||||||
static get kubeStorageClass(): string {
|
|
||||||
return CloudRunnerOptions.getInput('kubeStorageClass') || '';
|
|
||||||
}
|
|
||||||
|
|
||||||
// ### ### ###
|
|
||||||
// Caching
|
|
||||||
// ### ### ###
|
|
||||||
|
|
||||||
static get cacheKey(): string {
|
|
||||||
return CloudRunnerOptions.getInput('cacheKey') || CloudRunnerOptions.branch;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ### ### ###
|
|
||||||
// Utility Parameters
|
|
||||||
// ### ### ###
|
|
||||||
|
|
||||||
static get cloudRunnerDebug(): boolean {
|
|
||||||
return (
|
|
||||||
CloudRunnerOptions.getInput(`cloudRunnerTests`) === `true` ||
|
|
||||||
CloudRunnerOptions.getInput(`cloudRunnerDebug`) === `true` ||
|
|
||||||
CloudRunnerOptions.getInput(`cloudRunnerDebugTree`) === `true` ||
|
|
||||||
CloudRunnerOptions.getInput(`cloudRunnerDebugEnv`) === `true` ||
|
|
||||||
false
|
|
||||||
);
|
|
||||||
}
|
|
||||||
static get skipLfs(): boolean {
|
|
||||||
return CloudRunnerOptions.getInput(`skipLfs`) === `true`;
|
|
||||||
}
|
|
||||||
static get skipCache(): boolean {
|
|
||||||
return CloudRunnerOptions.getInput(`skipCache`) === `true`;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get asyncCloudRunner(): boolean {
|
|
||||||
return CloudRunnerOptions.getInput('asyncCloudRunner') === 'true';
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get useLargePackages(): boolean {
|
|
||||||
return CloudRunnerOptions.getInput(`useLargePackages`) === `true`;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get useSharedBuilder(): boolean {
|
|
||||||
return CloudRunnerOptions.getInput(`useSharedBuilder`) === `true`;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get useCompressionStrategy(): boolean {
|
|
||||||
return CloudRunnerOptions.getInput(`useCompressionStrategy`) === `true`;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static get useCleanupCron(): boolean {
|
|
||||||
return (CloudRunnerOptions.getInput(`useCleanupCron`) || 'true') === 'true';
|
|
||||||
}
|
|
||||||
|
|
||||||
// ### ### ###
|
|
||||||
// Retained Workspace
|
|
||||||
// ### ### ###
|
|
||||||
|
|
||||||
public static get maxRetainedWorkspaces(): string {
|
|
||||||
return CloudRunnerOptions.getInput(`maxRetainedWorkspaces`) || `0`;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ### ### ###
|
|
||||||
// Garbage Collection
|
|
||||||
// ### ### ###
|
|
||||||
|
|
||||||
static get garbageMaxAge(): number {
|
|
||||||
return Number(CloudRunnerOptions.getInput(`garbageMaxAge`)) || 24;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export default CloudRunnerOptions;
|
|
||||||
@@ -1,67 +0,0 @@
|
|||||||
import Input from '../../input';
|
|
||||||
import { GenericInputReader } from '../../input-readers/generic-input-reader';
|
|
||||||
import CloudRunnerOptions from './cloud-runner-options';
|
|
||||||
|
|
||||||
const formatFunction = (value: string, arguments_: any[]) => {
|
|
||||||
for (const element of arguments_) {
|
|
||||||
value = value.replace(`{${element.key}}`, element.value);
|
|
||||||
}
|
|
||||||
|
|
||||||
return value;
|
|
||||||
};
|
|
||||||
|
|
||||||
class CloudRunnerQueryOverride {
|
|
||||||
static queryOverrides: { [key: string]: string } | undefined;
|
|
||||||
|
|
||||||
// TODO accept premade secret sources or custom secret source definition yamls
|
|
||||||
|
|
||||||
public static query(key: string, alternativeKey: string) {
|
|
||||||
if (CloudRunnerQueryOverride.queryOverrides && CloudRunnerQueryOverride.queryOverrides[key] !== undefined) {
|
|
||||||
return CloudRunnerQueryOverride.queryOverrides[key];
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
CloudRunnerQueryOverride.queryOverrides &&
|
|
||||||
alternativeKey &&
|
|
||||||
CloudRunnerQueryOverride.queryOverrides[alternativeKey] !== undefined
|
|
||||||
) {
|
|
||||||
return CloudRunnerQueryOverride.queryOverrides[alternativeKey];
|
|
||||||
}
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static shouldUseOverride(query: string) {
|
|
||||||
if (CloudRunnerOptions.inputPullCommand !== '') {
|
|
||||||
if (CloudRunnerOptions.pullInputList.length > 0) {
|
|
||||||
const doesInclude =
|
|
||||||
CloudRunnerOptions.pullInputList.includes(query) ||
|
|
||||||
CloudRunnerOptions.pullInputList.includes(Input.ToEnvVarFormat(query));
|
|
||||||
|
|
||||||
return doesInclude ? true : false;
|
|
||||||
} else {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static async queryOverride(query: string) {
|
|
||||||
if (!this.shouldUseOverride(query)) {
|
|
||||||
throw new Error(`Should not be trying to run override query on ${query}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return await GenericInputReader.Run(
|
|
||||||
formatFunction(CloudRunnerOptions.inputPullCommand, [{ key: 0, value: query }]),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async PopulateQueryOverrideInput() {
|
|
||||||
const queries = CloudRunnerOptions.pullInputList;
|
|
||||||
CloudRunnerQueryOverride.queryOverrides = {};
|
|
||||||
for (const element of queries) {
|
|
||||||
if (CloudRunnerQueryOverride.shouldUseOverride(element)) {
|
|
||||||
CloudRunnerQueryOverride.queryOverrides[element] = await CloudRunnerQueryOverride.queryOverride(element);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
export default CloudRunnerQueryOverride;
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
class CloudRunnerSecret {
|
|
||||||
public ParameterKey!: string;
|
|
||||||
public EnvironmentVariable!: string;
|
|
||||||
public ParameterValue!: string;
|
|
||||||
}
|
|
||||||
export default CloudRunnerSecret;
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
export class CloudRunnerStatics {
|
|
||||||
public static readonly logPrefix = `Cloud-Runner`;
|
|
||||||
}
|
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
import CloudRunnerEnvironmentVariable from './cloud-runner-environment-variable';
|
|
||||||
import CloudRunnerSecret from './cloud-runner-secret';
|
|
||||||
|
|
||||||
export class CloudRunnerStepParameters {
|
|
||||||
public image: string;
|
|
||||||
public environment: CloudRunnerEnvironmentVariable[];
|
|
||||||
public secrets: CloudRunnerSecret[];
|
|
||||||
constructor(image: string, environmentVariables: CloudRunnerEnvironmentVariable[], secrets: CloudRunnerSecret[]) {
|
|
||||||
this.image = image;
|
|
||||||
this.environment = environmentVariables;
|
|
||||||
this.secrets = secrets;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,127 +0,0 @@
|
|||||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
|
||||||
import * as core from '@actions/core';
|
|
||||||
import {
|
|
||||||
CloudFormation,
|
|
||||||
CreateStackCommand,
|
|
||||||
CreateStackCommandInput,
|
|
||||||
DescribeStacksCommand,
|
|
||||||
DescribeStacksCommandInput,
|
|
||||||
ListStacksCommand,
|
|
||||||
Parameter,
|
|
||||||
UpdateStackCommand,
|
|
||||||
UpdateStackCommandInput,
|
|
||||||
waitUntilStackCreateComplete,
|
|
||||||
waitUntilStackUpdateComplete,
|
|
||||||
} from '@aws-sdk/client-cloudformation';
|
|
||||||
import { BaseStackFormation } from './cloud-formations/base-stack-formation';
|
|
||||||
import crypto from 'node:crypto';
|
|
||||||
|
|
||||||
export class AWSBaseStack {
|
|
||||||
constructor(baseStackName: string) {
|
|
||||||
this.baseStackName = baseStackName;
|
|
||||||
}
|
|
||||||
private baseStackName: string;
|
|
||||||
|
|
||||||
async setupBaseStack(CF: CloudFormation) {
|
|
||||||
const baseStackName = this.baseStackName;
|
|
||||||
|
|
||||||
const baseStack = BaseStackFormation.formation;
|
|
||||||
|
|
||||||
// Cloud Formation Input
|
|
||||||
const describeStackInput: DescribeStacksCommandInput = {
|
|
||||||
StackName: baseStackName,
|
|
||||||
};
|
|
||||||
const parametersWithoutHash: Parameter[] = [{ ParameterKey: 'EnvironmentName', ParameterValue: baseStackName }];
|
|
||||||
const parametersHash = crypto
|
|
||||||
.createHash('md5')
|
|
||||||
.update(baseStack + JSON.stringify(parametersWithoutHash))
|
|
||||||
.digest('hex');
|
|
||||||
const parameters: Parameter[] = [
|
|
||||||
...parametersWithoutHash,
|
|
||||||
...[{ ParameterKey: 'Version', ParameterValue: parametersHash }],
|
|
||||||
];
|
|
||||||
const updateInput: UpdateStackCommandInput = {
|
|
||||||
StackName: baseStackName,
|
|
||||||
TemplateBody: baseStack,
|
|
||||||
Parameters: parameters,
|
|
||||||
Capabilities: ['CAPABILITY_IAM'],
|
|
||||||
};
|
|
||||||
const createStackInput: CreateStackCommandInput = {
|
|
||||||
StackName: baseStackName,
|
|
||||||
TemplateBody: baseStack,
|
|
||||||
Parameters: parameters,
|
|
||||||
Capabilities: ['CAPABILITY_IAM'],
|
|
||||||
};
|
|
||||||
|
|
||||||
const stacks = await CF.send(
|
|
||||||
new ListStacksCommand({ StackStatusFilter: ['UPDATE_COMPLETE', 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE'] }),
|
|
||||||
);
|
|
||||||
const stackNames = stacks.StackSummaries?.map((x) => x.StackName) || [];
|
|
||||||
const stackExists: Boolean = stackNames.includes(baseStackName) || false;
|
|
||||||
const describeStack = async () => {
|
|
||||||
return await CF.send(new DescribeStacksCommand(describeStackInput));
|
|
||||||
};
|
|
||||||
try {
|
|
||||||
if (!stackExists) {
|
|
||||||
CloudRunnerLogger.log(`${baseStackName} stack does not exist (${JSON.stringify(stackNames)})`);
|
|
||||||
await CF.send(new CreateStackCommand(createStackInput));
|
|
||||||
CloudRunnerLogger.log(`created stack (version: ${parametersHash})`);
|
|
||||||
}
|
|
||||||
const CFState = await describeStack();
|
|
||||||
let stack = CFState.Stacks?.[0];
|
|
||||||
if (!stack) {
|
|
||||||
throw new Error(`Base stack doesn't exist, even after creation, stackExists check: ${stackExists}`);
|
|
||||||
}
|
|
||||||
const stackVersion = stack.Parameters?.find((x) => x.ParameterKey === 'Version')?.ParameterValue;
|
|
||||||
|
|
||||||
if (stack.StackStatus === 'CREATE_IN_PROGRESS') {
|
|
||||||
await waitUntilStackCreateComplete(
|
|
||||||
{
|
|
||||||
client: CF,
|
|
||||||
maxWaitTime: 200,
|
|
||||||
},
|
|
||||||
describeStackInput,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (stackExists) {
|
|
||||||
CloudRunnerLogger.log(`Base stack exists (version: ${stackVersion}, local version: ${parametersHash})`);
|
|
||||||
if (parametersHash !== stackVersion) {
|
|
||||||
CloudRunnerLogger.log(`Attempting update of base stack`);
|
|
||||||
try {
|
|
||||||
await CF.send(new UpdateStackCommand(updateInput));
|
|
||||||
} catch (error: any) {
|
|
||||||
if (error['message'].includes('No updates are to be performed')) {
|
|
||||||
CloudRunnerLogger.log(`No updates are to be performed`);
|
|
||||||
} else {
|
|
||||||
CloudRunnerLogger.log(`Update Failed (Stack name: ${baseStackName})`);
|
|
||||||
CloudRunnerLogger.log(error['message']);
|
|
||||||
}
|
|
||||||
CloudRunnerLogger.log(`Continuing...`);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
CloudRunnerLogger.log(`No update required`);
|
|
||||||
}
|
|
||||||
stack = (await describeStack()).Stacks?.[0];
|
|
||||||
if (!stack) {
|
|
||||||
throw new Error(
|
|
||||||
`Base stack doesn't exist, even after updating and creation, stackExists check: ${stackExists}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
if (stack.StackStatus === 'UPDATE_IN_PROGRESS') {
|
|
||||||
await waitUntilStackUpdateComplete(
|
|
||||||
{
|
|
||||||
client: CF,
|
|
||||||
maxWaitTime: 200,
|
|
||||||
},
|
|
||||||
describeStackInput,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
CloudRunnerLogger.log('base stack is now ready');
|
|
||||||
} catch (error) {
|
|
||||||
core.error(JSON.stringify(await describeStack(), undefined, 4));
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,39 +0,0 @@
|
|||||||
import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation';
|
|
||||||
|
|
||||||
export class AWSCloudFormationTemplates {
|
|
||||||
public static getParameterTemplate(p1: string) {
|
|
||||||
return `
|
|
||||||
${p1}:
|
|
||||||
Type: String
|
|
||||||
Default: ''
|
|
||||||
`;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static getSecretTemplate(p1: string) {
|
|
||||||
return `
|
|
||||||
${p1}Secret:
|
|
||||||
Type: AWS::SecretsManager::Secret
|
|
||||||
Properties:
|
|
||||||
Name: '${p1}'
|
|
||||||
SecretString: !Ref ${p1}
|
|
||||||
`;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static getSecretDefinitionTemplate(p1: string, p2: string) {
|
|
||||||
return `
|
|
||||||
- Name: '${p1}'
|
|
||||||
ValueFrom: !Ref ${p2}Secret
|
|
||||||
`;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static insertAtTemplate(template: string, insertionKey: string, insertion: string) {
|
|
||||||
const index = template.search(insertionKey) + insertionKey.length + '\n'.length;
|
|
||||||
template = [template.slice(0, index), insertion, template.slice(index)].join('');
|
|
||||||
|
|
||||||
return template;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static readTaskCloudFormationTemplate(): string {
|
|
||||||
return TaskDefinitionFormation.formation;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
|
||||||
import { CloudFormation, DescribeStackEventsCommand } from '@aws-sdk/client-cloudformation';
|
|
||||||
import * as core from '@actions/core';
|
|
||||||
import CloudRunner from '../../cloud-runner';
|
|
||||||
|
|
||||||
export class AWSError {
|
|
||||||
static async handleStackCreationFailure(error: any, CF: CloudFormation, taskDefStackName: string) {
|
|
||||||
CloudRunnerLogger.log('aws error: ');
|
|
||||||
core.error(JSON.stringify(error, undefined, 4));
|
|
||||||
if (CloudRunner.buildParameters.cloudRunnerDebug) {
|
|
||||||
CloudRunnerLogger.log('Getting events and resources for task stack');
|
|
||||||
const events = (await CF.send(new DescribeStackEventsCommand({ StackName: taskDefStackName }))).StackEvents;
|
|
||||||
CloudRunnerLogger.log(JSON.stringify(events, undefined, 4));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,227 +0,0 @@
|
|||||||
import {
|
|
||||||
CloudFormation,
|
|
||||||
CreateStackCommand,
|
|
||||||
CreateStackCommandInput,
|
|
||||||
DescribeStackResourcesCommand,
|
|
||||||
DescribeStacksCommand,
|
|
||||||
ListStacksCommand,
|
|
||||||
waitUntilStackCreateComplete,
|
|
||||||
} from '@aws-sdk/client-cloudformation';
|
|
||||||
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
|
|
||||||
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
|
||||||
import { AWSCloudFormationTemplates } from './aws-cloud-formation-templates';
|
|
||||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
|
||||||
import { AWSError } from './aws-error';
|
|
||||||
import CloudRunner from '../../cloud-runner';
|
|
||||||
import { CleanupCronFormation } from './cloud-formations/cleanup-cron-formation';
|
|
||||||
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
|
||||||
import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation';
|
|
||||||
|
|
||||||
export class AWSJobStack {
|
|
||||||
private baseStackName: string;
|
|
||||||
constructor(baseStackName: string) {
|
|
||||||
this.baseStackName = baseStackName;
|
|
||||||
}
|
|
||||||
|
|
||||||
public async setupCloudFormations(
|
|
||||||
CF: CloudFormation,
|
|
||||||
buildGuid: string,
|
|
||||||
image: string,
|
|
||||||
entrypoint: string[],
|
|
||||||
commands: string,
|
|
||||||
mountdir: string,
|
|
||||||
workingdir: string,
|
|
||||||
secrets: CloudRunnerSecret[],
|
|
||||||
): Promise<CloudRunnerAWSTaskDef> {
|
|
||||||
const taskDefStackName = `${this.baseStackName}-${buildGuid}`;
|
|
||||||
let taskDefCloudFormation = AWSCloudFormationTemplates.readTaskCloudFormationTemplate();
|
|
||||||
taskDefCloudFormation = taskDefCloudFormation.replace(
|
|
||||||
`ContainerCpu:
|
|
||||||
Default: 1024`,
|
|
||||||
`ContainerCpu:
|
|
||||||
Default: ${Number.parseInt(CloudRunner.buildParameters.containerCpu)}`,
|
|
||||||
);
|
|
||||||
taskDefCloudFormation = taskDefCloudFormation.replace(
|
|
||||||
`ContainerMemory:
|
|
||||||
Default: 2048`,
|
|
||||||
`ContainerMemory:
|
|
||||||
Default: ${Number.parseInt(CloudRunner.buildParameters.containerMemory)}`,
|
|
||||||
);
|
|
||||||
if (!CloudRunnerOptions.asyncCloudRunner) {
|
|
||||||
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
|
|
||||||
taskDefCloudFormation,
|
|
||||||
'# template resources logstream',
|
|
||||||
TaskDefinitionFormation.streamLogs,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
for (const secret of secrets) {
|
|
||||||
secret.ParameterKey = `${buildGuid.replace(/[^\dA-Za-z]/g, '')}${secret.ParameterKey.replace(
|
|
||||||
/[^\dA-Za-z]/g,
|
|
||||||
'',
|
|
||||||
)}`;
|
|
||||||
if (typeof secret.ParameterValue == 'number') {
|
|
||||||
secret.ParameterValue = `${secret.ParameterValue}`;
|
|
||||||
}
|
|
||||||
if (!secret.ParameterValue || secret.ParameterValue === '') {
|
|
||||||
secrets = secrets.filter((x) => x !== secret);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
|
|
||||||
taskDefCloudFormation,
|
|
||||||
'p1 - input',
|
|
||||||
AWSCloudFormationTemplates.getParameterTemplate(secret.ParameterKey),
|
|
||||||
);
|
|
||||||
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
|
|
||||||
taskDefCloudFormation,
|
|
||||||
'# template resources secrets',
|
|
||||||
AWSCloudFormationTemplates.getSecretTemplate(`${secret.ParameterKey}`),
|
|
||||||
);
|
|
||||||
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
|
|
||||||
taskDefCloudFormation,
|
|
||||||
'p3 - container def',
|
|
||||||
AWSCloudFormationTemplates.getSecretDefinitionTemplate(secret.EnvironmentVariable, secret.ParameterKey),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
const secretsMappedToCloudFormationParameters = secrets.map((x) => {
|
|
||||||
return { ParameterKey: x.ParameterKey.replace(/[^\dA-Za-z]/g, ''), ParameterValue: x.ParameterValue };
|
|
||||||
});
|
|
||||||
const logGroupName = `${this.baseStackName}/${taskDefStackName}`;
|
|
||||||
const parameters = [
|
|
||||||
{
|
|
||||||
ParameterKey: 'EnvironmentName',
|
|
||||||
ParameterValue: this.baseStackName,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
ParameterKey: 'ImageUrl',
|
|
||||||
ParameterValue: image,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
ParameterKey: 'ServiceName',
|
|
||||||
ParameterValue: taskDefStackName,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
ParameterKey: 'LogGroupName',
|
|
||||||
ParameterValue: logGroupName,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
ParameterKey: 'Command',
|
|
||||||
ParameterValue: 'echo "this template should be overwritten when running a task"',
|
|
||||||
},
|
|
||||||
{
|
|
||||||
ParameterKey: 'EntryPoint',
|
|
||||||
ParameterValue: entrypoint.join(','),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
ParameterKey: 'WorkingDirectory',
|
|
||||||
ParameterValue: workingdir,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
ParameterKey: 'EFSMountDirectory',
|
|
||||||
ParameterValue: mountdir,
|
|
||||||
},
|
|
||||||
...secretsMappedToCloudFormationParameters,
|
|
||||||
];
|
|
||||||
CloudRunnerLogger.log(
|
|
||||||
`Starting AWS job with memory: ${CloudRunner.buildParameters.containerMemory} cpu: ${CloudRunner.buildParameters.containerCpu}`,
|
|
||||||
);
|
|
||||||
let previousStackExists = true;
|
|
||||||
while (previousStackExists) {
|
|
||||||
previousStackExists = false;
|
|
||||||
const stacks = await CF.send(new ListStacksCommand({}));
|
|
||||||
if (!stacks.StackSummaries) {
|
|
||||||
throw new Error('Faild to get stacks');
|
|
||||||
}
|
|
||||||
for (let index = 0; index < stacks.StackSummaries.length; index++) {
|
|
||||||
const element = stacks.StackSummaries[index];
|
|
||||||
if (element.StackName === taskDefStackName && element.StackStatus !== 'DELETE_COMPLETE') {
|
|
||||||
previousStackExists = true;
|
|
||||||
CloudRunnerLogger.log(`Previous stack still exists: ${JSON.stringify(element)}`);
|
|
||||||
await new Promise((promise) => setTimeout(promise, 5000));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const createStackInput: CreateStackCommandInput = {
|
|
||||||
StackName: taskDefStackName,
|
|
||||||
TemplateBody: taskDefCloudFormation,
|
|
||||||
Capabilities: ['CAPABILITY_IAM'],
|
|
||||||
Parameters: parameters,
|
|
||||||
};
|
|
||||||
try {
|
|
||||||
CloudRunnerLogger.log(`Creating job aws formation ${taskDefStackName}`);
|
|
||||||
await CF.send(new CreateStackCommand(createStackInput));
|
|
||||||
await waitUntilStackCreateComplete(
|
|
||||||
{
|
|
||||||
client: CF,
|
|
||||||
maxWaitTime: 200,
|
|
||||||
},
|
|
||||||
{ StackName: taskDefStackName },
|
|
||||||
);
|
|
||||||
const describeStack = await CF.send(new DescribeStacksCommand({ StackName: taskDefStackName }));
|
|
||||||
for (const parameter of parameters) {
|
|
||||||
if (!describeStack.Stacks?.[0].Parameters?.some((x) => x.ParameterKey === parameter.ParameterKey)) {
|
|
||||||
throw new Error(`Parameter ${parameter.ParameterKey} not found in stack`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
await AWSError.handleStackCreationFailure(error, CF, taskDefStackName);
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
|
|
||||||
const createCleanupStackInput: CreateStackCommandInput = {
|
|
||||||
StackName: `${taskDefStackName}-cleanup`,
|
|
||||||
TemplateBody: CleanupCronFormation.formation,
|
|
||||||
Capabilities: ['CAPABILITY_IAM'],
|
|
||||||
Parameters: [
|
|
||||||
{
|
|
||||||
ParameterKey: 'StackName',
|
|
||||||
ParameterValue: taskDefStackName,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
ParameterKey: 'DeleteStackName',
|
|
||||||
ParameterValue: `${taskDefStackName}-cleanup`,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
ParameterKey: 'TTL',
|
|
||||||
ParameterValue: `1080`,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
ParameterKey: 'BUILDGUID',
|
|
||||||
ParameterValue: CloudRunner.buildParameters.buildGuid,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
ParameterKey: 'EnvironmentName',
|
|
||||||
ParameterValue: this.baseStackName,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
|
||||||
if (CloudRunnerOptions.useCleanupCron) {
|
|
||||||
try {
|
|
||||||
CloudRunnerLogger.log(`Creating job cleanup formation`);
|
|
||||||
await CF.send(new CreateStackCommand(createCleanupStackInput));
|
|
||||||
|
|
||||||
// await CF.waitFor('stackCreateComplete', { StackName: createCleanupStackInput.StackName }).promise();
|
|
||||||
} catch (error) {
|
|
||||||
await AWSError.handleStackCreationFailure(error, CF, taskDefStackName);
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const taskDefResources = (
|
|
||||||
await CF.send(
|
|
||||||
new DescribeStackResourcesCommand({
|
|
||||||
StackName: taskDefStackName,
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
).StackResources;
|
|
||||||
|
|
||||||
const baseResources = (await CF.send(new DescribeStackResourcesCommand({ StackName: this.baseStackName })))
|
|
||||||
.StackResources;
|
|
||||||
|
|
||||||
return {
|
|
||||||
taskDefStackName,
|
|
||||||
taskDefCloudFormation,
|
|
||||||
taskDefResources,
|
|
||||||
baseResources,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,268 +0,0 @@
|
|||||||
import {
|
|
||||||
DescribeTasksCommand,
|
|
||||||
ECS,
|
|
||||||
RunTaskCommand,
|
|
||||||
RunTaskCommandInput,
|
|
||||||
Task,
|
|
||||||
waitUntilTasksRunning,
|
|
||||||
} from '@aws-sdk/client-ecs';
|
|
||||||
import {
|
|
||||||
DescribeStreamCommand,
|
|
||||||
DescribeStreamCommandOutput,
|
|
||||||
GetRecordsCommand,
|
|
||||||
GetRecordsCommandOutput,
|
|
||||||
GetShardIteratorCommand,
|
|
||||||
Kinesis,
|
|
||||||
} from '@aws-sdk/client-kinesis';
|
|
||||||
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
|
||||||
import * as core from '@actions/core';
|
|
||||||
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
|
|
||||||
import * as zlib from 'node:zlib';
|
|
||||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
|
||||||
import { Input } from '../../..';
|
|
||||||
import CloudRunner from '../../cloud-runner';
|
|
||||||
import { CommandHookService } from '../../services/hooks/command-hook-service';
|
|
||||||
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
|
|
||||||
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
|
||||||
import GitHub from '../../../github';
|
|
||||||
|
|
||||||
class AWSTaskRunner {
|
|
||||||
public static ECS: ECS;
|
|
||||||
public static Kinesis: Kinesis;
|
|
||||||
private static readonly encodedUnderscore = `$252F`;
|
|
||||||
static async runTask(
|
|
||||||
taskDef: CloudRunnerAWSTaskDef,
|
|
||||||
environment: CloudRunnerEnvironmentVariable[],
|
|
||||||
commands: string,
|
|
||||||
): Promise<{ output: string; shouldCleanup: boolean }> {
|
|
||||||
const cluster = taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ECSCluster')?.PhysicalResourceId || '';
|
|
||||||
const taskDefinition =
|
|
||||||
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'TaskDefinition')?.PhysicalResourceId || '';
|
|
||||||
const SubnetOne =
|
|
||||||
taskDef.baseResources?.find((x) => x.LogicalResourceId === 'PublicSubnetOne')?.PhysicalResourceId || '';
|
|
||||||
const SubnetTwo =
|
|
||||||
taskDef.baseResources?.find((x) => x.LogicalResourceId === 'PublicSubnetTwo')?.PhysicalResourceId || '';
|
|
||||||
const ContainerSecurityGroup =
|
|
||||||
taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ContainerSecurityGroup')?.PhysicalResourceId || '';
|
|
||||||
const streamName =
|
|
||||||
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'KinesisStream')?.PhysicalResourceId || '';
|
|
||||||
|
|
||||||
const runParameters = {
|
|
||||||
cluster,
|
|
||||||
taskDefinition,
|
|
||||||
platformVersion: '1.4.0',
|
|
||||||
overrides: {
|
|
||||||
containerOverrides: [
|
|
||||||
{
|
|
||||||
name: taskDef.taskDefStackName,
|
|
||||||
environment,
|
|
||||||
command: ['-c', CommandHookService.ApplyHooksToCommands(commands, CloudRunner.buildParameters)],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
launchType: 'FARGATE',
|
|
||||||
networkConfiguration: {
|
|
||||||
awsvpcConfiguration: {
|
|
||||||
subnets: [SubnetOne, SubnetTwo],
|
|
||||||
assignPublicIp: 'ENABLED',
|
|
||||||
securityGroups: [ContainerSecurityGroup],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
if (JSON.stringify(runParameters.overrides.containerOverrides).length > 8192) {
|
|
||||||
CloudRunnerLogger.log(JSON.stringify(runParameters.overrides.containerOverrides, undefined, 4));
|
|
||||||
throw new Error(`Container Overrides length must be at most 8192`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const task = await AWSTaskRunner.ECS.send(new RunTaskCommand(runParameters as RunTaskCommandInput));
|
|
||||||
const taskArn = task.tasks?.[0].taskArn || '';
|
|
||||||
CloudRunnerLogger.log('Cloud runner job is starting');
|
|
||||||
await AWSTaskRunner.waitUntilTaskRunning(taskArn, cluster);
|
|
||||||
CloudRunnerLogger.log(
|
|
||||||
`Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Async:${
|
|
||||||
CloudRunnerOptions.asyncCloudRunner
|
|
||||||
}`,
|
|
||||||
);
|
|
||||||
if (CloudRunnerOptions.asyncCloudRunner) {
|
|
||||||
const shouldCleanup: boolean = false;
|
|
||||||
const output: string = '';
|
|
||||||
CloudRunnerLogger.log(`Watch Cloud Runner To End: false`);
|
|
||||||
|
|
||||||
return { output, shouldCleanup };
|
|
||||||
}
|
|
||||||
|
|
||||||
CloudRunnerLogger.log(`Streaming...`);
|
|
||||||
const { output, shouldCleanup } = await this.streamLogsUntilTaskStops(cluster, taskArn, streamName);
|
|
||||||
let exitCode;
|
|
||||||
let containerState;
|
|
||||||
let taskData;
|
|
||||||
while (exitCode === undefined) {
|
|
||||||
await new Promise((resolve) => resolve(10000));
|
|
||||||
taskData = await AWSTaskRunner.describeTasks(cluster, taskArn);
|
|
||||||
containerState = taskData.containers?.[0];
|
|
||||||
exitCode = containerState?.exitCode;
|
|
||||||
}
|
|
||||||
CloudRunnerLogger.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`);
|
|
||||||
if (exitCode === undefined) {
|
|
||||||
CloudRunnerLogger.logWarning(`Undefined exitcode for container`);
|
|
||||||
}
|
|
||||||
const wasSuccessful = exitCode === 0;
|
|
||||||
if (wasSuccessful) {
|
|
||||||
CloudRunnerLogger.log(`Cloud runner job has finished successfully`);
|
|
||||||
|
|
||||||
return { output, shouldCleanup };
|
|
||||||
}
|
|
||||||
|
|
||||||
if (taskData?.stoppedReason === 'Essential container in task exited' && exitCode === 1) {
|
|
||||||
throw new Error('Container exited with code 1');
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new Error(`Task failed`);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static async waitUntilTaskRunning(taskArn: string, cluster: string) {
|
|
||||||
try {
|
|
||||||
await waitUntilTasksRunning(
|
|
||||||
{
|
|
||||||
client: AWSTaskRunner.ECS,
|
|
||||||
maxWaitTime: 120,
|
|
||||||
},
|
|
||||||
{ tasks: [taskArn], cluster },
|
|
||||||
);
|
|
||||||
} catch (error_) {
|
|
||||||
const error = error_ as Error;
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
|
||||||
CloudRunnerLogger.log(
|
|
||||||
`Cloud runner job has ended ${
|
|
||||||
(await AWSTaskRunner.describeTasks(cluster, taskArn)).containers?.[0].lastStatus
|
|
||||||
}`,
|
|
||||||
);
|
|
||||||
|
|
||||||
core.setFailed(error);
|
|
||||||
core.error(error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static async describeTasks(clusterName: string, taskArn: string) {
|
|
||||||
const tasks = await AWSTaskRunner.ECS.send(new DescribeTasksCommand({ cluster: clusterName, tasks: [taskArn] }));
|
|
||||||
if (tasks.tasks?.[0]) {
|
|
||||||
return tasks.tasks?.[0];
|
|
||||||
} else {
|
|
||||||
throw new Error('No task found');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static async streamLogsUntilTaskStops(clusterName: string, taskArn: string, kinesisStreamName: string) {
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
|
||||||
CloudRunnerLogger.log(`Streaming...`);
|
|
||||||
const stream = await AWSTaskRunner.getLogStream(kinesisStreamName);
|
|
||||||
let iterator = await AWSTaskRunner.getLogIterator(stream);
|
|
||||||
|
|
||||||
const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsStackName}${AWSTaskRunner.encodedUnderscore}${CloudRunner.buildParameters.awsStackName}-${CloudRunner.buildParameters.buildGuid}`;
|
|
||||||
CloudRunnerLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`);
|
|
||||||
await GitHub.updateGitHubCheck(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`, ``);
|
|
||||||
let shouldReadLogs = true;
|
|
||||||
let shouldCleanup = true;
|
|
||||||
let timestamp: number = 0;
|
|
||||||
let output = '';
|
|
||||||
while (shouldReadLogs) {
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 1500));
|
|
||||||
const taskData = await AWSTaskRunner.describeTasks(clusterName, taskArn);
|
|
||||||
({ timestamp, shouldReadLogs } = AWSTaskRunner.checkStreamingShouldContinue(taskData, timestamp, shouldReadLogs));
|
|
||||||
({ iterator, shouldReadLogs, output, shouldCleanup } = await AWSTaskRunner.handleLogStreamIteration(
|
|
||||||
iterator,
|
|
||||||
shouldReadLogs,
|
|
||||||
output,
|
|
||||||
shouldCleanup,
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
return { output, shouldCleanup };
|
|
||||||
}
|
|
||||||
|
|
||||||
private static async handleLogStreamIteration(
|
|
||||||
iterator: string,
|
|
||||||
shouldReadLogs: boolean,
|
|
||||||
output: string,
|
|
||||||
shouldCleanup: boolean,
|
|
||||||
) {
|
|
||||||
const records = await AWSTaskRunner.Kinesis.send(new GetRecordsCommand({ ShardIterator: iterator }));
|
|
||||||
iterator = records.NextShardIterator || '';
|
|
||||||
({ shouldReadLogs, output, shouldCleanup } = AWSTaskRunner.logRecords(
|
|
||||||
records,
|
|
||||||
iterator,
|
|
||||||
shouldReadLogs,
|
|
||||||
output,
|
|
||||||
shouldCleanup,
|
|
||||||
));
|
|
||||||
|
|
||||||
return { iterator, shouldReadLogs, output, shouldCleanup };
|
|
||||||
}
|
|
||||||
|
|
||||||
private static checkStreamingShouldContinue(taskData: Task, timestamp: number, shouldReadLogs: boolean) {
|
|
||||||
if (taskData?.lastStatus === 'UNKNOWN') {
|
|
||||||
CloudRunnerLogger.log('## Cloud runner job unknwon');
|
|
||||||
}
|
|
||||||
if (taskData?.lastStatus !== 'RUNNING') {
|
|
||||||
if (timestamp === 0) {
|
|
||||||
CloudRunnerLogger.log('## Cloud runner job stopped, streaming end of logs');
|
|
||||||
timestamp = Date.now();
|
|
||||||
}
|
|
||||||
if (timestamp !== 0 && Date.now() - timestamp > 30000) {
|
|
||||||
CloudRunnerLogger.log('## Cloud runner status is not RUNNING for 30 seconds, last query for logs');
|
|
||||||
shouldReadLogs = false;
|
|
||||||
}
|
|
||||||
CloudRunnerLogger.log(`## Status of job: ${taskData.lastStatus}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return { timestamp, shouldReadLogs };
|
|
||||||
}
|
|
||||||
|
|
||||||
private static logRecords(
|
|
||||||
records: GetRecordsCommandOutput,
|
|
||||||
iterator: string,
|
|
||||||
shouldReadLogs: boolean,
|
|
||||||
output: string,
|
|
||||||
shouldCleanup: boolean,
|
|
||||||
) {
|
|
||||||
if ((records.Records ?? []).length > 0 && iterator) {
|
|
||||||
for (const record of records.Records ?? []) {
|
|
||||||
const json = JSON.parse(
|
|
||||||
zlib.gunzipSync(Buffer.from(record.Data as unknown as string, 'base64')).toString('utf8'),
|
|
||||||
);
|
|
||||||
if (json.messageType === 'DATA_MESSAGE') {
|
|
||||||
for (const logEvent of json.logEvents) {
|
|
||||||
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
|
|
||||||
logEvent.message,
|
|
||||||
shouldReadLogs,
|
|
||||||
shouldCleanup,
|
|
||||||
output,
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return { shouldReadLogs, output, shouldCleanup };
|
|
||||||
}
|
|
||||||
|
|
||||||
private static async getLogStream(kinesisStreamName: string) {
|
|
||||||
return await AWSTaskRunner.Kinesis.send(new DescribeStreamCommand({ StreamName: kinesisStreamName }));
|
|
||||||
}
|
|
||||||
|
|
||||||
private static async getLogIterator(stream: DescribeStreamCommandOutput) {
|
|
||||||
return (
|
|
||||||
(
|
|
||||||
await AWSTaskRunner.Kinesis.send(
|
|
||||||
new GetShardIteratorCommand({
|
|
||||||
ShardIteratorType: 'TRIM_HORIZON',
|
|
||||||
StreamName: stream.StreamDescription?.StreamName ?? '',
|
|
||||||
ShardId: stream.StreamDescription?.Shards?.[0]?.ShardId || '',
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
).ShardIterator || ''
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
export default AWSTaskRunner;
|
|
||||||
@@ -1,397 +0,0 @@
|
|||||||
export class BaseStackFormation {
|
|
||||||
public static readonly baseStackDecription = `Game-CI base stack`;
|
|
||||||
public static readonly formation: string = `AWSTemplateFormatVersion: '2010-09-09'
|
|
||||||
Description: ${BaseStackFormation.baseStackDecription}
|
|
||||||
Parameters:
|
|
||||||
EnvironmentName:
|
|
||||||
Type: String
|
|
||||||
Default: development
|
|
||||||
Description: 'Your deployment environment: DEV, QA , PROD'
|
|
||||||
Version:
|
|
||||||
Type: String
|
|
||||||
Description: 'hash of template'
|
|
||||||
|
|
||||||
# ContainerPort:
|
|
||||||
# Type: Number
|
|
||||||
# Default: 80
|
|
||||||
# Description: What port number the application inside the docker container is binding to
|
|
||||||
|
|
||||||
Mappings:
|
|
||||||
# Hard values for the subnet masks. These masks define
|
|
||||||
# the range of internal IP addresses that can be assigned.
|
|
||||||
# The VPC can have all IP's from 10.0.0.0 to 10.0.255.255
|
|
||||||
# There are four subnets which cover the ranges:
|
|
||||||
#
|
|
||||||
# 10.0.0.0 - 10.0.0.255
|
|
||||||
# 10.0.1.0 - 10.0.1.255
|
|
||||||
# 10.0.2.0 - 10.0.2.255
|
|
||||||
# 10.0.3.0 - 10.0.3.255
|
|
||||||
|
|
||||||
SubnetConfig:
|
|
||||||
VPC:
|
|
||||||
CIDR: '10.0.0.0/16'
|
|
||||||
PublicOne:
|
|
||||||
CIDR: '10.0.0.0/24'
|
|
||||||
PublicTwo:
|
|
||||||
CIDR: '10.0.1.0/24'
|
|
||||||
|
|
||||||
Resources:
|
|
||||||
# VPC in which containers will be networked.
|
|
||||||
# It has two public subnets, and two private subnets.
|
|
||||||
# We distribute the subnets across the first two available subnets
|
|
||||||
# for the region, for high availability.
|
|
||||||
VPC:
|
|
||||||
Type: AWS::EC2::VPC
|
|
||||||
Properties:
|
|
||||||
EnableDnsSupport: true
|
|
||||||
EnableDnsHostnames: true
|
|
||||||
CidrBlock: !FindInMap ['SubnetConfig', 'VPC', 'CIDR']
|
|
||||||
|
|
||||||
MainBucket:
|
|
||||||
Type: "AWS::S3::Bucket"
|
|
||||||
Properties:
|
|
||||||
BucketName: !Ref EnvironmentName
|
|
||||||
|
|
||||||
EFSServerSecurityGroup:
|
|
||||||
Type: AWS::EC2::SecurityGroup
|
|
||||||
Properties:
|
|
||||||
GroupName: 'efs-server-endpoints'
|
|
||||||
GroupDescription: Which client ip addrs are allowed to access EFS server
|
|
||||||
VpcId: !Ref 'VPC'
|
|
||||||
SecurityGroupIngress:
|
|
||||||
- IpProtocol: tcp
|
|
||||||
FromPort: 2049
|
|
||||||
ToPort: 2049
|
|
||||||
SourceSecurityGroupId: !Ref ContainerSecurityGroup
|
|
||||||
#CidrIp: !FindInMap ['SubnetConfig', 'VPC', 'CIDR']
|
|
||||||
# A security group for the containers we will run in Fargate.
|
|
||||||
# Rules are added to this security group based on what ingress you
|
|
||||||
# add for the cluster.
|
|
||||||
ContainerSecurityGroup:
|
|
||||||
Type: AWS::EC2::SecurityGroup
|
|
||||||
Properties:
|
|
||||||
GroupName: 'task security group'
|
|
||||||
GroupDescription: Access to the Fargate containers
|
|
||||||
VpcId: !Ref 'VPC'
|
|
||||||
# SecurityGroupIngress:
|
|
||||||
# - IpProtocol: tcp
|
|
||||||
# FromPort: !Ref ContainerPort
|
|
||||||
# ToPort: !Ref ContainerPort
|
|
||||||
# CidrIp: 0.0.0.0/0
|
|
||||||
SecurityGroupEgress:
|
|
||||||
- IpProtocol: -1
|
|
||||||
FromPort: 2049
|
|
||||||
ToPort: 2049
|
|
||||||
CidrIp: '0.0.0.0/0'
|
|
||||||
|
|
||||||
# Two public subnets, where containers can have public IP addresses
|
|
||||||
PublicSubnetOne:
|
|
||||||
Type: AWS::EC2::Subnet
|
|
||||||
Properties:
|
|
||||||
AvailabilityZone: !Select
|
|
||||||
- 0
|
|
||||||
- Fn::GetAZs: !Ref 'AWS::Region'
|
|
||||||
VpcId: !Ref 'VPC'
|
|
||||||
CidrBlock: !FindInMap ['SubnetConfig', 'PublicOne', 'CIDR']
|
|
||||||
# MapPublicIpOnLaunch: true
|
|
||||||
|
|
||||||
PublicSubnetTwo:
|
|
||||||
Type: AWS::EC2::Subnet
|
|
||||||
Properties:
|
|
||||||
AvailabilityZone: !Select
|
|
||||||
- 1
|
|
||||||
- Fn::GetAZs: !Ref 'AWS::Region'
|
|
||||||
VpcId: !Ref 'VPC'
|
|
||||||
CidrBlock: !FindInMap ['SubnetConfig', 'PublicTwo', 'CIDR']
|
|
||||||
# MapPublicIpOnLaunch: true
|
|
||||||
|
|
||||||
# Setup networking resources for the public subnets. Containers
|
|
||||||
# in the public subnets have public IP addresses and the routing table
|
|
||||||
# sends network traffic via the internet gateway.
|
|
||||||
InternetGateway:
|
|
||||||
Type: AWS::EC2::InternetGateway
|
|
||||||
GatewayAttachement:
|
|
||||||
Type: AWS::EC2::VPCGatewayAttachment
|
|
||||||
Properties:
|
|
||||||
VpcId: !Ref 'VPC'
|
|
||||||
InternetGatewayId: !Ref 'InternetGateway'
|
|
||||||
|
|
||||||
# Attaching a Internet Gateway to route table makes it public.
|
|
||||||
PublicRouteTable:
|
|
||||||
Type: AWS::EC2::RouteTable
|
|
||||||
Properties:
|
|
||||||
VpcId: !Ref 'VPC'
|
|
||||||
PublicRoute:
|
|
||||||
Type: AWS::EC2::Route
|
|
||||||
DependsOn: GatewayAttachement
|
|
||||||
Properties:
|
|
||||||
RouteTableId: !Ref 'PublicRouteTable'
|
|
||||||
DestinationCidrBlock: '0.0.0.0/0'
|
|
||||||
GatewayId: !Ref 'InternetGateway'
|
|
||||||
|
|
||||||
# Attaching a public route table makes a subnet public.
|
|
||||||
PublicSubnetOneRouteTableAssociation:
|
|
||||||
Type: AWS::EC2::SubnetRouteTableAssociation
|
|
||||||
Properties:
|
|
||||||
SubnetId: !Ref PublicSubnetOne
|
|
||||||
RouteTableId: !Ref PublicRouteTable
|
|
||||||
PublicSubnetTwoRouteTableAssociation:
|
|
||||||
Type: AWS::EC2::SubnetRouteTableAssociation
|
|
||||||
Properties:
|
|
||||||
SubnetId: !Ref PublicSubnetTwo
|
|
||||||
RouteTableId: !Ref PublicRouteTable
|
|
||||||
|
|
||||||
# ECS Resources
|
|
||||||
ECSCluster:
|
|
||||||
Type: AWS::ECS::Cluster
|
|
||||||
|
|
||||||
# A role used to allow AWS Autoscaling to inspect stats and adjust scaleable targets
|
|
||||||
# on your AWS account
|
|
||||||
AutoscalingRole:
|
|
||||||
Type: AWS::IAM::Role
|
|
||||||
Properties:
|
|
||||||
AssumeRolePolicyDocument:
|
|
||||||
Statement:
|
|
||||||
- Effect: Allow
|
|
||||||
Principal:
|
|
||||||
Service: [application-autoscaling.amazonaws.com]
|
|
||||||
Action: ['sts:AssumeRole']
|
|
||||||
Path: /
|
|
||||||
Policies:
|
|
||||||
- PolicyName: service-autoscaling
|
|
||||||
PolicyDocument:
|
|
||||||
Statement:
|
|
||||||
- Effect: Allow
|
|
||||||
Action:
|
|
||||||
- 'application-autoscaling:*'
|
|
||||||
- 'cloudwatch:DescribeAlarms'
|
|
||||||
- 'cloudwatch:PutMetricAlarm'
|
|
||||||
- 'ecs:DescribeServices'
|
|
||||||
- 'ecs:UpdateService'
|
|
||||||
Resource: '*'
|
|
||||||
|
|
||||||
# This is an IAM role which authorizes ECS to manage resources on your
|
|
||||||
# account on your behalf, such as updating your load balancer with the
|
|
||||||
# details of where your containers are, so that traffic can reach your
|
|
||||||
# containers.
|
|
||||||
ECSRole:
|
|
||||||
Type: AWS::IAM::Role
|
|
||||||
Properties:
|
|
||||||
AssumeRolePolicyDocument:
|
|
||||||
Statement:
|
|
||||||
- Effect: Allow
|
|
||||||
Principal:
|
|
||||||
Service: [ecs.amazonaws.com]
|
|
||||||
Action: ['sts:AssumeRole']
|
|
||||||
Path: /
|
|
||||||
Policies:
|
|
||||||
- PolicyName: ecs-service
|
|
||||||
PolicyDocument:
|
|
||||||
Statement:
|
|
||||||
- Effect: Allow
|
|
||||||
Action:
|
|
||||||
# Rules which allow ECS to attach network interfaces to instances
|
|
||||||
# on your behalf in order for awsvpc networking mode to work right
|
|
||||||
- 'ec2:AttachNetworkInterface'
|
|
||||||
- 'ec2:CreateNetworkInterface'
|
|
||||||
- 'ec2:CreateNetworkInterfacePermission'
|
|
||||||
- 'ec2:DeleteNetworkInterface'
|
|
||||||
- 'ec2:DeleteNetworkInterfacePermission'
|
|
||||||
- 'ec2:Describe*'
|
|
||||||
- 'ec2:DetachNetworkInterface'
|
|
||||||
|
|
||||||
# Rules which allow ECS to update load balancers on your behalf
|
|
||||||
# with the information sabout how to send traffic to your containers
|
|
||||||
- 'elasticloadbalancing:DeregisterInstancesFromLoadBalancer'
|
|
||||||
- 'elasticloadbalancing:DeregisterTargets'
|
|
||||||
- 'elasticloadbalancing:Describe*'
|
|
||||||
- 'elasticloadbalancing:RegisterInstancesWithLoadBalancer'
|
|
||||||
- 'elasticloadbalancing:RegisterTargets'
|
|
||||||
Resource: '*'
|
|
||||||
|
|
||||||
# This is a role which is used by the ECS tasks themselves.
|
|
||||||
ECSTaskExecutionRole:
|
|
||||||
Type: AWS::IAM::Role
|
|
||||||
Properties:
|
|
||||||
AssumeRolePolicyDocument:
|
|
||||||
Statement:
|
|
||||||
- Effect: Allow
|
|
||||||
Principal:
|
|
||||||
Service: [ecs-tasks.amazonaws.com]
|
|
||||||
Action: ['sts:AssumeRole']
|
|
||||||
Path: /
|
|
||||||
Policies:
|
|
||||||
- PolicyName: AmazonECSTaskExecutionRolePolicy
|
|
||||||
PolicyDocument:
|
|
||||||
Statement:
|
|
||||||
- Effect: Allow
|
|
||||||
Action:
|
|
||||||
# Allow the use of secret manager
|
|
||||||
- 'secretsmanager:GetSecretValue'
|
|
||||||
- 'kms:Decrypt'
|
|
||||||
|
|
||||||
# Allow the ECS Tasks to download images from ECR
|
|
||||||
- 'ecr:GetAuthorizationToken'
|
|
||||||
- 'ecr:BatchCheckLayerAvailability'
|
|
||||||
- 'ecr:GetDownloadUrlForLayer'
|
|
||||||
- 'ecr:BatchGetImage'
|
|
||||||
|
|
||||||
# Allow the ECS tasks to upload logs to CloudWatch
|
|
||||||
- 'logs:CreateLogStream'
|
|
||||||
- 'logs:PutLogEvents'
|
|
||||||
Resource: '*'
|
|
||||||
|
|
||||||
DeleteCFNLambdaExecutionRole:
|
|
||||||
Type: 'AWS::IAM::Role'
|
|
||||||
Properties:
|
|
||||||
AssumeRolePolicyDocument:
|
|
||||||
Version: '2012-10-17'
|
|
||||||
Statement:
|
|
||||||
- Effect: 'Allow'
|
|
||||||
Principal:
|
|
||||||
Service: ['lambda.amazonaws.com']
|
|
||||||
Action: 'sts:AssumeRole'
|
|
||||||
Path: '/'
|
|
||||||
Policies:
|
|
||||||
- PolicyName: DeleteCFNLambdaExecutionRole
|
|
||||||
PolicyDocument:
|
|
||||||
Version: '2012-10-17'
|
|
||||||
Statement:
|
|
||||||
- Effect: 'Allow'
|
|
||||||
Action:
|
|
||||||
- 'logs:CreateLogGroup'
|
|
||||||
- 'logs:CreateLogStream'
|
|
||||||
- 'logs:PutLogEvents'
|
|
||||||
Resource: 'arn:aws:logs:*:*:*'
|
|
||||||
- Effect: 'Allow'
|
|
||||||
Action:
|
|
||||||
- 'cloudformation:DeleteStack'
|
|
||||||
- 'kinesis:DeleteStream'
|
|
||||||
- 'secretsmanager:DeleteSecret'
|
|
||||||
- 'kinesis:DescribeStreamSummary'
|
|
||||||
- 'logs:DeleteLogGroup'
|
|
||||||
- 'logs:DeleteSubscriptionFilter'
|
|
||||||
- 'ecs:DeregisterTaskDefinition'
|
|
||||||
- 'lambda:DeleteFunction'
|
|
||||||
- 'lambda:InvokeFunction'
|
|
||||||
- 'events:RemoveTargets'
|
|
||||||
- 'events:DeleteRule'
|
|
||||||
- 'lambda:RemovePermission'
|
|
||||||
Resource: '*'
|
|
||||||
|
|
||||||
### cloud watch to kinesis role
|
|
||||||
CloudWatchIAMRole:
|
|
||||||
Type: AWS::IAM::Role
|
|
||||||
Properties:
|
|
||||||
AssumeRolePolicyDocument:
|
|
||||||
Statement:
|
|
||||||
- Effect: Allow
|
|
||||||
Principal:
|
|
||||||
Service: [logs.amazonaws.com]
|
|
||||||
Action: ['sts:AssumeRole']
|
|
||||||
Path: /
|
|
||||||
Policies:
|
|
||||||
- PolicyName: service-autoscaling
|
|
||||||
PolicyDocument:
|
|
||||||
Statement:
|
|
||||||
- Effect: Allow
|
|
||||||
Action:
|
|
||||||
- 'kinesis:PutRecord'
|
|
||||||
Resource: '*'
|
|
||||||
|
|
||||||
#####################EFS#####################
|
|
||||||
EfsFileStorage:
|
|
||||||
Type: 'AWS::EFS::FileSystem'
|
|
||||||
Properties:
|
|
||||||
BackupPolicy:
|
|
||||||
Status: ENABLED
|
|
||||||
PerformanceMode: maxIO
|
|
||||||
Encrypted: false
|
|
||||||
|
|
||||||
FileSystemPolicy:
|
|
||||||
Version: '2012-10-17'
|
|
||||||
Statement:
|
|
||||||
- Effect: 'Allow'
|
|
||||||
Action:
|
|
||||||
- 'elasticfilesystem:ClientMount'
|
|
||||||
- 'elasticfilesystem:ClientWrite'
|
|
||||||
- 'elasticfilesystem:ClientRootAccess'
|
|
||||||
Principal:
|
|
||||||
AWS: '*'
|
|
||||||
|
|
||||||
MountTargetResource1:
|
|
||||||
Type: AWS::EFS::MountTarget
|
|
||||||
Properties:
|
|
||||||
FileSystemId: !Ref EfsFileStorage
|
|
||||||
SubnetId: !Ref PublicSubnetOne
|
|
||||||
SecurityGroups:
|
|
||||||
- !Ref EFSServerSecurityGroup
|
|
||||||
|
|
||||||
MountTargetResource2:
|
|
||||||
Type: AWS::EFS::MountTarget
|
|
||||||
Properties:
|
|
||||||
FileSystemId: !Ref EfsFileStorage
|
|
||||||
SubnetId: !Ref PublicSubnetTwo
|
|
||||||
SecurityGroups:
|
|
||||||
- !Ref EFSServerSecurityGroup
|
|
||||||
|
|
||||||
Outputs:
|
|
||||||
EfsFileStorageId:
|
|
||||||
Description: 'The connection endpoint for the database.'
|
|
||||||
Value: !Ref EfsFileStorage
|
|
||||||
Export:
|
|
||||||
Name: !Sub ${'${EnvironmentName}'}:EfsFileStorageId
|
|
||||||
ClusterName:
|
|
||||||
Description: The name of the ECS cluster
|
|
||||||
Value: !Ref 'ECSCluster'
|
|
||||||
Export:
|
|
||||||
Name: !Sub${' ${EnvironmentName}'}:ClusterName
|
|
||||||
AutoscalingRole:
|
|
||||||
Description: The ARN of the role used for autoscaling
|
|
||||||
Value: !GetAtt 'AutoscalingRole.Arn'
|
|
||||||
Export:
|
|
||||||
Name: !Sub ${'${EnvironmentName}'}:AutoscalingRole
|
|
||||||
ECSRole:
|
|
||||||
Description: The ARN of the ECS role
|
|
||||||
Value: !GetAtt 'ECSRole.Arn'
|
|
||||||
Export:
|
|
||||||
Name: !Sub ${'${EnvironmentName}'}:ECSRole
|
|
||||||
ECSTaskExecutionRole:
|
|
||||||
Description: The ARN of the ECS role tsk execution role
|
|
||||||
Value: !GetAtt 'ECSTaskExecutionRole.Arn'
|
|
||||||
Export:
|
|
||||||
Name: !Sub ${'${EnvironmentName}'}:ECSTaskExecutionRole
|
|
||||||
|
|
||||||
DeleteCFNLambdaExecutionRole:
|
|
||||||
Description: Lambda execution role for cleaning up cloud formations
|
|
||||||
Value: !GetAtt 'DeleteCFNLambdaExecutionRole.Arn'
|
|
||||||
Export:
|
|
||||||
Name: !Sub ${'${EnvironmentName}'}:DeleteCFNLambdaExecutionRole
|
|
||||||
|
|
||||||
CloudWatchIAMRole:
|
|
||||||
Description: The ARN of the CloudWatch role for subscription filter
|
|
||||||
Value: !GetAtt 'CloudWatchIAMRole.Arn'
|
|
||||||
Export:
|
|
||||||
Name: !Sub ${'${EnvironmentName}'}:CloudWatchIAMRole
|
|
||||||
VpcId:
|
|
||||||
Description: The ID of the VPC that this stack is deployed in
|
|
||||||
Value: !Ref 'VPC'
|
|
||||||
Export:
|
|
||||||
Name: !Sub ${'${EnvironmentName}'}:VpcId
|
|
||||||
PublicSubnetOne:
|
|
||||||
Description: Public subnet one
|
|
||||||
Value: !Ref 'PublicSubnetOne'
|
|
||||||
Export:
|
|
||||||
Name: !Sub ${'${EnvironmentName}'}:PublicSubnetOne
|
|
||||||
PublicSubnetTwo:
|
|
||||||
Description: Public subnet two
|
|
||||||
Value: !Ref 'PublicSubnetTwo'
|
|
||||||
Export:
|
|
||||||
Name: !Sub ${'${EnvironmentName}'}:PublicSubnetTwo
|
|
||||||
ContainerSecurityGroup:
|
|
||||||
Description: A security group used to allow Fargate containers to receive traffic
|
|
||||||
Value: !Ref 'ContainerSecurityGroup'
|
|
||||||
Export:
|
|
||||||
Name: !Sub ${'${EnvironmentName}'}:ContainerSecurityGroup
|
|
||||||
`;
|
|
||||||
}
|
|
||||||
@@ -1,146 +0,0 @@
|
|||||||
export class CleanupCronFormation {
|
|
||||||
public static readonly formation: string = `AWSTemplateFormatVersion: '2010-09-09'
|
|
||||||
Description: Schedule automatic deletion of CloudFormation stacks
|
|
||||||
Metadata:
|
|
||||||
AWS::CloudFormation::Interface:
|
|
||||||
ParameterGroups:
|
|
||||||
- Label:
|
|
||||||
default: Input configuration
|
|
||||||
Parameters:
|
|
||||||
- StackName
|
|
||||||
- TTL
|
|
||||||
ParameterLabels:
|
|
||||||
StackName:
|
|
||||||
default: Stack name
|
|
||||||
TTL:
|
|
||||||
default: Time-to-live
|
|
||||||
Parameters:
|
|
||||||
EnvironmentName:
|
|
||||||
Type: String
|
|
||||||
Default: development
|
|
||||||
Description: 'Your deployment environment: DEV, QA , PROD'
|
|
||||||
BUILDGUID:
|
|
||||||
Type: String
|
|
||||||
Default: ''
|
|
||||||
StackName:
|
|
||||||
Type: String
|
|
||||||
Description: Stack name that will be deleted.
|
|
||||||
DeleteStackName:
|
|
||||||
Type: String
|
|
||||||
Description: Stack name that will be deleted.
|
|
||||||
TTL:
|
|
||||||
Type: Number
|
|
||||||
Description: Time-to-live in minutes for the stack.
|
|
||||||
Resources:
|
|
||||||
DeleteCFNLambda:
|
|
||||||
Type: "AWS::Lambda::Function"
|
|
||||||
Properties:
|
|
||||||
FunctionName: !Join [ "", [ 'DeleteCFNLambda', !Ref BUILDGUID ] ]
|
|
||||||
Code:
|
|
||||||
ZipFile: |
|
|
||||||
import boto3
|
|
||||||
import os
|
|
||||||
import json
|
|
||||||
|
|
||||||
stack_name = os.environ['stackName']
|
|
||||||
delete_stack_name = os.environ['deleteStackName']
|
|
||||||
|
|
||||||
def delete_cfn(stack_name):
|
|
||||||
try:
|
|
||||||
cfn = boto3.resource('cloudformation')
|
|
||||||
stack = cfn.Stack(stack_name)
|
|
||||||
stack.delete()
|
|
||||||
return "SUCCESS"
|
|
||||||
except:
|
|
||||||
return "ERROR"
|
|
||||||
|
|
||||||
def handler(event, context):
|
|
||||||
print("Received event:")
|
|
||||||
print(json.dumps(event))
|
|
||||||
result = delete_cfn(stack_name)
|
|
||||||
delete_cfn(delete_stack_name)
|
|
||||||
return result
|
|
||||||
Environment:
|
|
||||||
Variables:
|
|
||||||
stackName: !Ref 'StackName'
|
|
||||||
deleteStackName: !Ref 'DeleteStackName'
|
|
||||||
Handler: "index.handler"
|
|
||||||
Runtime: "python3.9"
|
|
||||||
Timeout: "5"
|
|
||||||
Role:
|
|
||||||
'Fn::ImportValue': !Sub '\${EnvironmentName}:DeleteCFNLambdaExecutionRole'
|
|
||||||
DeleteStackEventRule:
|
|
||||||
DependsOn:
|
|
||||||
- DeleteCFNLambda
|
|
||||||
- GenerateCronExpression
|
|
||||||
Type: "AWS::Events::Rule"
|
|
||||||
Properties:
|
|
||||||
Name: !Join [ "", [ 'DeleteStackEventRule', !Ref BUILDGUID ] ]
|
|
||||||
Description: Delete stack event
|
|
||||||
ScheduleExpression: !GetAtt GenerateCronExpression.cron_exp
|
|
||||||
State: "ENABLED"
|
|
||||||
Targets:
|
|
||||||
-
|
|
||||||
Arn: !GetAtt DeleteCFNLambda.Arn
|
|
||||||
Id: 'DeleteCFNLambda'
|
|
||||||
PermissionForDeleteCFNLambda:
|
|
||||||
Type: "AWS::Lambda::Permission"
|
|
||||||
DependsOn:
|
|
||||||
- DeleteStackEventRule
|
|
||||||
Properties:
|
|
||||||
FunctionName: !Join [ "", [ 'DeleteCFNLambda', !Ref BUILDGUID ] ]
|
|
||||||
Action: "lambda:InvokeFunction"
|
|
||||||
Principal: "events.amazonaws.com"
|
|
||||||
SourceArn: !GetAtt DeleteStackEventRule.Arn
|
|
||||||
GenerateCronExpLambda:
|
|
||||||
Type: "AWS::Lambda::Function"
|
|
||||||
Properties:
|
|
||||||
FunctionName: !Join [ "", [ 'GenerateCronExpressionLambda', !Ref BUILDGUID ] ]
|
|
||||||
Code:
|
|
||||||
ZipFile: |
|
|
||||||
from datetime import datetime, timedelta
|
|
||||||
import os
|
|
||||||
import logging
|
|
||||||
import json
|
|
||||||
import cfnresponse
|
|
||||||
|
|
||||||
def deletion_time(ttl):
|
|
||||||
delete_at_time = datetime.now() + timedelta(minutes=int(ttl))
|
|
||||||
hh = delete_at_time.hour
|
|
||||||
mm = delete_at_time.minute
|
|
||||||
yyyy = delete_at_time.year
|
|
||||||
month = delete_at_time.month
|
|
||||||
dd = delete_at_time.day
|
|
||||||
# minutes hours day month day-of-week year
|
|
||||||
cron_exp = "cron({} {} {} {} ? {})".format(mm, hh, dd, month, yyyy)
|
|
||||||
return cron_exp
|
|
||||||
|
|
||||||
def handler(event, context):
|
|
||||||
print('Received event: %s' % json.dumps(event))
|
|
||||||
status = cfnresponse.SUCCESS
|
|
||||||
try:
|
|
||||||
if event['RequestType'] == 'Delete':
|
|
||||||
cfnresponse.send(event, context, status, {})
|
|
||||||
else:
|
|
||||||
ttl = event['ResourceProperties']['ttl']
|
|
||||||
responseData = {}
|
|
||||||
responseData['cron_exp'] = deletion_time(ttl)
|
|
||||||
cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData)
|
|
||||||
except Exception as e:
|
|
||||||
logging.error('Exception: %s' % e, exc_info=True)
|
|
||||||
status = cfnresponse.FAILED
|
|
||||||
cfnresponse.send(event, context, status, {}, None)
|
|
||||||
Handler: "index.handler"
|
|
||||||
Runtime: "python3.9"
|
|
||||||
Timeout: "5"
|
|
||||||
Role:
|
|
||||||
'Fn::ImportValue': !Sub '\${EnvironmentName}:DeleteCFNLambdaExecutionRole'
|
|
||||||
GenerateCronExpression:
|
|
||||||
Type: "Custom::GenerateCronExpression"
|
|
||||||
Version: "1.0"
|
|
||||||
Properties:
|
|
||||||
Name: !Join [ "", [ 'GenerateCronExpression', !Ref BUILDGUID ] ]
|
|
||||||
ServiceToken: !GetAtt GenerateCronExpLambda.Arn
|
|
||||||
ttl: !Ref 'TTL'
|
|
||||||
`;
|
|
||||||
}
|
|
||||||
@@ -1,169 +0,0 @@
|
|||||||
import CloudRunner from '../../../cloud-runner';
|
|
||||||
|
|
||||||
export class TaskDefinitionFormation {
|
|
||||||
public static readonly description: string = `Game CI Cloud Runner Task Stack`;
|
|
||||||
public static get formation(): string {
|
|
||||||
return `AWSTemplateFormatVersion: 2010-09-09
|
|
||||||
Description: ${TaskDefinitionFormation.description}
|
|
||||||
Parameters:
|
|
||||||
EnvironmentName:
|
|
||||||
Type: String
|
|
||||||
Default: development
|
|
||||||
Description: 'Your deployment environment: DEV, QA , PROD'
|
|
||||||
ServiceName:
|
|
||||||
Type: String
|
|
||||||
Default: example
|
|
||||||
Description: A name for the service
|
|
||||||
LogGroupName:
|
|
||||||
Type: String
|
|
||||||
Default: example
|
|
||||||
Description: Name to use for the log group created for this task
|
|
||||||
ImageUrl:
|
|
||||||
Type: String
|
|
||||||
Default: nginx
|
|
||||||
Description: >-
|
|
||||||
The url of a docker image that contains the application process that will
|
|
||||||
handle the traffic for this service
|
|
||||||
ContainerPort:
|
|
||||||
Type: Number
|
|
||||||
Default: 80
|
|
||||||
Description: What port number the application inside the docker container is binding to
|
|
||||||
ContainerCpu:
|
|
||||||
Default: ${CloudRunner.buildParameters.containerCpu}
|
|
||||||
Type: Number
|
|
||||||
Description: How much CPU to give the container. 1024 is 1 CPU
|
|
||||||
ContainerMemory:
|
|
||||||
Default: ${CloudRunner.buildParameters.containerMemory}
|
|
||||||
Type: Number
|
|
||||||
Description: How much memory in megabytes to give the container
|
|
||||||
BUILDGUID:
|
|
||||||
Type: String
|
|
||||||
Default: ''
|
|
||||||
Command:
|
|
||||||
Type: String
|
|
||||||
Default: 'ls'
|
|
||||||
EntryPoint:
|
|
||||||
Type: String
|
|
||||||
Default: '/bin/sh'
|
|
||||||
WorkingDirectory:
|
|
||||||
Type: String
|
|
||||||
Default: '/efsdata/'
|
|
||||||
Role:
|
|
||||||
Type: String
|
|
||||||
Default: ''
|
|
||||||
Description: >-
|
|
||||||
(Optional) An IAM role to give the service's containers if the code within
|
|
||||||
needs to access other AWS resources like S3 buckets, DynamoDB tables, etc
|
|
||||||
EFSMountDirectory:
|
|
||||||
Type: String
|
|
||||||
Default: '/efsdata'
|
|
||||||
# template secrets p1 - input
|
|
||||||
Mappings:
|
|
||||||
SubnetConfig:
|
|
||||||
VPC:
|
|
||||||
CIDR: 10.0.0.0/16
|
|
||||||
PublicOne:
|
|
||||||
CIDR: 10.0.0.0/24
|
|
||||||
PublicTwo:
|
|
||||||
CIDR: 10.0.1.0/24
|
|
||||||
Conditions:
|
|
||||||
HasCustomRole: !Not
|
|
||||||
- !Equals
|
|
||||||
- Ref: Role
|
|
||||||
- ''
|
|
||||||
Resources:
|
|
||||||
LogGroup:
|
|
||||||
Type: 'AWS::Logs::LogGroup'
|
|
||||||
Properties:
|
|
||||||
LogGroupName: !Ref LogGroupName
|
|
||||||
Metadata:
|
|
||||||
'AWS::CloudFormation::Designer':
|
|
||||||
id: aece53ae-b82d-4267-bc16-ed964b05db27
|
|
||||||
# template resources secrets
|
|
||||||
|
|
||||||
# template resources logstream
|
|
||||||
|
|
||||||
TaskDefinition:
|
|
||||||
Type: 'AWS::ECS::TaskDefinition'
|
|
||||||
Properties:
|
|
||||||
Family: !Ref ServiceName
|
|
||||||
Cpu: !Ref ContainerCpu
|
|
||||||
Memory: !Ref ContainerMemory
|
|
||||||
NetworkMode: awsvpc
|
|
||||||
Volumes:
|
|
||||||
- Name: efs-data
|
|
||||||
EFSVolumeConfiguration:
|
|
||||||
FilesystemId:
|
|
||||||
'Fn::ImportValue': !Sub '${'${EnvironmentName}'}:EfsFileStorageId'
|
|
||||||
TransitEncryption: DISABLED
|
|
||||||
RequiresCompatibilities:
|
|
||||||
- FARGATE
|
|
||||||
ExecutionRoleArn:
|
|
||||||
'Fn::ImportValue': !Sub '${'${EnvironmentName}'}:ECSTaskExecutionRole'
|
|
||||||
TaskRoleArn:
|
|
||||||
'Fn::If':
|
|
||||||
- HasCustomRole
|
|
||||||
- !Ref Role
|
|
||||||
- !Ref 'AWS::NoValue'
|
|
||||||
ContainerDefinitions:
|
|
||||||
- Name: !Ref ServiceName
|
|
||||||
Cpu: !Ref ContainerCpu
|
|
||||||
Memory: !Ref ContainerMemory
|
|
||||||
Image: !Ref ImageUrl
|
|
||||||
EntryPoint:
|
|
||||||
Fn::Split:
|
|
||||||
- ','
|
|
||||||
- !Ref EntryPoint
|
|
||||||
Command:
|
|
||||||
Fn::Split:
|
|
||||||
- ','
|
|
||||||
- !Ref Command
|
|
||||||
WorkingDirectory: !Ref WorkingDirectory
|
|
||||||
Environment:
|
|
||||||
- Name: ALLOW_EMPTY_PASSWORD
|
|
||||||
Value: 'yes'
|
|
||||||
# template - env vars
|
|
||||||
MountPoints:
|
|
||||||
- SourceVolume: efs-data
|
|
||||||
ContainerPath: !Ref EFSMountDirectory
|
|
||||||
ReadOnly: false
|
|
||||||
Secrets:
|
|
||||||
# template secrets p3 - container def
|
|
||||||
LogConfiguration:
|
|
||||||
LogDriver: awslogs
|
|
||||||
Options:
|
|
||||||
awslogs-group: !Ref LogGroupName
|
|
||||||
awslogs-region: !Ref 'AWS::Region'
|
|
||||||
awslogs-stream-prefix: !Ref ServiceName
|
|
||||||
DependsOn:
|
|
||||||
- LogGroup
|
|
||||||
`;
|
|
||||||
}
|
|
||||||
public static streamLogs = `
|
|
||||||
SubscriptionFilter:
|
|
||||||
Type: 'AWS::Logs::SubscriptionFilter'
|
|
||||||
Properties:
|
|
||||||
FilterPattern: ''
|
|
||||||
RoleArn:
|
|
||||||
'Fn::ImportValue': !Sub '${'${EnvironmentName}'}:CloudWatchIAMRole'
|
|
||||||
LogGroupName: !Ref LogGroupName
|
|
||||||
DestinationArn:
|
|
||||||
'Fn::GetAtt':
|
|
||||||
- KinesisStream
|
|
||||||
- Arn
|
|
||||||
Metadata:
|
|
||||||
'AWS::CloudFormation::Designer':
|
|
||||||
id: 7f809e91-9e5d-4678-98c1-c5085956c480
|
|
||||||
DependsOn:
|
|
||||||
- LogGroup
|
|
||||||
- KinesisStream
|
|
||||||
KinesisStream:
|
|
||||||
Type: 'AWS::Kinesis::Stream'
|
|
||||||
Properties:
|
|
||||||
Name: !Ref ServiceName
|
|
||||||
ShardCount: 1
|
|
||||||
Metadata:
|
|
||||||
'AWS::CloudFormation::Designer':
|
|
||||||
id: c6f18447-b879-4696-8873-f981b2cedd2b
|
|
||||||
`;
|
|
||||||
}
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
import { StackResource } from '@aws-sdk/client-cloudformation';
|
|
||||||
|
|
||||||
class CloudRunnerAWSTaskDef {
|
|
||||||
public taskDefStackName!: string;
|
|
||||||
public taskDefCloudFormation!: string;
|
|
||||||
public taskDefResources: StackResource[] | undefined;
|
|
||||||
public baseResources: StackResource[] | undefined;
|
|
||||||
}
|
|
||||||
export default CloudRunnerAWSTaskDef;
|
|
||||||
@@ -1,163 +0,0 @@
|
|||||||
import { CloudFormation, DeleteStackCommand, waitUntilStackDeleteComplete } from '@aws-sdk/client-cloudformation';
|
|
||||||
import { ECS as ECSClient } from '@aws-sdk/client-ecs';
|
|
||||||
import { Kinesis } from '@aws-sdk/client-kinesis';
|
|
||||||
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
|
||||||
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
|
||||||
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
|
|
||||||
import AwsTaskRunner from './aws-task-runner';
|
|
||||||
import { ProviderInterface } from '../provider-interface';
|
|
||||||
import BuildParameters from '../../../build-parameters';
|
|
||||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
|
||||||
import { AWSJobStack as AwsJobStack } from './aws-job-stack';
|
|
||||||
import { AWSBaseStack as AwsBaseStack } from './aws-base-stack';
|
|
||||||
import { Input } from '../../..';
|
|
||||||
import { GarbageCollectionService } from './services/garbage-collection-service';
|
|
||||||
import { ProviderResource } from '../provider-resource';
|
|
||||||
import { ProviderWorkflow } from '../provider-workflow';
|
|
||||||
import { TaskService } from './services/task-service';
|
|
||||||
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
|
||||||
|
|
||||||
class AWSBuildEnvironment implements ProviderInterface {
|
|
||||||
private baseStackName: string;
|
|
||||||
|
|
||||||
constructor(buildParameters: BuildParameters) {
|
|
||||||
this.baseStackName = buildParameters.awsStackName;
|
|
||||||
}
|
|
||||||
async listResources(): Promise<ProviderResource[]> {
|
|
||||||
await TaskService.getCloudFormationJobStacks();
|
|
||||||
await TaskService.getLogGroups();
|
|
||||||
await TaskService.getTasks();
|
|
||||||
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
listWorkflow(): Promise<ProviderWorkflow[]> {
|
|
||||||
throw new Error('Method not implemented.');
|
|
||||||
}
|
|
||||||
async watchWorkflow(): Promise<string> {
|
|
||||||
return await TaskService.watch();
|
|
||||||
}
|
|
||||||
|
|
||||||
async listOtherResources(): Promise<string> {
|
|
||||||
await TaskService.getLogGroups();
|
|
||||||
|
|
||||||
return '';
|
|
||||||
}
|
|
||||||
|
|
||||||
async garbageCollect(
|
|
||||||
filter: string,
|
|
||||||
previewOnly: boolean,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
olderThan: Number,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
fullCache: boolean,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
baseDependencies: boolean,
|
|
||||||
): Promise<string> {
|
|
||||||
await GarbageCollectionService.cleanup(!previewOnly);
|
|
||||||
|
|
||||||
return ``;
|
|
||||||
}
|
|
||||||
|
|
||||||
async cleanupWorkflow(
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
buildParameters: BuildParameters,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
branchName: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
|
||||||
) {}
|
|
||||||
async setupWorkflow(
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
buildGuid: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
buildParameters: BuildParameters,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
branchName: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
|
||||||
) {
|
|
||||||
process.env.AWS_REGION = Input.region;
|
|
||||||
const CF = new CloudFormation({ region: Input.region });
|
|
||||||
await new AwsBaseStack(this.baseStackName).setupBaseStack(CF);
|
|
||||||
}
|
|
||||||
|
|
||||||
async runTaskInWorkflow(
|
|
||||||
buildGuid: string,
|
|
||||||
image: string,
|
|
||||||
commands: string,
|
|
||||||
mountdir: string,
|
|
||||||
workingdir: string,
|
|
||||||
environment: CloudRunnerEnvironmentVariable[],
|
|
||||||
secrets: CloudRunnerSecret[],
|
|
||||||
): Promise<string> {
|
|
||||||
process.env.AWS_REGION = Input.region;
|
|
||||||
const ECS = new ECSClient({ region: Input.region });
|
|
||||||
const CF = new CloudFormation({ region: Input.region });
|
|
||||||
AwsTaskRunner.ECS = ECS;
|
|
||||||
AwsTaskRunner.Kinesis = new Kinesis({ region: Input.region });
|
|
||||||
CloudRunnerLogger.log(`AWS Region: ${CF.config.region}`);
|
|
||||||
const entrypoint = ['/bin/sh'];
|
|
||||||
const startTimeMs = Date.now();
|
|
||||||
const taskDef = await new AwsJobStack(this.baseStackName).setupCloudFormations(
|
|
||||||
CF,
|
|
||||||
buildGuid,
|
|
||||||
image,
|
|
||||||
entrypoint,
|
|
||||||
commands,
|
|
||||||
mountdir,
|
|
||||||
workingdir,
|
|
||||||
secrets,
|
|
||||||
);
|
|
||||||
|
|
||||||
let postRunTaskTimeMs;
|
|
||||||
try {
|
|
||||||
const postSetupStacksTimeMs = Date.now();
|
|
||||||
CloudRunnerLogger.log(`Setup job time: ${Math.floor((postSetupStacksTimeMs - startTimeMs) / 1000)}s`);
|
|
||||||
const { output, shouldCleanup } = await AwsTaskRunner.runTask(taskDef, environment, commands);
|
|
||||||
postRunTaskTimeMs = Date.now();
|
|
||||||
CloudRunnerLogger.log(`Run job time: ${Math.floor((postRunTaskTimeMs - postSetupStacksTimeMs) / 1000)}s`);
|
|
||||||
if (shouldCleanup) {
|
|
||||||
await this.cleanupResources(CF, taskDef);
|
|
||||||
}
|
|
||||||
const postCleanupTimeMs = Date.now();
|
|
||||||
if (postRunTaskTimeMs !== undefined)
|
|
||||||
CloudRunnerLogger.log(`Cleanup job time: ${Math.floor((postCleanupTimeMs - postRunTaskTimeMs) / 1000)}s`);
|
|
||||||
|
|
||||||
return output;
|
|
||||||
} catch (error) {
|
|
||||||
CloudRunnerLogger.log(`error running task ${error}`);
|
|
||||||
await this.cleanupResources(CF, taskDef);
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async cleanupResources(CF: CloudFormation, taskDef: CloudRunnerAWSTaskDef) {
|
|
||||||
CloudRunnerLogger.log('Cleanup starting');
|
|
||||||
await CF.send(new DeleteStackCommand({ StackName: taskDef.taskDefStackName }));
|
|
||||||
if (CloudRunnerOptions.useCleanupCron) {
|
|
||||||
await CF.send(new DeleteStackCommand({ StackName: `${taskDef.taskDefStackName}-cleanup` }));
|
|
||||||
}
|
|
||||||
|
|
||||||
await waitUntilStackDeleteComplete(
|
|
||||||
{
|
|
||||||
client: CF,
|
|
||||||
maxWaitTime: 200,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
StackName: taskDef.taskDefStackName,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
await waitUntilStackDeleteComplete(
|
|
||||||
{
|
|
||||||
client: CF,
|
|
||||||
maxWaitTime: 200,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
StackName: `${taskDef.taskDefStackName}-cleanup`,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
CloudRunnerLogger.log(`Deleted Stack: ${taskDef.taskDefStackName}`);
|
|
||||||
CloudRunnerLogger.log('Cleanup complete');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
export default AWSBuildEnvironment;
|
|
||||||
@@ -1,80 +0,0 @@
|
|||||||
import {
|
|
||||||
CloudFormation,
|
|
||||||
DeleteStackCommand,
|
|
||||||
DeleteStackCommandInput,
|
|
||||||
DescribeStackResourcesCommand,
|
|
||||||
} from '@aws-sdk/client-cloudformation';
|
|
||||||
import { CloudWatchLogs, DeleteLogGroupCommand } from '@aws-sdk/client-cloudwatch-logs';
|
|
||||||
import { ECS, StopTaskCommand } from '@aws-sdk/client-ecs';
|
|
||||||
import Input from '../../../../input';
|
|
||||||
import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
|
|
||||||
import { TaskService } from './task-service';
|
|
||||||
|
|
||||||
export class GarbageCollectionService {
|
|
||||||
static isOlderThan1day(date: Date) {
|
|
||||||
const ageDate = new Date(date.getTime() - Date.now());
|
|
||||||
|
|
||||||
return ageDate.getDay() > 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async cleanup(deleteResources = false, OneDayOlderOnly: boolean = false) {
|
|
||||||
process.env.AWS_REGION = Input.region;
|
|
||||||
const CF = new CloudFormation({ region: Input.region });
|
|
||||||
const ecs = new ECS({ region: Input.region });
|
|
||||||
const cwl = new CloudWatchLogs({ region: Input.region });
|
|
||||||
const taskDefinitionsInUse = new Array();
|
|
||||||
const tasks = await TaskService.getTasks();
|
|
||||||
|
|
||||||
for (const task of tasks) {
|
|
||||||
const { taskElement, element } = task;
|
|
||||||
taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
|
|
||||||
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.createdAt!))) {
|
|
||||||
CloudRunnerLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
|
|
||||||
await ecs.send(new StopTaskCommand({ task: taskElement.taskArn || '', cluster: element }));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const jobStacks = await TaskService.getCloudFormationJobStacks();
|
|
||||||
for (const element of jobStacks) {
|
|
||||||
if (
|
|
||||||
(await CF.send(new DescribeStackResourcesCommand({ StackName: element.StackName }))).StackResources?.some(
|
|
||||||
(x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId),
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
CloudRunnerLogger.log(`Skipping ${element.StackName} - active task was running not deleting`);
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (
|
|
||||||
deleteResources &&
|
|
||||||
(!OneDayOlderOnly || (element.CreationTime && GarbageCollectionService.isOlderThan1day(element.CreationTime)))
|
|
||||||
) {
|
|
||||||
if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') {
|
|
||||||
CloudRunnerLogger.log(`Skipping ${element.StackName} ignore list`);
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
CloudRunnerLogger.log(`Deleting ${element.StackName}`);
|
|
||||||
const deleteStackInput: DeleteStackCommandInput = { StackName: element.StackName };
|
|
||||||
await CF.send(new DeleteStackCommand(deleteStackInput));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const logGroups = await TaskService.getLogGroups();
|
|
||||||
for (const element of logGroups) {
|
|
||||||
if (
|
|
||||||
deleteResources &&
|
|
||||||
(!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.creationTime!)))
|
|
||||||
) {
|
|
||||||
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
|
|
||||||
await cwl.send(new DeleteLogGroupCommand({ logGroupName: element.logGroupName || '' }));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const locks = await TaskService.getLocks();
|
|
||||||
for (const element of locks) {
|
|
||||||
CloudRunnerLogger.log(`Lock: ${element.Key}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,209 +0,0 @@
|
|||||||
import {
|
|
||||||
CloudFormation,
|
|
||||||
DescribeStackResourcesCommand,
|
|
||||||
DescribeStacksCommand,
|
|
||||||
ListStacksCommand,
|
|
||||||
StackSummary,
|
|
||||||
} from '@aws-sdk/client-cloudformation';
|
|
||||||
import {
|
|
||||||
CloudWatchLogs,
|
|
||||||
DescribeLogGroupsCommand,
|
|
||||||
DescribeLogGroupsCommandInput,
|
|
||||||
LogGroup,
|
|
||||||
} from '@aws-sdk/client-cloudwatch-logs';
|
|
||||||
import {
|
|
||||||
DescribeTasksCommand,
|
|
||||||
DescribeTasksCommandInput,
|
|
||||||
ECS,
|
|
||||||
ListClustersCommand,
|
|
||||||
ListTasksCommand,
|
|
||||||
ListTasksCommandInput,
|
|
||||||
Task,
|
|
||||||
} from '@aws-sdk/client-ecs';
|
|
||||||
import { ListObjectsCommand, ListObjectsCommandInput, S3 } from '@aws-sdk/client-s3';
|
|
||||||
import Input from '../../../../input';
|
|
||||||
import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
|
|
||||||
import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
|
|
||||||
import AwsTaskRunner from '../aws-task-runner';
|
|
||||||
import CloudRunner from '../../../cloud-runner';
|
|
||||||
|
|
||||||
export class TaskService {
|
|
||||||
static async watch() {
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
const { output, shouldCleanup } = await AwsTaskRunner.streamLogsUntilTaskStops(
|
|
||||||
process.env.cluster || ``,
|
|
||||||
process.env.taskArn || ``,
|
|
||||||
process.env.streamName || ``,
|
|
||||||
);
|
|
||||||
|
|
||||||
return output;
|
|
||||||
}
|
|
||||||
public static async getCloudFormationJobStacks() {
|
|
||||||
const result: StackSummary[] = [];
|
|
||||||
CloudRunnerLogger.log(``);
|
|
||||||
CloudRunnerLogger.log(`List Cloud Formation Stacks`);
|
|
||||||
process.env.AWS_REGION = Input.region;
|
|
||||||
const CF = new CloudFormation({ region: Input.region });
|
|
||||||
const stacks =
|
|
||||||
(await CF.send(new ListStacksCommand({}))).StackSummaries?.filter(
|
|
||||||
(_x) =>
|
|
||||||
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription,
|
|
||||||
) || [];
|
|
||||||
CloudRunnerLogger.log(``);
|
|
||||||
CloudRunnerLogger.log(`Cloud Formation Stacks ${stacks.length}`);
|
|
||||||
for (const element of stacks) {
|
|
||||||
if (!element.CreationTime) {
|
|
||||||
CloudRunnerLogger.log(`${element.StackName} due to undefined CreationTime`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const ageDate: Date = new Date(Date.now() - (element.CreationTime?.getTime() ?? 0));
|
|
||||||
|
|
||||||
CloudRunnerLogger.log(
|
|
||||||
`Task Stack ${element.StackName} - Age D${Math.floor(
|
|
||||||
ageDate.getHours() / 24,
|
|
||||||
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
|
||||||
);
|
|
||||||
result.push(element);
|
|
||||||
}
|
|
||||||
const baseStacks =
|
|
||||||
(await CF.send(new ListStacksCommand({}))).StackSummaries?.filter(
|
|
||||||
(_x) =>
|
|
||||||
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription,
|
|
||||||
) || [];
|
|
||||||
CloudRunnerLogger.log(``);
|
|
||||||
CloudRunnerLogger.log(`Base Stacks ${baseStacks.length}`);
|
|
||||||
for (const element of baseStacks) {
|
|
||||||
if (!element.CreationTime) {
|
|
||||||
CloudRunnerLogger.log(`${element.StackName} due to undefined CreationTime`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const ageDate: Date = new Date(Date.now() - (element.CreationTime?.getTime() ?? 0));
|
|
||||||
|
|
||||||
CloudRunnerLogger.log(
|
|
||||||
`Task Stack ${element.StackName} - Age D${Math.floor(
|
|
||||||
ageDate.getHours() / 24,
|
|
||||||
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
|
||||||
);
|
|
||||||
result.push(element);
|
|
||||||
}
|
|
||||||
CloudRunnerLogger.log(``);
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
public static async getTasks() {
|
|
||||||
const result: { taskElement: Task; element: string }[] = [];
|
|
||||||
CloudRunnerLogger.log(``);
|
|
||||||
CloudRunnerLogger.log(`List Tasks`);
|
|
||||||
process.env.AWS_REGION = Input.region;
|
|
||||||
const ecs = new ECS({ region: Input.region });
|
|
||||||
const clusters = (await ecs.send(new ListClustersCommand({}))).clusterArns || [];
|
|
||||||
CloudRunnerLogger.log(`Task Clusters ${clusters.length}`);
|
|
||||||
for (const element of clusters) {
|
|
||||||
const input: ListTasksCommandInput = {
|
|
||||||
cluster: element,
|
|
||||||
};
|
|
||||||
|
|
||||||
const list = (await ecs.send(new ListTasksCommand(input))).taskArns || [];
|
|
||||||
if (list.length > 0) {
|
|
||||||
const describeInput: DescribeTasksCommandInput = { tasks: list, cluster: element };
|
|
||||||
const describeList = (await ecs.send(new DescribeTasksCommand(describeInput))).tasks || [];
|
|
||||||
if (describeList.length === 0) {
|
|
||||||
CloudRunnerLogger.log(`No Tasks`);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
CloudRunnerLogger.log(`Tasks ${describeList.length}`);
|
|
||||||
for (const taskElement of describeList) {
|
|
||||||
if (taskElement === undefined) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
taskElement.overrides = {};
|
|
||||||
taskElement.attachments = [];
|
|
||||||
if (taskElement.createdAt === undefined) {
|
|
||||||
CloudRunnerLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
result.push({ taskElement, element });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
CloudRunnerLogger.log(``);
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
public static async awsDescribeJob(job: string) {
|
|
||||||
process.env.AWS_REGION = Input.region;
|
|
||||||
const CF = new CloudFormation({ region: Input.region });
|
|
||||||
try {
|
|
||||||
const stack =
|
|
||||||
(await CF.send(new ListStacksCommand({}))).StackSummaries?.find((_x) => _x.StackName === job) || undefined;
|
|
||||||
const stackInfo = (await CF.send(new DescribeStackResourcesCommand({ StackName: job }))) || undefined;
|
|
||||||
const stackInfo2 = (await CF.send(new DescribeStacksCommand({ StackName: job }))) || undefined;
|
|
||||||
if (stack === undefined) {
|
|
||||||
throw new Error('stack not defined');
|
|
||||||
}
|
|
||||||
if (!stack.CreationTime) {
|
|
||||||
CloudRunnerLogger.log(`${stack.StackName} due to undefined CreationTime`);
|
|
||||||
}
|
|
||||||
const ageDate: Date = new Date(Date.now() - (stack.CreationTime?.getTime() ?? 0));
|
|
||||||
const message = `
|
|
||||||
Task Stack ${stack.StackName}
|
|
||||||
Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}
|
|
||||||
${JSON.stringify(stack, undefined, 4)}
|
|
||||||
${JSON.stringify(stackInfo, undefined, 4)}
|
|
||||||
${JSON.stringify(stackInfo2, undefined, 4)}
|
|
||||||
`;
|
|
||||||
CloudRunnerLogger.log(message);
|
|
||||||
|
|
||||||
return message;
|
|
||||||
} catch (error) {
|
|
||||||
CloudRunnerLogger.error(
|
|
||||||
`Failed to describe job ${job}: ${error instanceof Error ? error.message : String(error)}`,
|
|
||||||
);
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
public static async getLogGroups() {
|
|
||||||
const result: Array<LogGroup> = [];
|
|
||||||
process.env.AWS_REGION = Input.region;
|
|
||||||
const ecs = new CloudWatchLogs();
|
|
||||||
let logStreamInput: DescribeLogGroupsCommandInput = {
|
|
||||||
/* logGroupNamePrefix: 'game-ci' */
|
|
||||||
};
|
|
||||||
let logGroupsDescribe = await ecs.send(new DescribeLogGroupsCommand(logStreamInput));
|
|
||||||
const logGroups = logGroupsDescribe.logGroups || [];
|
|
||||||
while (logGroupsDescribe.nextToken) {
|
|
||||||
logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
|
|
||||||
logGroupsDescribe = await ecs.send(new DescribeLogGroupsCommand(logStreamInput));
|
|
||||||
logGroups.push(...(logGroupsDescribe?.logGroups || []));
|
|
||||||
}
|
|
||||||
|
|
||||||
CloudRunnerLogger.log(`Log Groups ${logGroups.length}`);
|
|
||||||
for (const element of logGroups) {
|
|
||||||
if (element.creationTime === undefined) {
|
|
||||||
CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const ageDate: Date = new Date(Date.now() - element.creationTime);
|
|
||||||
|
|
||||||
CloudRunnerLogger.log(
|
|
||||||
`Task Stack ${element.logGroupName} - Age D${Math.floor(
|
|
||||||
ageDate.getHours() / 24,
|
|
||||||
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
|
|
||||||
);
|
|
||||||
result.push(element);
|
|
||||||
}
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
public static async getLocks() {
|
|
||||||
process.env.AWS_REGION = Input.region;
|
|
||||||
const s3 = new S3({ region: Input.region });
|
|
||||||
const listRequest: ListObjectsCommandInput = {
|
|
||||||
Bucket: CloudRunner.buildParameters.awsStackName,
|
|
||||||
};
|
|
||||||
|
|
||||||
const results = await s3.send(new ListObjectsCommand(listRequest));
|
|
||||||
|
|
||||||
return results.Contents || [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,163 +0,0 @@
|
|||||||
import BuildParameters from '../../../build-parameters';
|
|
||||||
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
|
||||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
|
||||||
import { ProviderInterface } from '../provider-interface';
|
|
||||||
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
|
||||||
import Docker from '../../../docker';
|
|
||||||
import { Action } from '../../..';
|
|
||||||
import { writeFileSync } from 'node:fs';
|
|
||||||
import CloudRunner from '../../cloud-runner';
|
|
||||||
import { ProviderResource } from '../provider-resource';
|
|
||||||
import { ProviderWorkflow } from '../provider-workflow';
|
|
||||||
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
|
|
||||||
import * as fs from 'node:fs';
|
|
||||||
import { CommandHookService } from '../../services/hooks/command-hook-service';
|
|
||||||
import { StringKeyValuePair } from '../../../shared-types';
|
|
||||||
|
|
||||||
class LocalDockerCloudRunner implements ProviderInterface {
|
|
||||||
public buildParameters!: BuildParameters;
|
|
||||||
|
|
||||||
listResources(): Promise<ProviderResource[]> {
|
|
||||||
return new Promise((resolve) => resolve([]));
|
|
||||||
}
|
|
||||||
listWorkflow(): Promise<ProviderWorkflow[]> {
|
|
||||||
throw new Error('Method not implemented.');
|
|
||||||
}
|
|
||||||
watchWorkflow(): Promise<string> {
|
|
||||||
throw new Error('Method not implemented.');
|
|
||||||
}
|
|
||||||
garbageCollect(
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
filter: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
previewOnly: boolean,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
olderThan: Number,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
fullCache: boolean,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
baseDependencies: boolean,
|
|
||||||
): Promise<string> {
|
|
||||||
return new Promise((result) => result(``));
|
|
||||||
}
|
|
||||||
async cleanupWorkflow(
|
|
||||||
buildParameters: BuildParameters,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
branchName: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
|
||||||
) {
|
|
||||||
const { workspace } = Action;
|
|
||||||
if (
|
|
||||||
fs.existsSync(
|
|
||||||
`${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
|
|
||||||
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
|
||||||
}`,
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache/cache/build/`);
|
|
||||||
await CloudRunnerSystem.Run(
|
|
||||||
`rm -r ${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
|
|
||||||
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
|
||||||
}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
setupWorkflow(
|
|
||||||
buildGuid: string,
|
|
||||||
buildParameters: BuildParameters,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
branchName: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
|
||||||
) {
|
|
||||||
this.buildParameters = buildParameters;
|
|
||||||
}
|
|
||||||
|
|
||||||
public async runTaskInWorkflow(
|
|
||||||
buildGuid: string,
|
|
||||||
image: string,
|
|
||||||
commands: string,
|
|
||||||
mountdir: string,
|
|
||||||
workingdir: string,
|
|
||||||
environment: CloudRunnerEnvironmentVariable[],
|
|
||||||
secrets: CloudRunnerSecret[],
|
|
||||||
): Promise<string> {
|
|
||||||
CloudRunnerLogger.log(buildGuid);
|
|
||||||
CloudRunnerLogger.log(commands);
|
|
||||||
|
|
||||||
const { workspace, actionFolder } = Action;
|
|
||||||
const content: StringKeyValuePair[] = [];
|
|
||||||
for (const x of secrets) {
|
|
||||||
content.push({ name: x.EnvironmentVariable, value: x.ParameterValue });
|
|
||||||
}
|
|
||||||
for (const x of environment) {
|
|
||||||
content.push({ name: x.name, value: x.value });
|
|
||||||
}
|
|
||||||
|
|
||||||
// if (this.buildParameters?.cloudRunnerIntegrationTests) {
|
|
||||||
// core.info(JSON.stringify(content, undefined, 4));
|
|
||||||
// core.info(JSON.stringify(secrets, undefined, 4));
|
|
||||||
// core.info(JSON.stringify(environment, undefined, 4));
|
|
||||||
// }
|
|
||||||
|
|
||||||
// eslint-disable-next-line unicorn/no-for-loop
|
|
||||||
for (let index = 0; index < content.length; index++) {
|
|
||||||
if (content[index] === undefined) {
|
|
||||||
delete content[index];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let myOutput = '';
|
|
||||||
const sharedFolder = `/data/`;
|
|
||||||
|
|
||||||
// core.info(JSON.stringify({ workspace, actionFolder, ...this.buildParameters, ...content }, undefined, 4));
|
|
||||||
const entrypointFilePath = `start.sh`;
|
|
||||||
const fileContents = `#!/bin/bash
|
|
||||||
set -e
|
|
||||||
|
|
||||||
mkdir -p /github/workspace/cloud-runner-cache
|
|
||||||
mkdir -p /data/cache
|
|
||||||
cp -a /github/workspace/cloud-runner-cache/. ${sharedFolder}
|
|
||||||
${CommandHookService.ApplyHooksToCommands(commands, this.buildParameters)}
|
|
||||||
cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
|
|
||||||
`;
|
|
||||||
writeFileSync(`${workspace}/${entrypointFilePath}`, fileContents, {
|
|
||||||
flag: 'w',
|
|
||||||
});
|
|
||||||
|
|
||||||
if (CloudRunner.buildParameters.cloudRunnerDebug) {
|
|
||||||
CloudRunnerLogger.log(`Running local-docker: \n ${fileContents}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (fs.existsSync(`${workspace}/cloud-runner-cache`)) {
|
|
||||||
await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache && du -sh ${workspace}/cloud-runner-cache`);
|
|
||||||
}
|
|
||||||
const exitCode = await Docker.run(
|
|
||||||
image,
|
|
||||||
{ workspace, actionFolder, ...this.buildParameters },
|
|
||||||
false,
|
|
||||||
`chmod +x /github/workspace/${entrypointFilePath} && /github/workspace/${entrypointFilePath}`,
|
|
||||||
content,
|
|
||||||
{
|
|
||||||
listeners: {
|
|
||||||
stdout: (data: Buffer) => {
|
|
||||||
myOutput += data.toString();
|
|
||||||
},
|
|
||||||
stderr: (data: Buffer) => {
|
|
||||||
myOutput += `[LOCAL-DOCKER-ERROR]${data.toString()}`;
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
true,
|
|
||||||
);
|
|
||||||
|
|
||||||
// Docker doesn't exit on fail now so adding this to ensure behavior is unchanged
|
|
||||||
// TODO: Is there a helpful way to consume the exit code or is it best to except
|
|
||||||
if (exitCode !== 0) {
|
|
||||||
throw new Error(`Build failed with exit code ${exitCode}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return myOutput;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
export default LocalDockerCloudRunner;
|
|
||||||
@@ -1,337 +0,0 @@
|
|||||||
import * as k8s from '@kubernetes/client-node';
|
|
||||||
import { BuildParameters } from '../../..';
|
|
||||||
import * as core from '@actions/core';
|
|
||||||
import { ProviderInterface } from '../provider-interface';
|
|
||||||
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
|
||||||
import KubernetesStorage from './kubernetes-storage';
|
|
||||||
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
|
||||||
import KubernetesTaskRunner from './kubernetes-task-runner';
|
|
||||||
import KubernetesSecret from './kubernetes-secret';
|
|
||||||
import KubernetesJobSpecFactory from './kubernetes-job-spec-factory';
|
|
||||||
import KubernetesServiceAccount from './kubernetes-service-account';
|
|
||||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
|
||||||
import { CoreV1Api } from '@kubernetes/client-node';
|
|
||||||
import CloudRunner from '../../cloud-runner';
|
|
||||||
import { ProviderResource } from '../provider-resource';
|
|
||||||
import { ProviderWorkflow } from '../provider-workflow';
|
|
||||||
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
|
|
||||||
import { KubernetesRole } from './kubernetes-role';
|
|
||||||
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
|
|
||||||
|
|
||||||
class Kubernetes implements ProviderInterface {
|
|
||||||
public static Instance: Kubernetes;
|
|
||||||
public kubeConfig!: k8s.KubeConfig;
|
|
||||||
public kubeClient!: k8s.CoreV1Api;
|
|
||||||
public kubeClientApps!: k8s.AppsV1Api;
|
|
||||||
public kubeClientBatch!: k8s.BatchV1Api;
|
|
||||||
public rbacAuthorizationV1Api!: k8s.RbacAuthorizationV1Api;
|
|
||||||
public buildGuid: string = '';
|
|
||||||
public buildParameters!: BuildParameters;
|
|
||||||
public pvcName: string = '';
|
|
||||||
public secretName: string = '';
|
|
||||||
public jobName: string = '';
|
|
||||||
public namespace!: string;
|
|
||||||
public podName: string = '';
|
|
||||||
public containerName: string = '';
|
|
||||||
public cleanupCronJobName: string = '';
|
|
||||||
public serviceAccountName: string = '';
|
|
||||||
public ip: string = '';
|
|
||||||
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
constructor(buildParameters: BuildParameters) {
|
|
||||||
Kubernetes.Instance = this;
|
|
||||||
this.kubeConfig = new k8s.KubeConfig();
|
|
||||||
this.kubeConfig.loadFromDefault();
|
|
||||||
this.kubeClient = this.kubeConfig.makeApiClient(k8s.CoreV1Api);
|
|
||||||
this.kubeClientApps = this.kubeConfig.makeApiClient(k8s.AppsV1Api);
|
|
||||||
this.kubeClientBatch = this.kubeConfig.makeApiClient(k8s.BatchV1Api);
|
|
||||||
this.rbacAuthorizationV1Api = this.kubeConfig.makeApiClient(k8s.RbacAuthorizationV1Api);
|
|
||||||
this.namespace = 'default';
|
|
||||||
CloudRunnerLogger.log('Loaded default Kubernetes configuration for this environment');
|
|
||||||
}
|
|
||||||
|
|
||||||
async PushLogUpdate(logs: string) {
|
|
||||||
// push logs to nginx file server via 'LOG_SERVICE_IP' env var
|
|
||||||
const ip = process.env[`LOG_SERVICE_IP`];
|
|
||||||
if (ip === undefined) {
|
|
||||||
RemoteClientLogger.logWarning(`LOG_SERVICE_IP not set, skipping log push`);
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const url = `http://${ip}/api/log`;
|
|
||||||
RemoteClientLogger.log(`Pushing logs to ${url}`);
|
|
||||||
|
|
||||||
// logs to base64
|
|
||||||
logs = Buffer.from(logs).toString('base64');
|
|
||||||
const response = await CloudRunnerSystem.Run(`curl -X POST -d "${logs}" ${url}`, false, true);
|
|
||||||
RemoteClientLogger.log(`Pushed logs to ${url} ${response}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
async listResources(): Promise<ProviderResource[]> {
|
|
||||||
const pods = await this.kubeClient.listNamespacedPod(this.namespace);
|
|
||||||
const serviceAccounts = await this.kubeClient.listNamespacedServiceAccount(this.namespace);
|
|
||||||
const secrets = await this.kubeClient.listNamespacedSecret(this.namespace);
|
|
||||||
const jobs = await this.kubeClientBatch.listNamespacedJob(this.namespace);
|
|
||||||
|
|
||||||
return [
|
|
||||||
...pods.body.items.map((x) => {
|
|
||||||
return { Name: x.metadata?.name || `` };
|
|
||||||
}),
|
|
||||||
...serviceAccounts.body.items.map((x) => {
|
|
||||||
return { Name: x.metadata?.name || `` };
|
|
||||||
}),
|
|
||||||
...secrets.body.items.map((x) => {
|
|
||||||
return { Name: x.metadata?.name || `` };
|
|
||||||
}),
|
|
||||||
...jobs.body.items.map((x) => {
|
|
||||||
return { Name: x.metadata?.name || `` };
|
|
||||||
}),
|
|
||||||
];
|
|
||||||
}
|
|
||||||
listWorkflow(): Promise<ProviderWorkflow[]> {
|
|
||||||
throw new Error('Method not implemented.');
|
|
||||||
}
|
|
||||||
watchWorkflow(): Promise<string> {
|
|
||||||
throw new Error('Method not implemented.');
|
|
||||||
}
|
|
||||||
garbageCollect(
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
filter: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
previewOnly: boolean,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
olderThan: Number,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
fullCache: boolean,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
baseDependencies: boolean,
|
|
||||||
): Promise<string> {
|
|
||||||
return new Promise((result) => result(``));
|
|
||||||
}
|
|
||||||
public async setupWorkflow(
|
|
||||||
buildGuid: string,
|
|
||||||
buildParameters: BuildParameters,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
branchName: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
|
||||||
) {
|
|
||||||
try {
|
|
||||||
this.buildParameters = buildParameters;
|
|
||||||
this.cleanupCronJobName = `unity-builder-cronjob-${buildParameters.buildGuid}`;
|
|
||||||
this.serviceAccountName = `service-account-${buildParameters.buildGuid}`;
|
|
||||||
|
|
||||||
await KubernetesServiceAccount.createServiceAccount(this.serviceAccountName, this.namespace, this.kubeClient);
|
|
||||||
} catch (error) {
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async runTaskInWorkflow(
|
|
||||||
buildGuid: string,
|
|
||||||
image: string,
|
|
||||||
commands: string,
|
|
||||||
mountdir: string,
|
|
||||||
workingdir: string,
|
|
||||||
environment: CloudRunnerEnvironmentVariable[],
|
|
||||||
secrets: CloudRunnerSecret[],
|
|
||||||
): Promise<string> {
|
|
||||||
try {
|
|
||||||
CloudRunnerLogger.log('Cloud Runner K8s workflow!');
|
|
||||||
|
|
||||||
// Setup
|
|
||||||
const id =
|
|
||||||
BuildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(this.buildParameters)
|
|
||||||
? CloudRunner.lockedWorkspace
|
|
||||||
: this.buildParameters.buildGuid;
|
|
||||||
this.pvcName = `unity-builder-pvc-${id}`;
|
|
||||||
await KubernetesStorage.createPersistentVolumeClaim(
|
|
||||||
this.buildParameters,
|
|
||||||
this.pvcName,
|
|
||||||
this.kubeClient,
|
|
||||||
this.namespace,
|
|
||||||
);
|
|
||||||
this.buildGuid = buildGuid;
|
|
||||||
this.secretName = `build-credentials-${this.buildGuid}`;
|
|
||||||
this.jobName = `unity-builder-job-${this.buildGuid}`;
|
|
||||||
this.containerName = `main`;
|
|
||||||
await KubernetesSecret.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
|
|
||||||
let output = '';
|
|
||||||
try {
|
|
||||||
CloudRunnerLogger.log('Job does not exist');
|
|
||||||
await this.createJob(commands, image, mountdir, workingdir, environment, secrets);
|
|
||||||
CloudRunnerLogger.log('Watching pod until running');
|
|
||||||
await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
|
|
||||||
|
|
||||||
CloudRunnerLogger.log('Pod is running');
|
|
||||||
output += await KubernetesTaskRunner.runTask(
|
|
||||||
this.kubeConfig,
|
|
||||||
this.kubeClient,
|
|
||||||
this.jobName,
|
|
||||||
this.podName,
|
|
||||||
this.containerName,
|
|
||||||
this.namespace,
|
|
||||||
);
|
|
||||||
} catch (error: any) {
|
|
||||||
CloudRunnerLogger.log(`error running k8s workflow ${error}`);
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
|
||||||
CloudRunnerLogger.log(
|
|
||||||
JSON.stringify(
|
|
||||||
(await this.kubeClient.listNamespacedEvent(this.namespace)).body.items
|
|
||||||
.map((x) => {
|
|
||||||
return {
|
|
||||||
message: x.message || ``,
|
|
||||||
name: x.metadata.name || ``,
|
|
||||||
reason: x.reason || ``,
|
|
||||||
};
|
|
||||||
})
|
|
||||||
.filter((x) => x.name.includes(this.podName)),
|
|
||||||
undefined,
|
|
||||||
4,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
await this.cleanupTaskResources();
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
|
|
||||||
await this.cleanupTaskResources();
|
|
||||||
|
|
||||||
return output;
|
|
||||||
} catch (error) {
|
|
||||||
CloudRunnerLogger.log('Running job failed');
|
|
||||||
core.error(JSON.stringify(error, undefined, 4));
|
|
||||||
|
|
||||||
// await this.cleanupTaskResources();
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async createJob(
|
|
||||||
commands: string,
|
|
||||||
image: string,
|
|
||||||
mountdir: string,
|
|
||||||
workingdir: string,
|
|
||||||
environment: CloudRunnerEnvironmentVariable[],
|
|
||||||
secrets: CloudRunnerSecret[],
|
|
||||||
) {
|
|
||||||
await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
|
|
||||||
const find = await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace);
|
|
||||||
this.setPodNameAndContainerName(find);
|
|
||||||
}
|
|
||||||
|
|
||||||
private async doesJobExist(name: string) {
|
|
||||||
const jobs = await this.kubeClientBatch.listNamespacedJob(this.namespace);
|
|
||||||
|
|
||||||
return jobs.body.items.some((x) => x.metadata?.name === name);
|
|
||||||
}
|
|
||||||
|
|
||||||
private async doesFailedJobExist() {
|
|
||||||
const podStatus = await this.kubeClient.readNamespacedPodStatus(this.podName, this.namespace);
|
|
||||||
|
|
||||||
return podStatus.body.status?.phase === `Failed`;
|
|
||||||
}
|
|
||||||
|
|
||||||
private async createNamespacedJob(
|
|
||||||
commands: string,
|
|
||||||
image: string,
|
|
||||||
mountdir: string,
|
|
||||||
workingdir: string,
|
|
||||||
environment: CloudRunnerEnvironmentVariable[],
|
|
||||||
secrets: CloudRunnerSecret[],
|
|
||||||
) {
|
|
||||||
for (let index = 0; index < 3; index++) {
|
|
||||||
try {
|
|
||||||
const jobSpec = KubernetesJobSpecFactory.getJobSpec(
|
|
||||||
commands,
|
|
||||||
image,
|
|
||||||
mountdir,
|
|
||||||
workingdir,
|
|
||||||
environment,
|
|
||||||
secrets,
|
|
||||||
this.buildGuid,
|
|
||||||
this.buildParameters,
|
|
||||||
this.secretName,
|
|
||||||
this.pvcName,
|
|
||||||
this.jobName,
|
|
||||||
k8s,
|
|
||||||
this.containerName,
|
|
||||||
this.ip,
|
|
||||||
);
|
|
||||||
await new Promise((promise) => setTimeout(promise, 15000));
|
|
||||||
|
|
||||||
// await KubernetesRole.createRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api);
|
|
||||||
|
|
||||||
const result = await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
|
|
||||||
CloudRunnerLogger.log(`Build job created`);
|
|
||||||
await new Promise((promise) => setTimeout(promise, 5000));
|
|
||||||
CloudRunnerLogger.log('Job created');
|
|
||||||
|
|
||||||
return result.body.metadata?.name;
|
|
||||||
} catch (error) {
|
|
||||||
CloudRunnerLogger.log(`Error occured creating job: ${error}`);
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
setPodNameAndContainerName(pod: k8s.V1Pod) {
|
|
||||||
this.podName = pod.metadata?.name || '';
|
|
||||||
this.containerName = pod.status?.containerStatuses?.[0].name || this.containerName;
|
|
||||||
}
|
|
||||||
|
|
||||||
async cleanupTaskResources() {
|
|
||||||
CloudRunnerLogger.log('cleaning up');
|
|
||||||
try {
|
|
||||||
await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
|
|
||||||
await this.kubeClient.deleteNamespacedPod(this.podName, this.namespace);
|
|
||||||
await KubernetesRole.deleteRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api);
|
|
||||||
} catch (error: any) {
|
|
||||||
CloudRunnerLogger.log(`Failed to cleanup`);
|
|
||||||
if (error.response.body.reason !== `NotFound`) {
|
|
||||||
CloudRunnerLogger.log(`Wasn't a not found error: ${error.response.body.reason}`);
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
await this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace);
|
|
||||||
} catch (error: any) {
|
|
||||||
CloudRunnerLogger.log(`Failed to cleanup secret`);
|
|
||||||
CloudRunnerLogger.log(error.response.body.reason);
|
|
||||||
}
|
|
||||||
CloudRunnerLogger.log('cleaned up Secret, Job and Pod');
|
|
||||||
CloudRunnerLogger.log('cleaning up finished');
|
|
||||||
}
|
|
||||||
|
|
||||||
async cleanupWorkflow(
|
|
||||||
buildParameters: BuildParameters,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
branchName: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
|
||||||
) {
|
|
||||||
if (BuildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
CloudRunnerLogger.log(`deleting PVC`);
|
|
||||||
|
|
||||||
try {
|
|
||||||
await this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace);
|
|
||||||
await this.kubeClient.deleteNamespacedServiceAccount(this.serviceAccountName, this.namespace);
|
|
||||||
CloudRunnerLogger.log('cleaned up PVC and Service Account');
|
|
||||||
} catch (error: any) {
|
|
||||||
CloudRunnerLogger.log(`Cleanup failed ${JSON.stringify(error, undefined, 4)}`);
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static async findPodFromJob(kubeClient: CoreV1Api, jobName: string, namespace: string) {
|
|
||||||
const namespacedPods = await kubeClient.listNamespacedPod(namespace);
|
|
||||||
const pod = namespacedPods.body.items.find((x) => x.metadata?.labels?.['job-name'] === jobName);
|
|
||||||
if (pod === undefined) {
|
|
||||||
throw new Error("pod with job-name label doesn't exist");
|
|
||||||
}
|
|
||||||
|
|
||||||
return pod;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
export default Kubernetes;
|
|
||||||
@@ -1,127 +0,0 @@
|
|||||||
import { V1EnvVar, V1EnvVarSource, V1SecretKeySelector } from '@kubernetes/client-node';
|
|
||||||
import BuildParameters from '../../../build-parameters';
|
|
||||||
import { CommandHookService } from '../../services/hooks/command-hook-service';
|
|
||||||
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
|
||||||
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
|
||||||
import CloudRunner from '../../cloud-runner';
|
|
||||||
|
|
||||||
class KubernetesJobSpecFactory {
|
|
||||||
static getJobSpec(
|
|
||||||
command: string,
|
|
||||||
image: string,
|
|
||||||
mountdir: string,
|
|
||||||
workingDirectory: string,
|
|
||||||
environment: CloudRunnerEnvironmentVariable[],
|
|
||||||
secrets: CloudRunnerSecret[],
|
|
||||||
buildGuid: string,
|
|
||||||
buildParameters: BuildParameters,
|
|
||||||
secretName: string,
|
|
||||||
pvcName: string,
|
|
||||||
jobName: string,
|
|
||||||
k8s: any,
|
|
||||||
containerName: string,
|
|
||||||
ip: string = '',
|
|
||||||
) {
|
|
||||||
const job = new k8s.V1Job();
|
|
||||||
job.apiVersion = 'batch/v1';
|
|
||||||
job.kind = 'Job';
|
|
||||||
job.metadata = {
|
|
||||||
name: jobName,
|
|
||||||
labels: {
|
|
||||||
app: 'unity-builder',
|
|
||||||
buildGuid,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
job.spec = {
|
|
||||||
ttlSecondsAfterFinished: 9999,
|
|
||||||
backoffLimit: 0,
|
|
||||||
template: {
|
|
||||||
spec: {
|
|
||||||
volumes: [
|
|
||||||
{
|
|
||||||
name: 'build-mount',
|
|
||||||
persistentVolumeClaim: {
|
|
||||||
claimName: pvcName,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
containers: [
|
|
||||||
{
|
|
||||||
ttlSecondsAfterFinished: 9999,
|
|
||||||
name: containerName,
|
|
||||||
image,
|
|
||||||
command: ['/bin/sh'],
|
|
||||||
args: [
|
|
||||||
'-c',
|
|
||||||
`${CommandHookService.ApplyHooksToCommands(`${command}\nsleep 2m`, CloudRunner.buildParameters)}`,
|
|
||||||
],
|
|
||||||
|
|
||||||
workingDir: `${workingDirectory}`,
|
|
||||||
resources: {
|
|
||||||
requests: {
|
|
||||||
memory: `${Number.parseInt(buildParameters.containerMemory) / 1024}G` || '750M',
|
|
||||||
cpu: Number.parseInt(buildParameters.containerCpu) / 1024 || '1',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
env: [
|
|
||||||
...environment.map((x) => {
|
|
||||||
const environmentVariable = new V1EnvVar();
|
|
||||||
environmentVariable.name = x.name;
|
|
||||||
environmentVariable.value = x.value;
|
|
||||||
|
|
||||||
return environmentVariable;
|
|
||||||
}),
|
|
||||||
...secrets.map((x) => {
|
|
||||||
const secret = new V1EnvVarSource();
|
|
||||||
secret.secretKeyRef = new V1SecretKeySelector();
|
|
||||||
secret.secretKeyRef.key = x.ParameterKey;
|
|
||||||
secret.secretKeyRef.name = secretName;
|
|
||||||
const environmentVariable = new V1EnvVar();
|
|
||||||
environmentVariable.name = x.EnvironmentVariable;
|
|
||||||
environmentVariable.valueFrom = secret;
|
|
||||||
|
|
||||||
return environmentVariable;
|
|
||||||
}),
|
|
||||||
{ name: 'LOG_SERVICE_IP', value: ip },
|
|
||||||
],
|
|
||||||
volumeMounts: [
|
|
||||||
{
|
|
||||||
name: 'build-mount',
|
|
||||||
mountPath: `${mountdir}`,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
lifecycle: {
|
|
||||||
preStop: {
|
|
||||||
exec: {
|
|
||||||
command: [
|
|
||||||
`wait 60s;
|
|
||||||
cd /data/builder/action/steps;
|
|
||||||
chmod +x /return_license.sh;
|
|
||||||
/return_license.sh;`,
|
|
||||||
],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
restartPolicy: 'Never',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
if (process.env['CLOUD_RUNNER_MINIKUBE']) {
|
|
||||||
job.spec.template.spec.volumes[0] = {
|
|
||||||
name: 'build-mount',
|
|
||||||
hostPath: {
|
|
||||||
path: `/data`,
|
|
||||||
type: `Directory`,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '10Gi';
|
|
||||||
|
|
||||||
return job;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
export default KubernetesJobSpecFactory;
|
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
|
||||||
import { CoreV1Api } from '@kubernetes/client-node';
|
|
||||||
class KubernetesPods {
|
|
||||||
public static async IsPodRunning(podName: string, namespace: string, kubeClient: CoreV1Api) {
|
|
||||||
const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.filter((x) => podName === x.metadata?.name);
|
|
||||||
const running = pods.length > 0 && (pods[0].status?.phase === `Running` || pods[0].status?.phase === `Pending`);
|
|
||||||
const phase = pods[0]?.status?.phase || 'undefined status';
|
|
||||||
CloudRunnerLogger.log(`Getting pod status: ${phase}`);
|
|
||||||
if (phase === `Failed`) {
|
|
||||||
throw new Error(`K8s pod failed`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return running;
|
|
||||||
}
|
|
||||||
public static async GetPodStatus(podName: string, namespace: string, kubeClient: CoreV1Api) {
|
|
||||||
const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.find((x) => podName === x.metadata?.name);
|
|
||||||
const phase = pods?.status?.phase || 'undefined status';
|
|
||||||
|
|
||||||
return phase;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export default KubernetesPods;
|
|
||||||
@@ -1,53 +0,0 @@
|
|||||||
import { RbacAuthorizationV1Api } from '@kubernetes/client-node';
|
|
||||||
|
|
||||||
class KubernetesRole {
|
|
||||||
static async createRole(serviceAccountName: string, namespace: string, rbac: RbacAuthorizationV1Api) {
|
|
||||||
// create admin kubernetes role and role binding
|
|
||||||
const roleBinding = {
|
|
||||||
apiVersion: 'rbac.authorization.k8s.io/v1',
|
|
||||||
kind: 'RoleBinding',
|
|
||||||
metadata: {
|
|
||||||
name: `${serviceAccountName}-admin`,
|
|
||||||
namespace,
|
|
||||||
},
|
|
||||||
subjects: [
|
|
||||||
{
|
|
||||||
kind: 'ServiceAccount',
|
|
||||||
name: serviceAccountName,
|
|
||||||
namespace,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
roleRef: {
|
|
||||||
apiGroup: 'rbac.authorization.k8s.io',
|
|
||||||
kind: 'Role',
|
|
||||||
name: `${serviceAccountName}-admin`,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
const role = {
|
|
||||||
apiVersion: 'rbac.authorization.k8s.io/v1',
|
|
||||||
kind: 'Role',
|
|
||||||
metadata: {
|
|
||||||
name: `${serviceAccountName}-admin`,
|
|
||||||
namespace,
|
|
||||||
},
|
|
||||||
rules: [
|
|
||||||
{
|
|
||||||
apiGroups: ['*'],
|
|
||||||
resources: ['*'],
|
|
||||||
verbs: ['*'],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
|
||||||
const roleBindingResponse = await rbac.createNamespacedRoleBinding(namespace, roleBinding);
|
|
||||||
const roleResponse = await rbac.createNamespacedRole(namespace, role);
|
|
||||||
|
|
||||||
return { roleBindingResponse, roleResponse };
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async deleteRole(serviceAccountName: string, namespace: string, rbac: RbacAuthorizationV1Api) {
|
|
||||||
await rbac.deleteNamespacedRoleBinding(`${serviceAccountName}-admin`, namespace);
|
|
||||||
await rbac.deleteNamespacedRole(`${serviceAccountName}-admin`, namespace);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
export { KubernetesRole };
|
|
||||||
@@ -1,45 +0,0 @@
|
|||||||
import { CoreV1Api } from '@kubernetes/client-node';
|
|
||||||
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
|
||||||
import * as k8s from '@kubernetes/client-node';
|
|
||||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
|
||||||
import * as base64 from 'base-64';
|
|
||||||
|
|
||||||
class KubernetesSecret {
|
|
||||||
static async createSecret(
|
|
||||||
secrets: CloudRunnerSecret[],
|
|
||||||
secretName: string,
|
|
||||||
namespace: string,
|
|
||||||
kubeClient: CoreV1Api,
|
|
||||||
) {
|
|
||||||
try {
|
|
||||||
const secret = new k8s.V1Secret();
|
|
||||||
secret.apiVersion = 'v1';
|
|
||||||
secret.kind = 'Secret';
|
|
||||||
secret.type = 'Opaque';
|
|
||||||
secret.metadata = {
|
|
||||||
name: secretName,
|
|
||||||
};
|
|
||||||
secret.data = {};
|
|
||||||
for (const buildSecret of secrets) {
|
|
||||||
secret.data[buildSecret.ParameterKey] = base64.encode(buildSecret.ParameterValue);
|
|
||||||
}
|
|
||||||
CloudRunnerLogger.log(`Creating secret: ${secretName}`);
|
|
||||||
const existingSecrets = await kubeClient.listNamespacedSecret(namespace);
|
|
||||||
const mappedSecrets = existingSecrets.body.items.map((x) => {
|
|
||||||
return x.metadata?.name || `no name`;
|
|
||||||
});
|
|
||||||
|
|
||||||
CloudRunnerLogger.log(
|
|
||||||
`ExistsAlready: ${mappedSecrets.includes(secretName)} SecretsCount: ${mappedSecrets.length}`,
|
|
||||||
);
|
|
||||||
await new Promise((promise) => setTimeout(promise, 15000));
|
|
||||||
await kubeClient.createNamespacedSecret(namespace, secret);
|
|
||||||
CloudRunnerLogger.log('Created secret');
|
|
||||||
} catch (error) {
|
|
||||||
CloudRunnerLogger.log(`Created secret failed ${error}`);
|
|
||||||
throw new Error(`Failed to create kubernetes secret`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export default KubernetesSecret;
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
import { CoreV1Api } from '@kubernetes/client-node';
|
|
||||||
import * as k8s from '@kubernetes/client-node';
|
|
||||||
|
|
||||||
class KubernetesServiceAccount {
|
|
||||||
static async createServiceAccount(serviceAccountName: string, namespace: string, kubeClient: CoreV1Api) {
|
|
||||||
const serviceAccount = new k8s.V1ServiceAccount();
|
|
||||||
serviceAccount.apiVersion = 'v1';
|
|
||||||
serviceAccount.kind = 'ServiceAccount';
|
|
||||||
serviceAccount.metadata = {
|
|
||||||
name: serviceAccountName,
|
|
||||||
};
|
|
||||||
serviceAccount.automountServiceAccountToken = true;
|
|
||||||
|
|
||||||
return kubeClient.createNamespacedServiceAccount(namespace, serviceAccount);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export default KubernetesServiceAccount;
|
|
||||||
@@ -1,116 +0,0 @@
|
|||||||
import { waitUntil } from 'async-wait-until';
|
|
||||||
import * as core from '@actions/core';
|
|
||||||
import * as k8s from '@kubernetes/client-node';
|
|
||||||
import BuildParameters from '../../../build-parameters';
|
|
||||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
|
||||||
import { IncomingMessage } from 'node:http';
|
|
||||||
import GitHub from '../../../github';
|
|
||||||
|
|
||||||
class KubernetesStorage {
|
|
||||||
public static async createPersistentVolumeClaim(
|
|
||||||
buildParameters: BuildParameters,
|
|
||||||
pvcName: string,
|
|
||||||
kubeClient: k8s.CoreV1Api,
|
|
||||||
namespace: string,
|
|
||||||
) {
|
|
||||||
if (buildParameters.kubeVolume !== ``) {
|
|
||||||
CloudRunnerLogger.log(`Kube Volume was input was set ${buildParameters.kubeVolume} overriding ${pvcName}`);
|
|
||||||
pvcName = buildParameters.kubeVolume;
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const allPvc = (await kubeClient.listNamespacedPersistentVolumeClaim(namespace)).body.items;
|
|
||||||
const pvcList = allPvc.map((x) => x.metadata?.name);
|
|
||||||
CloudRunnerLogger.log(`Current PVCs in namespace ${namespace}`);
|
|
||||||
CloudRunnerLogger.log(JSON.stringify(pvcList, undefined, 4));
|
|
||||||
if (pvcList.includes(pvcName)) {
|
|
||||||
CloudRunnerLogger.log(`pvc ${pvcName} already exists`);
|
|
||||||
if (GitHub.githubInputEnabled) {
|
|
||||||
core.setOutput('volume', pvcName);
|
|
||||||
}
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
CloudRunnerLogger.log(`Creating PVC ${pvcName} (does not exist)`);
|
|
||||||
const result = await KubernetesStorage.createPVC(pvcName, buildParameters, kubeClient, namespace);
|
|
||||||
await KubernetesStorage.handleResult(result, kubeClient, namespace, pvcName);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async getPVCPhase(kubeClient: k8s.CoreV1Api, name: string, namespace: string) {
|
|
||||||
try {
|
|
||||||
return (await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace)).body.status?.phase;
|
|
||||||
} catch (error) {
|
|
||||||
core.error('Failed to get PVC phase');
|
|
||||||
core.error(JSON.stringify(error, undefined, 4));
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async watchUntilPVCNotPending(kubeClient: k8s.CoreV1Api, name: string, namespace: string) {
|
|
||||||
try {
|
|
||||||
CloudRunnerLogger.log(`watch Until PVC Not Pending ${name} ${namespace}`);
|
|
||||||
CloudRunnerLogger.log(`${await this.getPVCPhase(kubeClient, name, namespace)}`);
|
|
||||||
await waitUntil(
|
|
||||||
async () => {
|
|
||||||
return (await this.getPVCPhase(kubeClient, name, namespace)) === 'Pending';
|
|
||||||
},
|
|
||||||
{
|
|
||||||
timeout: 750000,
|
|
||||||
intervalBetweenAttempts: 15000,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
} catch (error: any) {
|
|
||||||
core.error('Failed to watch PVC');
|
|
||||||
core.error(error.toString());
|
|
||||||
core.error(
|
|
||||||
`PVC Body: ${JSON.stringify(
|
|
||||||
(await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace)).body,
|
|
||||||
undefined,
|
|
||||||
4,
|
|
||||||
)}`,
|
|
||||||
);
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static async createPVC(
|
|
||||||
pvcName: string,
|
|
||||||
buildParameters: BuildParameters,
|
|
||||||
kubeClient: k8s.CoreV1Api,
|
|
||||||
namespace: string,
|
|
||||||
) {
|
|
||||||
const pvc = new k8s.V1PersistentVolumeClaim();
|
|
||||||
pvc.apiVersion = 'v1';
|
|
||||||
pvc.kind = 'PersistentVolumeClaim';
|
|
||||||
pvc.metadata = {
|
|
||||||
name: pvcName,
|
|
||||||
};
|
|
||||||
pvc.spec = {
|
|
||||||
accessModes: ['ReadWriteOnce'],
|
|
||||||
storageClassName: buildParameters.kubeStorageClass === '' ? 'standard' : buildParameters.kubeStorageClass,
|
|
||||||
resources: {
|
|
||||||
requests: {
|
|
||||||
storage: buildParameters.kubeVolumeSize,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
};
|
|
||||||
const result = await kubeClient.createNamespacedPersistentVolumeClaim(namespace, pvc);
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static async handleResult(
|
|
||||||
result: { response: IncomingMessage; body: k8s.V1PersistentVolumeClaim },
|
|
||||||
kubeClient: k8s.CoreV1Api,
|
|
||||||
namespace: string,
|
|
||||||
pvcName: string,
|
|
||||||
) {
|
|
||||||
const name = result.body.metadata?.name || '';
|
|
||||||
CloudRunnerLogger.log(`PVC ${name} created`);
|
|
||||||
await this.watchUntilPVCNotPending(kubeClient, name, namespace);
|
|
||||||
CloudRunnerLogger.log(`PVC ${name} is ready and not pending`);
|
|
||||||
core.setOutput('volume', pvcName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export default KubernetesStorage;
|
|
||||||
@@ -1,119 +0,0 @@
|
|||||||
import { CoreV1Api, KubeConfig } from '@kubernetes/client-node';
|
|
||||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
|
||||||
import { waitUntil } from 'async-wait-until';
|
|
||||||
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
|
|
||||||
import CloudRunner from '../../cloud-runner';
|
|
||||||
import KubernetesPods from './kubernetes-pods';
|
|
||||||
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
|
|
||||||
|
|
||||||
class KubernetesTaskRunner {
|
|
||||||
static readonly maxRetry: number = 3;
|
|
||||||
static lastReceivedMessage: string = ``;
|
|
||||||
|
|
||||||
static async runTask(
|
|
||||||
kubeConfig: KubeConfig,
|
|
||||||
kubeClient: CoreV1Api,
|
|
||||||
jobName: string,
|
|
||||||
podName: string,
|
|
||||||
containerName: string,
|
|
||||||
namespace: string,
|
|
||||||
) {
|
|
||||||
let output = '';
|
|
||||||
let shouldReadLogs = true;
|
|
||||||
let shouldCleanup = true;
|
|
||||||
let retriesAfterFinish = 0;
|
|
||||||
// eslint-disable-next-line no-constant-condition
|
|
||||||
while (true) {
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
|
||||||
CloudRunnerLogger.log(
|
|
||||||
`Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace} ${CloudRunner.buildParameters.kubeVolumeSize}/${CloudRunner.buildParameters.containerCpu}/${CloudRunner.buildParameters.containerMemory}`,
|
|
||||||
);
|
|
||||||
let extraFlags = ``;
|
|
||||||
extraFlags += (await KubernetesPods.IsPodRunning(podName, namespace, kubeClient))
|
|
||||||
? ` -f -c ${containerName}`
|
|
||||||
: ` --previous`;
|
|
||||||
|
|
||||||
const callback = (outputChunk: string) => {
|
|
||||||
output += outputChunk;
|
|
||||||
|
|
||||||
// split output chunk and handle per line
|
|
||||||
for (const chunk of outputChunk.split(`\n`)) {
|
|
||||||
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
|
|
||||||
chunk,
|
|
||||||
shouldReadLogs,
|
|
||||||
shouldCleanup,
|
|
||||||
output,
|
|
||||||
));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
try {
|
|
||||||
await CloudRunnerSystem.Run(`kubectl logs ${podName}${extraFlags}`, false, true, callback);
|
|
||||||
} catch (error: any) {
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
|
||||||
const continueStreaming = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient);
|
|
||||||
CloudRunnerLogger.log(`K8s logging error ${error} ${continueStreaming}`);
|
|
||||||
if (continueStreaming) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (retriesAfterFinish < KubernetesTaskRunner.maxRetry) {
|
|
||||||
retriesAfterFinish++;
|
|
||||||
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
if (FollowLogStreamService.DidReceiveEndOfTransmission) {
|
|
||||||
CloudRunnerLogger.log('end of log stream');
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return output;
|
|
||||||
}
|
|
||||||
|
|
||||||
static async watchUntilPodRunning(kubeClient: CoreV1Api, podName: string, namespace: string) {
|
|
||||||
let waitComplete: boolean = false;
|
|
||||||
let message = ``;
|
|
||||||
CloudRunnerLogger.log(`Watching ${podName} ${namespace}`);
|
|
||||||
await waitUntil(
|
|
||||||
async () => {
|
|
||||||
const status = await kubeClient.readNamespacedPodStatus(podName, namespace);
|
|
||||||
const phase = status?.body.status?.phase;
|
|
||||||
waitComplete = phase !== 'Pending';
|
|
||||||
message = `Phase:${status.body.status?.phase} \n Reason:${
|
|
||||||
status.body.status?.conditions?.[0].reason || ''
|
|
||||||
} \n Message:${status.body.status?.conditions?.[0].message || ''}`;
|
|
||||||
|
|
||||||
// CloudRunnerLogger.log(
|
|
||||||
// JSON.stringify(
|
|
||||||
// (await kubeClient.listNamespacedEvent(namespace)).body.items
|
|
||||||
// .map((x) => {
|
|
||||||
// return {
|
|
||||||
// message: x.message || ``,
|
|
||||||
// name: x.metadata.name || ``,
|
|
||||||
// reason: x.reason || ``,
|
|
||||||
// };
|
|
||||||
// })
|
|
||||||
// .filter((x) => x.name.includes(podName)),
|
|
||||||
// undefined,
|
|
||||||
// 4,
|
|
||||||
// ),
|
|
||||||
// );
|
|
||||||
if (waitComplete || phase !== 'Pending') return true;
|
|
||||||
|
|
||||||
return false;
|
|
||||||
},
|
|
||||||
{
|
|
||||||
timeout: 2000000,
|
|
||||||
intervalBetweenAttempts: 15000,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
if (!waitComplete) {
|
|
||||||
CloudRunnerLogger.log(message);
|
|
||||||
}
|
|
||||||
|
|
||||||
return waitComplete;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export default KubernetesTaskRunner;
|
|
||||||
@@ -1,72 +0,0 @@
|
|||||||
import BuildParameters from '../../../build-parameters';
|
|
||||||
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
|
|
||||||
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
|
||||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
|
||||||
import { ProviderInterface } from '../provider-interface';
|
|
||||||
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
|
||||||
import { ProviderResource } from '../provider-resource';
|
|
||||||
import { ProviderWorkflow } from '../provider-workflow';
|
|
||||||
|
|
||||||
class LocalCloudRunner implements ProviderInterface {
|
|
||||||
listResources(): Promise<ProviderResource[]> {
|
|
||||||
throw new Error('Method not implemented.');
|
|
||||||
}
|
|
||||||
listWorkflow(): Promise<ProviderWorkflow[]> {
|
|
||||||
throw new Error('Method not implemented.');
|
|
||||||
}
|
|
||||||
watchWorkflow(): Promise<string> {
|
|
||||||
throw new Error('Method not implemented.');
|
|
||||||
}
|
|
||||||
garbageCollect(
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
filter: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
previewOnly: boolean,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
olderThan: Number,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
fullCache: boolean,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
baseDependencies: boolean,
|
|
||||||
): Promise<string> {
|
|
||||||
throw new Error('Method not implemented.');
|
|
||||||
}
|
|
||||||
cleanupWorkflow(
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
buildParameters: BuildParameters,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
branchName: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
|
||||||
) {}
|
|
||||||
public setupWorkflow(
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
buildGuid: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
buildParameters: BuildParameters,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
branchName: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
|
||||||
) {}
|
|
||||||
public async runTaskInWorkflow(
|
|
||||||
buildGuid: string,
|
|
||||||
image: string,
|
|
||||||
commands: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
mountdir: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
workingdir: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
environment: CloudRunnerEnvironmentVariable[],
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
secrets: CloudRunnerSecret[],
|
|
||||||
): Promise<string> {
|
|
||||||
CloudRunnerLogger.log(image);
|
|
||||||
CloudRunnerLogger.log(buildGuid);
|
|
||||||
CloudRunnerLogger.log(commands);
|
|
||||||
|
|
||||||
return await CloudRunnerSystem.Run(commands);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
export default LocalCloudRunner;
|
|
||||||
@@ -1,57 +0,0 @@
|
|||||||
import BuildParameters from '../../build-parameters';
|
|
||||||
import CloudRunnerEnvironmentVariable from '../options/cloud-runner-environment-variable';
|
|
||||||
import CloudRunnerSecret from '../options/cloud-runner-secret';
|
|
||||||
import { ProviderResource } from './provider-resource';
|
|
||||||
import { ProviderWorkflow } from './provider-workflow';
|
|
||||||
|
|
||||||
export interface ProviderInterface {
|
|
||||||
cleanupWorkflow(
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
buildParameters: BuildParameters,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
branchName: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
|
||||||
): any;
|
|
||||||
setupWorkflow(
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
buildGuid: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
buildParameters: BuildParameters,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
branchName: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
|
||||||
): any;
|
|
||||||
runTaskInWorkflow(
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
buildGuid: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
image: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
commands: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
mountdir: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
workingdir: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
environment: CloudRunnerEnvironmentVariable[],
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
secrets: CloudRunnerSecret[],
|
|
||||||
): Promise<string>;
|
|
||||||
garbageCollect(
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
filter: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
previewOnly: boolean,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
olderThan: Number,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
fullCache: boolean,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
baseDependencies: boolean,
|
|
||||||
): Promise<string>;
|
|
||||||
listResources(): Promise<ProviderResource[]>;
|
|
||||||
listWorkflow(): Promise<ProviderWorkflow[]>;
|
|
||||||
watchWorkflow(): Promise<string>;
|
|
||||||
}
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
export class ProviderResource {
|
|
||||||
public Name!: string;
|
|
||||||
}
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
export class ProviderWorkflow {
|
|
||||||
public Name!: string;
|
|
||||||
}
|
|
||||||
@@ -1,67 +0,0 @@
|
|||||||
import BuildParameters from '../../../build-parameters';
|
|
||||||
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
|
||||||
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
|
|
||||||
import { ProviderInterface } from '../provider-interface';
|
|
||||||
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
|
||||||
import { ProviderResource } from '../provider-resource';
|
|
||||||
import { ProviderWorkflow } from '../provider-workflow';
|
|
||||||
|
|
||||||
class TestCloudRunner implements ProviderInterface {
|
|
||||||
listResources(): Promise<ProviderResource[]> {
|
|
||||||
throw new Error('Method not implemented.');
|
|
||||||
}
|
|
||||||
listWorkflow(): Promise<ProviderWorkflow[]> {
|
|
||||||
throw new Error('Method not implemented.');
|
|
||||||
}
|
|
||||||
watchWorkflow(): Promise<string> {
|
|
||||||
throw new Error('Method not implemented.');
|
|
||||||
}
|
|
||||||
garbageCollect(
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
filter: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
previewOnly: boolean,
|
|
||||||
): Promise<string> {
|
|
||||||
throw new Error('Method not implemented.');
|
|
||||||
}
|
|
||||||
cleanupWorkflow(
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
buildParameters: BuildParameters,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
branchName: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
|
||||||
) {}
|
|
||||||
setupWorkflow(
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
buildGuid: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
buildParameters: BuildParameters,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
branchName: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
|
||||||
) {}
|
|
||||||
public async runTaskInWorkflow(
|
|
||||||
commands: string,
|
|
||||||
buildGuid: string,
|
|
||||||
image: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
mountdir: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
workingdir: string,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
environment: CloudRunnerEnvironmentVariable[],
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
secrets: CloudRunnerSecret[],
|
|
||||||
): Promise<string> {
|
|
||||||
CloudRunnerLogger.log(image);
|
|
||||||
CloudRunnerLogger.log(buildGuid);
|
|
||||||
CloudRunnerLogger.log(commands);
|
|
||||||
|
|
||||||
return await new Promise((result) => {
|
|
||||||
result(commands);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
export default TestCloudRunner;
|
|
||||||
@@ -1,181 +0,0 @@
|
|||||||
import { assert } from 'node:console';
|
|
||||||
import fs from 'node:fs';
|
|
||||||
import path from 'node:path';
|
|
||||||
import CloudRunner from '../cloud-runner';
|
|
||||||
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
|
||||||
import { CloudRunnerFolders } from '../options/cloud-runner-folders';
|
|
||||||
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
|
|
||||||
import { LfsHashing } from '../services/utility/lfs-hashing';
|
|
||||||
import { RemoteClientLogger } from './remote-client-logger';
|
|
||||||
import { Cli } from '../../cli/cli';
|
|
||||||
import { CliFunction } from '../../cli/cli-functions-repository';
|
|
||||||
// eslint-disable-next-line github/no-then
|
|
||||||
const fileExists = async (fpath: fs.PathLike) => !!(await fs.promises.stat(fpath).catch(() => false));
|
|
||||||
|
|
||||||
export class Caching {
|
|
||||||
@CliFunction(`cache-push`, `push to cache`)
|
|
||||||
static async cachePush() {
|
|
||||||
try {
|
|
||||||
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
|
|
||||||
CloudRunner.buildParameters = buildParameter;
|
|
||||||
await Caching.PushToCache(
|
|
||||||
Cli.options!['cachePushTo'],
|
|
||||||
Cli.options!['cachePushFrom'],
|
|
||||||
Cli.options!['artifactName'] || '',
|
|
||||||
);
|
|
||||||
} catch (error: any) {
|
|
||||||
CloudRunnerLogger.log(`${error}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliFunction(`cache-pull`, `pull from cache`)
|
|
||||||
static async cachePull() {
|
|
||||||
try {
|
|
||||||
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
|
|
||||||
CloudRunner.buildParameters = buildParameter;
|
|
||||||
await Caching.PullFromCache(
|
|
||||||
Cli.options!['cachePushFrom'],
|
|
||||||
Cli.options!['cachePushTo'],
|
|
||||||
Cli.options!['artifactName'] || '',
|
|
||||||
);
|
|
||||||
} catch (error: any) {
|
|
||||||
CloudRunnerLogger.log(`${error}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async PushToCache(cacheFolder: string, sourceFolder: string, cacheArtifactName: string) {
|
|
||||||
CloudRunnerLogger.log(`Pushing to cache ${sourceFolder}`);
|
|
||||||
cacheArtifactName = cacheArtifactName.replace(' ', '');
|
|
||||||
const startPath = process.cwd();
|
|
||||||
let compressionSuffix = '';
|
|
||||||
if (CloudRunner.buildParameters.useCompressionStrategy === true) {
|
|
||||||
compressionSuffix = `.lz4`;
|
|
||||||
}
|
|
||||||
CloudRunnerLogger.log(`Compression: ${CloudRunner.buildParameters.useCompressionStrategy} ${compressionSuffix}`);
|
|
||||||
try {
|
|
||||||
if (!(await fileExists(cacheFolder))) {
|
|
||||||
await CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
|
|
||||||
}
|
|
||||||
process.chdir(path.resolve(sourceFolder, '..'));
|
|
||||||
|
|
||||||
if (CloudRunner.buildParameters.cloudRunnerDebug === true) {
|
|
||||||
CloudRunnerLogger.log(
|
|
||||||
`Hashed cache folder ${await LfsHashing.hashAllFiles(sourceFolder)} ${sourceFolder} ${path.basename(
|
|
||||||
sourceFolder,
|
|
||||||
)}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
const contents = await fs.promises.readdir(path.basename(sourceFolder));
|
|
||||||
CloudRunnerLogger.log(
|
|
||||||
`There is ${contents.length} files/dir in the source folder ${path.basename(sourceFolder)}`,
|
|
||||||
);
|
|
||||||
|
|
||||||
if (contents.length === 0) {
|
|
||||||
CloudRunnerLogger.log(
|
|
||||||
`Did not push source folder to cache because it was empty ${path.basename(sourceFolder)}`,
|
|
||||||
);
|
|
||||||
process.chdir(`${startPath}`);
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
await CloudRunnerSystem.Run(
|
|
||||||
`tar -cf ${cacheArtifactName}.tar${compressionSuffix} "${path.basename(sourceFolder)}"`,
|
|
||||||
);
|
|
||||||
await CloudRunnerSystem.Run(`du ${cacheArtifactName}.tar${compressionSuffix}`);
|
|
||||||
assert(await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`), 'cache archive exists');
|
|
||||||
assert(await fileExists(path.basename(sourceFolder)), 'source folder exists');
|
|
||||||
await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar${compressionSuffix} ${cacheFolder}`);
|
|
||||||
RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`);
|
|
||||||
assert(
|
|
||||||
await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.tar${compressionSuffix}`),
|
|
||||||
'cache archive exists inside cache folder',
|
|
||||||
);
|
|
||||||
} catch (error) {
|
|
||||||
process.chdir(`${startPath}`);
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
process.chdir(`${startPath}`);
|
|
||||||
}
|
|
||||||
public static async PullFromCache(cacheFolder: string, destinationFolder: string, cacheArtifactName: string = ``) {
|
|
||||||
CloudRunnerLogger.log(`Pulling from cache ${destinationFolder} ${CloudRunner.buildParameters.skipCache}`);
|
|
||||||
if (`${CloudRunner.buildParameters.skipCache}` === `true`) {
|
|
||||||
CloudRunnerLogger.log(`Skipping cache debugSkipCache is true`);
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
cacheArtifactName = cacheArtifactName.replace(' ', '');
|
|
||||||
let compressionSuffix = '';
|
|
||||||
if (CloudRunner.buildParameters.useCompressionStrategy === true) {
|
|
||||||
compressionSuffix = `.lz4`;
|
|
||||||
}
|
|
||||||
const startPath = process.cwd();
|
|
||||||
RemoteClientLogger.log(`Caching for (lz4 ${compressionSuffix}) ${path.basename(destinationFolder)}`);
|
|
||||||
try {
|
|
||||||
if (!(await fileExists(cacheFolder))) {
|
|
||||||
await fs.promises.mkdir(cacheFolder);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!(await fileExists(destinationFolder))) {
|
|
||||||
await fs.promises.mkdir(destinationFolder);
|
|
||||||
}
|
|
||||||
|
|
||||||
const latestInBranch = await (
|
|
||||||
await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar${compressionSuffix}$ | head -1`)
|
|
||||||
)
|
|
||||||
.replace(/\n/g, ``)
|
|
||||||
.replace(`.tar${compressionSuffix}`, '');
|
|
||||||
|
|
||||||
process.chdir(cacheFolder);
|
|
||||||
|
|
||||||
const cacheSelection =
|
|
||||||
cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`))
|
|
||||||
? cacheArtifactName
|
|
||||||
: latestInBranch;
|
|
||||||
await CloudRunnerLogger.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`);
|
|
||||||
|
|
||||||
if (await fileExists(`${cacheSelection}.tar${compressionSuffix}`)) {
|
|
||||||
const resultsFolder = `results${CloudRunner.buildParameters.buildGuid}`;
|
|
||||||
await CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`);
|
|
||||||
RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar${compressionSuffix}`);
|
|
||||||
const fullResultsFolder = path.join(cacheFolder, resultsFolder);
|
|
||||||
await CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar${compressionSuffix} -C ${fullResultsFolder}`);
|
|
||||||
RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
|
|
||||||
assert(await fileExists(fullResultsFolder), `cache extraction results folder exists`);
|
|
||||||
const destinationParentFolder = path.resolve(destinationFolder, '..');
|
|
||||||
|
|
||||||
if (await fileExists(destinationFolder)) {
|
|
||||||
await fs.promises.rmdir(destinationFolder, { recursive: true });
|
|
||||||
}
|
|
||||||
await CloudRunnerSystem.Run(
|
|
||||||
`mv "${path.join(fullResultsFolder, path.basename(destinationFolder))}" "${destinationParentFolder}"`,
|
|
||||||
);
|
|
||||||
const contents = await fs.promises.readdir(
|
|
||||||
path.join(destinationParentFolder, path.basename(destinationFolder)),
|
|
||||||
);
|
|
||||||
CloudRunnerLogger.log(
|
|
||||||
`There is ${contents.length} files/dir in the cache pulled contents for ${path.basename(destinationFolder)}`,
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`);
|
|
||||||
if (cacheSelection !== ``) {
|
|
||||||
RemoteClientLogger.logWarning(
|
|
||||||
`cache item ${cacheArtifactName}.tar${compressionSuffix} doesn't exist ${destinationFolder}`,
|
|
||||||
);
|
|
||||||
throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
process.chdir(startPath);
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
process.chdir(startPath);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async handleCachePurging() {
|
|
||||||
if (process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined) {
|
|
||||||
RemoteClientLogger.log(`purging ${CloudRunnerFolders.purgeRemoteCaching}`);
|
|
||||||
fs.promises.rmdir(CloudRunnerFolders.cacheFolder, { recursive: true });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,251 +0,0 @@
|
|||||||
import fs from 'node:fs';
|
|
||||||
import CloudRunner from '../cloud-runner';
|
|
||||||
import { CloudRunnerFolders } from '../options/cloud-runner-folders';
|
|
||||||
import { Caching } from './caching';
|
|
||||||
import { LfsHashing } from '../services/utility/lfs-hashing';
|
|
||||||
import { RemoteClientLogger } from './remote-client-logger';
|
|
||||||
import path from 'node:path';
|
|
||||||
import { assert } from 'node:console';
|
|
||||||
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
|
||||||
import { CliFunction } from '../../cli/cli-functions-repository';
|
|
||||||
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
|
|
||||||
import YAML from 'yaml';
|
|
||||||
import GitHub from '../../github';
|
|
||||||
import BuildParameters from '../../build-parameters';
|
|
||||||
import { Cli } from '../../cli/cli';
|
|
||||||
import CloudRunnerOptions from '../options/cloud-runner-options';
|
|
||||||
|
|
||||||
export class RemoteClient {
|
|
||||||
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
|
|
||||||
static async setupRemoteClient() {
|
|
||||||
CloudRunnerLogger.log(`bootstrap game ci cloud runner...`);
|
|
||||||
if (!(await RemoteClient.handleRetainedWorkspace())) {
|
|
||||||
await RemoteClient.bootstrapRepository();
|
|
||||||
}
|
|
||||||
await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
|
|
||||||
await RemoteClient.runCustomHookFiles(`before-build`);
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliFunction('remote-cli-log-stream', `log stream from standard input`)
|
|
||||||
public static async remoteClientLogStream() {
|
|
||||||
const logFile = Cli.options!['logFile'];
|
|
||||||
process.stdin.resume();
|
|
||||||
process.stdin.setEncoding('utf8');
|
|
||||||
|
|
||||||
let lingeringLine = '';
|
|
||||||
|
|
||||||
process.stdin.on('data', (chunk) => {
|
|
||||||
const lines = chunk.toString().split('\n');
|
|
||||||
|
|
||||||
lines[0] = lingeringLine + lines[0];
|
|
||||||
lingeringLine = lines.pop() || '';
|
|
||||||
|
|
||||||
for (const element of lines) {
|
|
||||||
if (CloudRunnerOptions.providerStrategy !== 'k8s') {
|
|
||||||
CloudRunnerLogger.log(element);
|
|
||||||
} else {
|
|
||||||
fs.appendFileSync(logFile, element);
|
|
||||||
CloudRunnerLogger.log(element);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
process.stdin.on('end', () => {
|
|
||||||
if (CloudRunnerOptions.providerStrategy !== 'k8s') {
|
|
||||||
CloudRunnerLogger.log(lingeringLine);
|
|
||||||
} else {
|
|
||||||
fs.appendFileSync(logFile, lingeringLine);
|
|
||||||
CloudRunnerLogger.log(lingeringLine);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliFunction(`remote-cli-post-build`, `runs a cloud runner build`)
|
|
||||||
public static async remoteClientPostBuild(): Promise<string> {
|
|
||||||
RemoteClientLogger.log(`Running POST build tasks`);
|
|
||||||
|
|
||||||
await Caching.PushToCache(
|
|
||||||
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/Library`),
|
|
||||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
|
|
||||||
`lib-${CloudRunner.buildParameters.buildGuid}`,
|
|
||||||
);
|
|
||||||
|
|
||||||
await Caching.PushToCache(
|
|
||||||
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/build`),
|
|
||||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.projectBuildFolderAbsolute),
|
|
||||||
`build-${CloudRunner.buildParameters.buildGuid}`,
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)) {
|
|
||||||
await CloudRunnerSystem.Run(
|
|
||||||
`rm -r ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
await RemoteClient.runCustomHookFiles(`after-build`);
|
|
||||||
|
|
||||||
// WIP - need to give the pod permissions to create config map
|
|
||||||
await RemoteClientLogger.handleLogManagementPostJob();
|
|
||||||
|
|
||||||
return new Promise((result) => result(``));
|
|
||||||
}
|
|
||||||
static async runCustomHookFiles(hookLifecycle: string) {
|
|
||||||
RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
|
|
||||||
const gameCiCustomHooksPath = path.join(CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`);
|
|
||||||
try {
|
|
||||||
const files = fs.readdirSync(gameCiCustomHooksPath);
|
|
||||||
for (const file of files) {
|
|
||||||
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
|
|
||||||
const fileContentsObject = YAML.parse(fileContents.toString());
|
|
||||||
if (fileContentsObject.hook === hookLifecycle) {
|
|
||||||
RemoteClientLogger.log(`Active Hook File ${file} \n \n file contents: \n ${fileContents}`);
|
|
||||||
await CloudRunnerSystem.Run(fileContentsObject.commands);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
RemoteClientLogger.log(JSON.stringify(error, undefined, 4));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
public static async bootstrapRepository() {
|
|
||||||
await CloudRunnerSystem.Run(
|
|
||||||
`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
|
|
||||||
);
|
|
||||||
await CloudRunnerSystem.Run(
|
|
||||||
`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForCacheKeyFull)}`,
|
|
||||||
);
|
|
||||||
await RemoteClient.cloneRepoWithoutLFSFiles();
|
|
||||||
await RemoteClient.sizeOfFolder(
|
|
||||||
'repo before lfs cache pull',
|
|
||||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute),
|
|
||||||
);
|
|
||||||
const lfsHashes = await LfsHashing.createLFSHashFiles();
|
|
||||||
if (fs.existsSync(CloudRunnerFolders.libraryFolderAbsolute)) {
|
|
||||||
RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`);
|
|
||||||
}
|
|
||||||
await Caching.PullFromCache(
|
|
||||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
|
|
||||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
|
|
||||||
`${lfsHashes.lfsGuidSum}`,
|
|
||||||
);
|
|
||||||
await RemoteClient.sizeOfFolder('repo after lfs cache pull', CloudRunnerFolders.repoPathAbsolute);
|
|
||||||
await RemoteClient.pullLatestLFS();
|
|
||||||
await RemoteClient.sizeOfFolder('repo before lfs git pull', CloudRunnerFolders.repoPathAbsolute);
|
|
||||||
await Caching.PushToCache(
|
|
||||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
|
|
||||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
|
|
||||||
`${lfsHashes.lfsGuidSum}`,
|
|
||||||
);
|
|
||||||
await Caching.PullFromCache(
|
|
||||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryCacheFolderFull),
|
|
||||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
|
|
||||||
);
|
|
||||||
await RemoteClient.sizeOfFolder('repo after library cache pull', CloudRunnerFolders.repoPathAbsolute);
|
|
||||||
await Caching.handleCachePurging();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static async sizeOfFolder(message: string, folder: string) {
|
|
||||||
if (CloudRunner.buildParameters.cloudRunnerDebug) {
|
|
||||||
CloudRunnerLogger.log(`Size of ${message}`);
|
|
||||||
await CloudRunnerSystem.Run(`du -sh ${folder}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static async cloneRepoWithoutLFSFiles() {
|
|
||||||
process.chdir(`${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
|
|
||||||
if (
|
|
||||||
fs.existsSync(CloudRunnerFolders.repoPathAbsolute) &&
|
|
||||||
!fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))
|
|
||||||
) {
|
|
||||||
await CloudRunnerSystem.Run(`rm -r ${CloudRunnerFolders.repoPathAbsolute}`);
|
|
||||||
CloudRunnerLogger.log(`${CloudRunnerFolders.repoPathAbsolute} repo exists, but no git folder, cleaning up`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (
|
|
||||||
BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters) &&
|
|
||||||
fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))
|
|
||||||
) {
|
|
||||||
process.chdir(CloudRunnerFolders.repoPathAbsolute);
|
|
||||||
RemoteClientLogger.log(
|
|
||||||
`${
|
|
||||||
CloudRunnerFolders.repoPathAbsolute
|
|
||||||
} repo exists - skipping clone - retained workspace mode ${BuildParameters.shouldUseRetainedWorkspaceMode(
|
|
||||||
CloudRunner.buildParameters,
|
|
||||||
)}`,
|
|
||||||
);
|
|
||||||
await CloudRunnerSystem.Run(`git fetch && git reset --hard ${CloudRunner.buildParameters.gitSha}`);
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
|
|
||||||
await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
|
|
||||||
RemoteClientLogger.log(`Cloning the repository being built:`);
|
|
||||||
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f"`);
|
|
||||||
await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process --skip"`);
|
|
||||||
try {
|
|
||||||
await CloudRunnerSystem.Run(
|
|
||||||
`git clone ${CloudRunnerFolders.targetBuildRepoUrl} ${path.basename(CloudRunnerFolders.repoPathAbsolute)}`,
|
|
||||||
);
|
|
||||||
} catch (error: any) {
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
process.chdir(CloudRunnerFolders.repoPathAbsolute);
|
|
||||||
await CloudRunnerSystem.Run(`git lfs install`);
|
|
||||||
assert(fs.existsSync(`.git`), 'git folder exists');
|
|
||||||
RemoteClientLogger.log(`${CloudRunner.buildParameters.branch}`);
|
|
||||||
if (CloudRunner.buildParameters.gitSha !== undefined) {
|
|
||||||
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.gitSha}`);
|
|
||||||
} else {
|
|
||||||
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.branch}`);
|
|
||||||
RemoteClientLogger.log(`buildParameter Git Sha is empty`);
|
|
||||||
}
|
|
||||||
|
|
||||||
assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
|
|
||||||
RemoteClientLogger.log(`Checked out ${CloudRunner.buildParameters.branch}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
static async replaceLargePackageReferencesWithSharedReferences() {
|
|
||||||
CloudRunnerLogger.log(`Use Shared Pkgs ${CloudRunner.buildParameters.useLargePackages}`);
|
|
||||||
GitHub.updateGitHubCheck(`Use Shared Pkgs ${CloudRunner.buildParameters.useLargePackages}`, ``);
|
|
||||||
if (CloudRunner.buildParameters.useLargePackages) {
|
|
||||||
const filePath = path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`);
|
|
||||||
let manifest = fs.readFileSync(filePath, 'utf8');
|
|
||||||
manifest = manifest.replace(/LargeContent/g, '../../../LargeContent');
|
|
||||||
fs.writeFileSync(filePath, manifest);
|
|
||||||
CloudRunnerLogger.log(`Package Manifest \n ${manifest}`);
|
|
||||||
GitHub.updateGitHubCheck(`Package Manifest \n ${manifest}`, ``);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static async pullLatestLFS() {
|
|
||||||
process.chdir(CloudRunnerFolders.repoPathAbsolute);
|
|
||||||
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge -- %f"`);
|
|
||||||
await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process"`);
|
|
||||||
if (!CloudRunner.buildParameters.skipLfs) {
|
|
||||||
await CloudRunnerSystem.Run(`git lfs pull`);
|
|
||||||
RemoteClientLogger.log(`pulled latest LFS files`);
|
|
||||||
assert(fs.existsSync(CloudRunnerFolders.lfsFolderAbsolute));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
static async handleRetainedWorkspace() {
|
|
||||||
RemoteClientLogger.log(
|
|
||||||
`Retained Workspace: ${BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)}`,
|
|
||||||
);
|
|
||||||
if (
|
|
||||||
BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters) &&
|
|
||||||
fs.existsSync(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)) &&
|
|
||||||
fs.existsSync(CloudRunnerFolders.ToLinuxFolder(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`)))
|
|
||||||
) {
|
|
||||||
CloudRunnerLogger.log(`Retained Workspace Already Exists!`);
|
|
||||||
process.chdir(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute));
|
|
||||||
await CloudRunnerSystem.Run(`git fetch`);
|
|
||||||
await CloudRunnerSystem.Run(`git lfs pull`);
|
|
||||||
await CloudRunnerSystem.Run(`git reset --hard "${CloudRunner.buildParameters.gitSha}"`);
|
|
||||||
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.gitSha}`);
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,82 +0,0 @@
|
|||||||
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
|
||||||
import fs from 'node:fs';
|
|
||||||
import path from 'node:path';
|
|
||||||
import CloudRunner from '../cloud-runner';
|
|
||||||
import CloudRunnerOptions from '../options/cloud-runner-options';
|
|
||||||
|
|
||||||
export class RemoteClientLogger {
|
|
||||||
private static get LogFilePath() {
|
|
||||||
return path.join(`/home`, `job-log.txt`);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static log(message: string) {
|
|
||||||
const finalMessage = `[Client] ${message}`;
|
|
||||||
this.appendToFile(finalMessage);
|
|
||||||
CloudRunnerLogger.log(finalMessage);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static logCliError(message: string) {
|
|
||||||
CloudRunnerLogger.log(`[Client][Error] ${message}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static logCliDiagnostic(message: string) {
|
|
||||||
CloudRunnerLogger.log(`[Client][Diagnostic] ${message}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static logWarning(message: string) {
|
|
||||||
CloudRunnerLogger.logWarning(message);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static appendToFile(message: string) {
|
|
||||||
if (CloudRunner.isCloudRunnerEnvironment) {
|
|
||||||
fs.appendFileSync(RemoteClientLogger.LogFilePath, `${message}\n`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async handleLogManagementPostJob() {
|
|
||||||
if (CloudRunnerOptions.providerStrategy !== 'k8s') {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
CloudRunnerLogger.log(`Collected Logs`);
|
|
||||||
|
|
||||||
// check for log file not existing
|
|
||||||
if (!fs.existsSync(RemoteClientLogger.LogFilePath)) {
|
|
||||||
CloudRunnerLogger.log(`Log file does not exist`);
|
|
||||||
|
|
||||||
// check if CloudRunner.isCloudRunnerEnvironment is true, log
|
|
||||||
if (!CloudRunner.isCloudRunnerEnvironment) {
|
|
||||||
CloudRunnerLogger.log(`Cloud Runner is not running in a cloud environment, not collecting logs`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
CloudRunnerLogger.log(`Log file exist`);
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 1));
|
|
||||||
|
|
||||||
// let hashedLogs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
|
|
||||||
//
|
|
||||||
// hashedLogs = md5(hashedLogs);
|
|
||||||
//
|
|
||||||
// for (let index = 0; index < 3; index++) {
|
|
||||||
// CloudRunnerLogger.log(`LOGHASH: ${hashedLogs}`);
|
|
||||||
// const logs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
|
|
||||||
// CloudRunnerLogger.log(`LOGS: ${Buffer.from(logs).toString('base64')}`);
|
|
||||||
// CloudRunnerLogger.log(
|
|
||||||
// `Game CI's "Cloud Runner System" will cancel the log when it has successfully received the log data to verify all logs have been received.`,
|
|
||||||
// );
|
|
||||||
//
|
|
||||||
// // wait for 15 seconds to allow the log to be sent
|
|
||||||
// await new Promise((resolve) => setTimeout(resolve, 15000));
|
|
||||||
// }
|
|
||||||
}
|
|
||||||
public static HandleLog(message: string): boolean {
|
|
||||||
if (RemoteClientLogger.value !== '') {
|
|
||||||
RemoteClientLogger.value += `\n`;
|
|
||||||
}
|
|
||||||
|
|
||||||
RemoteClientLogger.value += message;
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
static value: string = '';
|
|
||||||
}
|
|
||||||
@@ -1,47 +0,0 @@
|
|||||||
import * as core from '@actions/core';
|
|
||||||
|
|
||||||
class CloudRunnerLogger {
|
|
||||||
private static timestamp: number;
|
|
||||||
private static globalTimestamp: number;
|
|
||||||
|
|
||||||
public static setup() {
|
|
||||||
this.timestamp = this.createTimestamp();
|
|
||||||
this.globalTimestamp = this.timestamp;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static log(message: string) {
|
|
||||||
core.info(message);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static logWarning(message: string) {
|
|
||||||
core.warning(message);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static logLine(message: string) {
|
|
||||||
core.info(`${message}\n`);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static error(message: string) {
|
|
||||||
core.error(message);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static logWithTime(message: string) {
|
|
||||||
const newTimestamp = this.createTimestamp();
|
|
||||||
core.info(
|
|
||||||
`${message} (Since previous: ${this.calculateTimeDiff(
|
|
||||||
newTimestamp,
|
|
||||||
this.timestamp,
|
|
||||||
)}, Total time: ${this.calculateTimeDiff(newTimestamp, this.globalTimestamp)})`,
|
|
||||||
);
|
|
||||||
this.timestamp = newTimestamp;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static calculateTimeDiff(x: number, y: number) {
|
|
||||||
return Math.floor((x - y) / 1000);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static createTimestamp() {
|
|
||||||
return Date.now();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
export default CloudRunnerLogger;
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
import BuildParameters from '../../../build-parameters';
|
|
||||||
|
|
||||||
class CloudRunnerResult {
|
|
||||||
public BuildParameters: BuildParameters;
|
|
||||||
public BuildResults: string;
|
|
||||||
public BuildSucceeded: boolean;
|
|
||||||
public BuildFinished: boolean;
|
|
||||||
public LibraryCacheUsed: boolean;
|
|
||||||
|
|
||||||
public constructor(
|
|
||||||
buildParameters: BuildParameters,
|
|
||||||
buildResults: string,
|
|
||||||
buildSucceeded: boolean,
|
|
||||||
buildFinished: boolean,
|
|
||||||
libraryCacheUsed: boolean,
|
|
||||||
) {
|
|
||||||
this.BuildParameters = buildParameters;
|
|
||||||
this.BuildResults = buildResults;
|
|
||||||
this.BuildSucceeded = buildSucceeded;
|
|
||||||
this.BuildFinished = buildFinished;
|
|
||||||
this.LibraryCacheUsed = libraryCacheUsed;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
export default CloudRunnerResult;
|
|
||||||
@@ -1,69 +0,0 @@
|
|||||||
import { exec } from 'child_process';
|
|
||||||
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
|
|
||||||
|
|
||||||
export class CloudRunnerSystem {
|
|
||||||
public static async RunAndReadLines(command: string): Promise<string[]> {
|
|
||||||
const result = await CloudRunnerSystem.Run(command, false, true);
|
|
||||||
|
|
||||||
return result
|
|
||||||
.split(`\n`)
|
|
||||||
.map((x) => x.replace(`\r`, ``))
|
|
||||||
.filter((x) => x !== ``)
|
|
||||||
.map((x) => {
|
|
||||||
const lineValues = x.split(` `);
|
|
||||||
|
|
||||||
return lineValues[lineValues.length - 1];
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async Run(
|
|
||||||
command: string,
|
|
||||||
suppressError = false,
|
|
||||||
suppressLogs = false,
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
outputCallback?: (output: string) => void,
|
|
||||||
) {
|
|
||||||
for (const element of command.split(`\n`)) {
|
|
||||||
if (!suppressLogs) {
|
|
||||||
RemoteClientLogger.log(element);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return await new Promise<string>((promise, throwError) => {
|
|
||||||
let output = '';
|
|
||||||
const child = exec(command, { maxBuffer: 1024 * 10000 }, (error, stdout, stderr) => {
|
|
||||||
if (!suppressError && error) {
|
|
||||||
RemoteClientLogger.log(error.toString());
|
|
||||||
throwError(error);
|
|
||||||
}
|
|
||||||
if (stderr) {
|
|
||||||
const diagnosticOutput = `${stderr.toString()}`;
|
|
||||||
if (!suppressLogs) {
|
|
||||||
RemoteClientLogger.logCliDiagnostic(diagnosticOutput);
|
|
||||||
}
|
|
||||||
output += diagnosticOutput;
|
|
||||||
}
|
|
||||||
const outputChunk = `${stdout}`;
|
|
||||||
if (outputCallback) {
|
|
||||||
outputCallback(outputChunk);
|
|
||||||
}
|
|
||||||
output += outputChunk;
|
|
||||||
});
|
|
||||||
child.on('close', (code) => {
|
|
||||||
if (!suppressLogs) {
|
|
||||||
RemoteClientLogger.log(`[${code}]`);
|
|
||||||
}
|
|
||||||
if (code !== 0 && !suppressError) {
|
|
||||||
throwError(output);
|
|
||||||
}
|
|
||||||
const outputLines = output.split(`\n`);
|
|
||||||
for (const element of outputLines) {
|
|
||||||
if (!suppressLogs) {
|
|
||||||
RemoteClientLogger.log(element);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
promise(output);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,57 +0,0 @@
|
|||||||
import GitHub from '../../../github';
|
|
||||||
import CloudRunner from '../../cloud-runner';
|
|
||||||
import { CloudRunnerStatics } from '../../options/cloud-runner-statics';
|
|
||||||
import CloudRunnerLogger from './cloud-runner-logger';
|
|
||||||
import * as core from '@actions/core';
|
|
||||||
|
|
||||||
export class FollowLogStreamService {
|
|
||||||
static Reset() {
|
|
||||||
FollowLogStreamService.DidReceiveEndOfTransmission = false;
|
|
||||||
}
|
|
||||||
static errors = ``;
|
|
||||||
public static DidReceiveEndOfTransmission = false;
|
|
||||||
public static handleIteration(message: string, shouldReadLogs: boolean, shouldCleanup: boolean, output: string) {
|
|
||||||
if (message.includes(`---${CloudRunner.buildParameters.logId}`)) {
|
|
||||||
CloudRunnerLogger.log('End of log transmission received');
|
|
||||||
FollowLogStreamService.DidReceiveEndOfTransmission = true;
|
|
||||||
shouldReadLogs = false;
|
|
||||||
} else if (message.includes('Rebuilding Library because the asset database could not be found!')) {
|
|
||||||
GitHub.updateGitHubCheck(`Library was not found, importing new Library`, ``);
|
|
||||||
core.warning('LIBRARY NOT FOUND!');
|
|
||||||
core.setOutput('library-found', 'false');
|
|
||||||
} else if (message.includes('Build succeeded')) {
|
|
||||||
GitHub.updateGitHubCheck(`Build succeeded`, `Build succeeded`);
|
|
||||||
core.setOutput('build-result', 'success');
|
|
||||||
} else if (message.includes('Build fail')) {
|
|
||||||
GitHub.updateGitHubCheck(
|
|
||||||
`Build failed\n${FollowLogStreamService.errors}`,
|
|
||||||
`Build failed`,
|
|
||||||
`failure`,
|
|
||||||
`completed`,
|
|
||||||
);
|
|
||||||
core.setOutput('build-result', 'failed');
|
|
||||||
core.setFailed('unity build failed');
|
|
||||||
core.error('BUILD FAILED!');
|
|
||||||
} else if (message.toLowerCase().includes('error ')) {
|
|
||||||
core.error(message);
|
|
||||||
FollowLogStreamService.errors += `\n${message}`;
|
|
||||||
} else if (message.toLowerCase().includes('error: ')) {
|
|
||||||
core.error(message);
|
|
||||||
FollowLogStreamService.errors += `\n${message}`;
|
|
||||||
} else if (message.toLowerCase().includes('command failed: ')) {
|
|
||||||
FollowLogStreamService.errors += `\n${message}`;
|
|
||||||
} else if (message.toLowerCase().includes('invalid ')) {
|
|
||||||
FollowLogStreamService.errors += `\n${message}`;
|
|
||||||
} else if (message.toLowerCase().includes('incompatible ')) {
|
|
||||||
FollowLogStreamService.errors += `\n${message}`;
|
|
||||||
} else if (message.toLowerCase().includes('cannot be found')) {
|
|
||||||
FollowLogStreamService.errors += `\n${message}`;
|
|
||||||
}
|
|
||||||
if (CloudRunner.buildParameters.cloudRunnerDebug) {
|
|
||||||
output += `${message}\n`;
|
|
||||||
}
|
|
||||||
CloudRunnerLogger.log(`[${CloudRunnerStatics.logPrefix}] ${message}`);
|
|
||||||
|
|
||||||
return { shouldReadLogs, shouldCleanup, output };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,300 +0,0 @@
|
|||||||
import { CloudRunnerSystem } from './cloud-runner-system';
|
|
||||||
import fs from 'node:fs';
|
|
||||||
import CloudRunnerLogger from './cloud-runner-logger';
|
|
||||||
import BuildParameters from '../../../build-parameters';
|
|
||||||
import CloudRunner from '../../cloud-runner';
|
|
||||||
export class SharedWorkspaceLocking {
|
|
||||||
public static get workspaceBucketRoot() {
|
|
||||||
return `s3://${CloudRunner.buildParameters.awsStackName}/`;
|
|
||||||
}
|
|
||||||
public static get workspaceRoot() {
|
|
||||||
return `${SharedWorkspaceLocking.workspaceBucketRoot}locks/`;
|
|
||||||
}
|
|
||||||
public static async GetAllWorkspaces(buildParametersContext: BuildParameters): Promise<string[]> {
|
|
||||||
if (!(await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParametersContext))) {
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
|
||||||
await SharedWorkspaceLocking.ReadLines(
|
|
||||||
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
.map((x) => x.replace(`/`, ``))
|
|
||||||
.filter((x) => x.endsWith(`_workspace`))
|
|
||||||
.map((x) => x.split(`_`)[1]);
|
|
||||||
}
|
|
||||||
public static async DoesCacheKeyTopLevelExist(buildParametersContext: BuildParameters) {
|
|
||||||
try {
|
|
||||||
const rootLines = await SharedWorkspaceLocking.ReadLines(
|
|
||||||
`aws s3 ls ${SharedWorkspaceLocking.workspaceBucketRoot}`,
|
|
||||||
);
|
|
||||||
const lockFolderExists = rootLines.map((x) => x.replace(`/`, ``)).includes(`locks`);
|
|
||||||
|
|
||||||
if (lockFolderExists) {
|
|
||||||
const lines = await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`);
|
|
||||||
|
|
||||||
return lines.map((x) => x.replace(`/`, ``)).includes(buildParametersContext.cacheKey);
|
|
||||||
} else {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static NewWorkspaceName() {
|
|
||||||
return `${CloudRunner.retainedWorkspacePrefix}-${CloudRunner.buildParameters.buildGuid}`;
|
|
||||||
}
|
|
||||||
public static async GetAllLocksForWorkspace(
|
|
||||||
workspace: string,
|
|
||||||
buildParametersContext: BuildParameters,
|
|
||||||
): Promise<string[]> {
|
|
||||||
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
|
||||||
await SharedWorkspaceLocking.ReadLines(
|
|
||||||
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
.map((x) => x.replace(`/`, ``))
|
|
||||||
.filter((x) => x.includes(workspace) && x.endsWith(`_lock`));
|
|
||||||
}
|
|
||||||
public static async GetLockedWorkspace(workspace: string, runId: string, buildParametersContext: BuildParameters) {
|
|
||||||
if (buildParametersContext.maxRetainedWorkspaces === 0) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParametersContext)) {
|
|
||||||
const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
|
|
||||||
CloudRunnerLogger.log(`run agent ${runId} is trying to access a workspace, free: ${JSON.stringify(workspaces)}`);
|
|
||||||
for (const element of workspaces) {
|
|
||||||
const lockResult = await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext);
|
|
||||||
CloudRunnerLogger.log(
|
|
||||||
`run agent: ${runId} try lock workspace: ${element} locking attempt result: ${lockResult}`,
|
|
||||||
);
|
|
||||||
|
|
||||||
if (lockResult) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext)) {
|
|
||||||
workspace = SharedWorkspaceLocking.NewWorkspaceName();
|
|
||||||
CloudRunner.lockedWorkspace = workspace;
|
|
||||||
}
|
|
||||||
|
|
||||||
const createResult = await SharedWorkspaceLocking.CreateWorkspace(workspace, buildParametersContext);
|
|
||||||
const lockResult = await SharedWorkspaceLocking.LockWorkspace(workspace, runId, buildParametersContext);
|
|
||||||
CloudRunnerLogger.log(
|
|
||||||
`run agent ${runId} didn't find a free workspace so created: ${workspace} createWorkspaceSuccess: ${createResult} Lock:${lockResult}`,
|
|
||||||
);
|
|
||||||
|
|
||||||
return createResult && lockResult;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async DoesWorkspaceExist(workspace: string, buildParametersContext: BuildParameters) {
|
|
||||||
return (
|
|
||||||
(await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext)).filter((x) => x.includes(workspace))
|
|
||||||
.length > 0
|
|
||||||
);
|
|
||||||
}
|
|
||||||
public static async HasWorkspaceLock(
|
|
||||||
workspace: string,
|
|
||||||
runId: string,
|
|
||||||
buildParametersContext: BuildParameters,
|
|
||||||
): Promise<boolean> {
|
|
||||||
const locks = (await SharedWorkspaceLocking.GetAllLocksForWorkspace(workspace, buildParametersContext))
|
|
||||||
.map((x) => {
|
|
||||||
return {
|
|
||||||
name: x,
|
|
||||||
timestamp: Number(x.split(`_`)[0]),
|
|
||||||
};
|
|
||||||
})
|
|
||||||
.sort((x) => x.timestamp);
|
|
||||||
const lockMatches = locks.filter((x) => x.name.includes(runId));
|
|
||||||
const includesRunLock = lockMatches.length > 0 && locks.indexOf(lockMatches[0]) === 0;
|
|
||||||
CloudRunnerLogger.log(
|
|
||||||
`Checking has workspace lock, runId: ${runId}, workspace: ${workspace}, success: ${includesRunLock} \n- Num of locks created by Run Agent: ${
|
|
||||||
lockMatches.length
|
|
||||||
} Num of Locks: ${locks.length}, Time ordered index for Run Agent: ${locks.indexOf(lockMatches[0])} \n \n`,
|
|
||||||
);
|
|
||||||
|
|
||||||
return includesRunLock;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async GetFreeWorkspaces(buildParametersContext: BuildParameters): Promise<string[]> {
|
|
||||||
const result: string[] = [];
|
|
||||||
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
|
|
||||||
for (const element of workspaces) {
|
|
||||||
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext);
|
|
||||||
const isBelowMax = await SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParametersContext);
|
|
||||||
CloudRunnerLogger.log(`workspace ${element} locked:${isLocked} below max:${isBelowMax}`);
|
|
||||||
if (!isLocked && isBelowMax) {
|
|
||||||
result.push(element);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async IsWorkspaceBelowMax(
|
|
||||||
workspace: string,
|
|
||||||
buildParametersContext: BuildParameters,
|
|
||||||
): Promise<boolean> {
|
|
||||||
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
|
|
||||||
if (workspace === ``) {
|
|
||||||
return (
|
|
||||||
workspaces.length < buildParametersContext.maxRetainedWorkspaces ||
|
|
||||||
buildParametersContext.maxRetainedWorkspaces === 0
|
|
||||||
);
|
|
||||||
}
|
|
||||||
const ordered: any[] = [];
|
|
||||||
for (const ws of workspaces) {
|
|
||||||
ordered.push({
|
|
||||||
name: ws,
|
|
||||||
timestamp: await SharedWorkspaceLocking.GetWorkspaceTimestamp(ws, buildParametersContext),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
ordered.sort((x) => x.timestamp);
|
|
||||||
const matches = ordered.filter((x) => x.name.includes(workspace));
|
|
||||||
const isWorkspaceBelowMax =
|
|
||||||
matches.length > 0 &&
|
|
||||||
(ordered.indexOf(matches[0]) < buildParametersContext.maxRetainedWorkspaces ||
|
|
||||||
buildParametersContext.maxRetainedWorkspaces === 0);
|
|
||||||
|
|
||||||
return isWorkspaceBelowMax;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async GetWorkspaceTimestamp(
|
|
||||||
workspace: string,
|
|
||||||
buildParametersContext: BuildParameters,
|
|
||||||
): Promise<Number> {
|
|
||||||
if (workspace.split(`_`).length > 0) {
|
|
||||||
return Number(workspace.split(`_`)[1]);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
|
|
||||||
throw new Error("Workspace doesn't exist, can't call get all locks");
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
|
||||||
await SharedWorkspaceLocking.ReadLines(
|
|
||||||
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
.map((x) => x.replace(`/`, ``))
|
|
||||||
.filter((x) => x.includes(workspace) && x.endsWith(`_workspace`))
|
|
||||||
.map((x) => Number(x))[0];
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async IsWorkspaceLocked(workspace: string, buildParametersContext: BuildParameters): Promise<boolean> {
|
|
||||||
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
|
|
||||||
throw new Error(`workspace doesn't exist ${workspace}`);
|
|
||||||
}
|
|
||||||
const files = await SharedWorkspaceLocking.ReadLines(
|
|
||||||
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
|
|
||||||
);
|
|
||||||
|
|
||||||
const lockFilesExist =
|
|
||||||
files.filter((x) => {
|
|
||||||
return x.includes(workspace) && x.endsWith(`_lock`);
|
|
||||||
}).length > 0;
|
|
||||||
|
|
||||||
return lockFilesExist;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async CreateWorkspace(workspace: string, buildParametersContext: BuildParameters): Promise<boolean> {
|
|
||||||
if (await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext)) {
|
|
||||||
throw new Error(`${workspace} already exists`);
|
|
||||||
}
|
|
||||||
const timestamp = Date.now();
|
|
||||||
const file = `${timestamp}_${workspace}_workspace`;
|
|
||||||
fs.writeFileSync(file, '');
|
|
||||||
await CloudRunnerSystem.Run(
|
|
||||||
`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
|
|
||||||
false,
|
|
||||||
true,
|
|
||||||
);
|
|
||||||
fs.rmSync(file);
|
|
||||||
|
|
||||||
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
|
|
||||||
|
|
||||||
CloudRunnerLogger.log(`All workspaces ${workspaces}`);
|
|
||||||
if (!(await SharedWorkspaceLocking.IsWorkspaceBelowMax(workspace, buildParametersContext))) {
|
|
||||||
CloudRunnerLogger.log(`Workspace is above max ${workspaces} ${buildParametersContext.maxRetainedWorkspaces}`);
|
|
||||||
await SharedWorkspaceLocking.CleanupWorkspace(workspace, buildParametersContext);
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async LockWorkspace(
|
|
||||||
workspace: string,
|
|
||||||
runId: string,
|
|
||||||
buildParametersContext: BuildParameters,
|
|
||||||
): Promise<boolean> {
|
|
||||||
const existingWorkspace = workspace.endsWith(`_workspace`);
|
|
||||||
const ending = existingWorkspace ? workspace : `${workspace}_workspace`;
|
|
||||||
const file = `${Date.now()}_${runId}_${ending}_lock`;
|
|
||||||
fs.writeFileSync(file, '');
|
|
||||||
await CloudRunnerSystem.Run(
|
|
||||||
`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
|
|
||||||
false,
|
|
||||||
true,
|
|
||||||
);
|
|
||||||
fs.rmSync(file);
|
|
||||||
|
|
||||||
const hasLock = await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
|
|
||||||
|
|
||||||
if (hasLock) {
|
|
||||||
CloudRunner.lockedWorkspace = workspace;
|
|
||||||
} else {
|
|
||||||
await CloudRunnerSystem.Run(
|
|
||||||
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
|
|
||||||
false,
|
|
||||||
true,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return hasLock;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async ReleaseWorkspace(
|
|
||||||
workspace: string,
|
|
||||||
runId: string,
|
|
||||||
buildParametersContext: BuildParameters,
|
|
||||||
): Promise<boolean> {
|
|
||||||
const files = await SharedWorkspaceLocking.GetAllLocksForWorkspace(workspace, buildParametersContext);
|
|
||||||
const file = files.find((x) => x.includes(workspace) && x.endsWith(`_lock`) && x.includes(runId));
|
|
||||||
CloudRunnerLogger.log(`All Locks ${files} ${workspace} ${runId}`);
|
|
||||||
CloudRunnerLogger.log(`Deleting lock ${workspace}/${file}`);
|
|
||||||
CloudRunnerLogger.log(`rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`);
|
|
||||||
await CloudRunnerSystem.Run(
|
|
||||||
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
|
|
||||||
false,
|
|
||||||
true,
|
|
||||||
);
|
|
||||||
|
|
||||||
return !(await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext));
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async CleanupWorkspace(workspace: string, buildParametersContext: BuildParameters) {
|
|
||||||
await CloudRunnerSystem.Run(
|
|
||||||
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey} --exclude "*" --include "*_${workspace}_*"`,
|
|
||||||
false,
|
|
||||||
true,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async ReadLines(command: string): Promise<string[]> {
|
|
||||||
return CloudRunnerSystem.RunAndReadLines(command);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export default SharedWorkspaceLocking;
|
|
||||||
@@ -1,175 +0,0 @@
|
|||||||
import BuildParameters from '../../../build-parameters';
|
|
||||||
import Input from '../../../input';
|
|
||||||
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
|
||||||
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
|
|
||||||
import CloudRunnerOptionsReader from '../../options/cloud-runner-options-reader';
|
|
||||||
import CloudRunnerQueryOverride from '../../options/cloud-runner-query-override';
|
|
||||||
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
|
||||||
import { CommandHookService } from '../hooks/command-hook-service';
|
|
||||||
|
|
||||||
export class TaskParameterSerializer {
|
|
||||||
static readonly blockedParameterNames: Set<string> = new Set([
|
|
||||||
'0',
|
|
||||||
'length',
|
|
||||||
'prototype',
|
|
||||||
'',
|
|
||||||
'unityVersion',
|
|
||||||
'CACHE_UNITY_INSTALLATION_ON_MAC',
|
|
||||||
'RUNNER_TEMP_PATH',
|
|
||||||
'NAME',
|
|
||||||
'CUSTOM_JOB',
|
|
||||||
]);
|
|
||||||
public static createCloudRunnerEnvironmentVariables(
|
|
||||||
buildParameters: BuildParameters,
|
|
||||||
): CloudRunnerEnvironmentVariable[] {
|
|
||||||
const result: CloudRunnerEnvironmentVariable[] = this.uniqBy(
|
|
||||||
[
|
|
||||||
...[
|
|
||||||
{ name: 'BUILD_TARGET', value: buildParameters.targetPlatform },
|
|
||||||
{ name: 'UNITY_VERSION', value: buildParameters.editorVersion },
|
|
||||||
{ name: 'GITHUB_TOKEN', value: process.env.GITHUB_TOKEN },
|
|
||||||
],
|
|
||||||
...TaskParameterSerializer.serializeFromObject(buildParameters),
|
|
||||||
...TaskParameterSerializer.serializeInput(),
|
|
||||||
...TaskParameterSerializer.serializeCloudRunnerOptions(),
|
|
||||||
...CommandHookService.getSecrets(CommandHookService.getHooks(buildParameters.commandHooks)),
|
|
||||||
]
|
|
||||||
.filter(
|
|
||||||
(x) =>
|
|
||||||
!TaskParameterSerializer.blockedParameterNames.has(x.name) &&
|
|
||||||
x.value !== '' &&
|
|
||||||
x.value !== undefined &&
|
|
||||||
x.value !== `undefined`,
|
|
||||||
)
|
|
||||||
.map((x) => {
|
|
||||||
x.name = `${TaskParameterSerializer.ToEnvVarFormat(x.name)}`;
|
|
||||||
x.value = `${x.value}`;
|
|
||||||
|
|
||||||
return x;
|
|
||||||
}),
|
|
||||||
(item: CloudRunnerEnvironmentVariable) => item.name,
|
|
||||||
);
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
// eslint-disable-next-line no-unused-vars
|
|
||||||
static uniqBy(a: CloudRunnerEnvironmentVariable[], key: (parameters: CloudRunnerEnvironmentVariable) => string) {
|
|
||||||
const seen: { [key: string]: boolean } = {};
|
|
||||||
|
|
||||||
return a.filter(function (item) {
|
|
||||||
const k = key(item);
|
|
||||||
|
|
||||||
return seen.hasOwnProperty(k) ? false : (seen[k] = true);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
public static readBuildParameterFromEnvironment(): BuildParameters {
|
|
||||||
const buildParameters = new BuildParameters();
|
|
||||||
const keys = [
|
|
||||||
...new Set(
|
|
||||||
Object.getOwnPropertyNames(process.env)
|
|
||||||
.filter((x) => !this.blockedParameterNames.has(x) && x.startsWith(''))
|
|
||||||
.map((x) => TaskParameterSerializer.UndoEnvVarFormat(x)),
|
|
||||||
),
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const element of keys) {
|
|
||||||
if (element !== `customJob`) {
|
|
||||||
buildParameters[element] = process.env[`${TaskParameterSerializer.ToEnvVarFormat(element)}`];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return buildParameters;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static serializeInput() {
|
|
||||||
return TaskParameterSerializer.serializeFromType(Input);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static serializeCloudRunnerOptions() {
|
|
||||||
return TaskParameterSerializer.serializeFromType(CloudRunnerOptions);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static ToEnvVarFormat(input: string): string {
|
|
||||||
return CloudRunnerOptions.ToEnvVarFormat(input);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static UndoEnvVarFormat(element: string): string {
|
|
||||||
return this.camelize(element.toLowerCase().replace(/_+/g, ' '));
|
|
||||||
}
|
|
||||||
|
|
||||||
private static camelize(string: string) {
|
|
||||||
return TaskParameterSerializer.uncapitalizeFirstLetter(
|
|
||||||
string
|
|
||||||
.replace(/(^\w)|([A-Z])|(\b\w)/g, function (word: string, index: number) {
|
|
||||||
return index === 0 ? word.toLowerCase() : word.toUpperCase();
|
|
||||||
})
|
|
||||||
.replace(/\s+/g, ''),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static uncapitalizeFirstLetter(string: string) {
|
|
||||||
return string.charAt(0).toLowerCase() + string.slice(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static serializeFromObject(buildParameters: any) {
|
|
||||||
const array: any[] = [];
|
|
||||||
const keys = Object.getOwnPropertyNames(buildParameters).filter((x) => !this.blockedParameterNames.has(x));
|
|
||||||
for (const element of keys) {
|
|
||||||
array.push({
|
|
||||||
name: TaskParameterSerializer.ToEnvVarFormat(element),
|
|
||||||
value: buildParameters[element],
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return array;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static serializeFromType(type: any) {
|
|
||||||
const array: any[] = [];
|
|
||||||
const input = CloudRunnerOptionsReader.GetProperties();
|
|
||||||
for (const element of input) {
|
|
||||||
if (typeof type[element] !== 'function' && array.filter((x) => x.name === element).length === 0) {
|
|
||||||
array.push({
|
|
||||||
name: element,
|
|
||||||
value: `${type[element]}`,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return array;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static readDefaultSecrets(): CloudRunnerSecret[] {
|
|
||||||
let array = new Array();
|
|
||||||
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_SERIAL');
|
|
||||||
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_EMAIL');
|
|
||||||
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_PASSWORD');
|
|
||||||
|
|
||||||
// array = TaskParameterSerializer.tryAddInput(array, 'UNITY_LICENSE');
|
|
||||||
array = TaskParameterSerializer.tryAddInput(array, 'GIT_PRIVATE_TOKEN');
|
|
||||||
|
|
||||||
return array;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static getValue(key: string) {
|
|
||||||
return CloudRunnerQueryOverride.queryOverrides !== undefined &&
|
|
||||||
CloudRunnerQueryOverride.queryOverrides[key] !== undefined
|
|
||||||
? CloudRunnerQueryOverride.queryOverrides[key]
|
|
||||||
: process.env[key];
|
|
||||||
}
|
|
||||||
|
|
||||||
private static tryAddInput(array: CloudRunnerSecret[], key: string): CloudRunnerSecret[] {
|
|
||||||
const value = TaskParameterSerializer.getValue(key);
|
|
||||||
if (value !== undefined && value !== '' && value !== 'null') {
|
|
||||||
array.push({
|
|
||||||
ParameterKey: key,
|
|
||||||
EnvironmentVariable: key,
|
|
||||||
ParameterValue: value,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return array;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,118 +0,0 @@
|
|||||||
import { BuildParameters, Input } from '../../..';
|
|
||||||
import YAML from 'yaml';
|
|
||||||
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
|
|
||||||
import path from 'node:path';
|
|
||||||
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
|
||||||
import * as fs from 'node:fs';
|
|
||||||
import CloudRunnerLogger from '../core/cloud-runner-logger';
|
|
||||||
import { CommandHook } from './command-hook';
|
|
||||||
|
|
||||||
// import CloudRunnerLogger from './cloud-runner-logger';
|
|
||||||
|
|
||||||
export class CommandHookService {
|
|
||||||
public static ApplyHooksToCommands(commands: string, buildParameters: BuildParameters): string {
|
|
||||||
const hooks = CommandHookService.getHooks(buildParameters.commandHooks);
|
|
||||||
CloudRunnerLogger.log(`Applying hooks ${hooks.length}`);
|
|
||||||
|
|
||||||
return `echo "---"
|
|
||||||
echo "start cloud runner init"
|
|
||||||
${CloudRunnerOptions.cloudRunnerDebug ? `printenv` : `#`}
|
|
||||||
echo "start of cloud runner job"
|
|
||||||
${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
|
|
||||||
${commands}
|
|
||||||
${hooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
|
|
||||||
echo "end of cloud runner job"
|
|
||||||
echo "---${buildParameters.logId}"`;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static getHooks(customCommandHooks: string): CommandHook[] {
|
|
||||||
const experimentHooks = customCommandHooks;
|
|
||||||
let output = new Array<CommandHook>();
|
|
||||||
if (experimentHooks && experimentHooks !== '') {
|
|
||||||
try {
|
|
||||||
output = YAML.parse(experimentHooks);
|
|
||||||
} catch (error) {
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return [
|
|
||||||
...output.filter((x) => x.hook !== undefined && x.hook.length > 0),
|
|
||||||
...CommandHookService.GetCustomHooksFromFiles(`before`),
|
|
||||||
...CommandHookService.GetCustomHooksFromFiles(`after`),
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
static GetCustomHooksFromFiles(hookLifecycle: string): CommandHook[] {
|
|
||||||
const results: CommandHook[] = [];
|
|
||||||
|
|
||||||
// RemoteClientLogger.log(`GetCustomHookFiles: ${hookLifecycle}`);
|
|
||||||
try {
|
|
||||||
const gameCiCustomHooksPath = path.join(process.cwd(), `game-ci`, `command-hooks`);
|
|
||||||
const files = fs.readdirSync(gameCiCustomHooksPath);
|
|
||||||
for (const file of files) {
|
|
||||||
if (!CloudRunnerOptions.commandHookFiles.includes(file.replace(`.yaml`, ``))) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
|
|
||||||
const fileContentsObject = CommandHookService.ParseHooks(fileContents)[0];
|
|
||||||
if (fileContentsObject.hook.includes(hookLifecycle)) {
|
|
||||||
results.push(fileContentsObject);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
RemoteClientLogger.log(`Failed Getting: ${hookLifecycle} \n ${JSON.stringify(error, undefined, 4)}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// RemoteClientLogger.log(`Active Steps From Hooks: \n ${JSON.stringify(results, undefined, 4)}`);
|
|
||||||
|
|
||||||
return results;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static ConvertYamlSecrets(object: CommandHook) {
|
|
||||||
if (object.secrets === undefined) {
|
|
||||||
object.secrets = [];
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
object.secrets = object.secrets.map((x: any) => {
|
|
||||||
return {
|
|
||||||
ParameterKey: x.name,
|
|
||||||
EnvironmentVariable: Input.ToEnvVarFormat(x.name),
|
|
||||||
ParameterValue: x.value,
|
|
||||||
};
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
public static ParseHooks(hooks: string): CommandHook[] {
|
|
||||||
if (hooks === '') {
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
|
|
||||||
// if (CloudRunner.buildParameters?.cloudRunnerIntegrationTests) {
|
|
||||||
|
|
||||||
// CloudRunnerLogger.log(`Parsing build hooks: ${steps}`);
|
|
||||||
|
|
||||||
// }
|
|
||||||
const isArray = hooks.replace(/\s/g, ``)[0] === `-`;
|
|
||||||
const object: CommandHook[] = isArray ? YAML.parse(hooks) : [YAML.parse(hooks)];
|
|
||||||
for (const hook of object) {
|
|
||||||
CommandHookService.ConvertYamlSecrets(hook);
|
|
||||||
if (hook.secrets === undefined) {
|
|
||||||
hook.secrets = [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (object === undefined) {
|
|
||||||
throw new Error(`Failed to parse ${hooks}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return object;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static getSecrets(hooks: any) {
|
|
||||||
const secrets = hooks.map((x: any) => x.secrets).filter((x: any) => x !== undefined && x.length > 0);
|
|
||||||
|
|
||||||
// eslint-disable-next-line unicorn/no-array-reduce
|
|
||||||
return secrets.length > 0 ? secrets.reduce((x: any, y: any) => [...x, ...y]) : [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
|
||||||
|
|
||||||
export class CommandHook {
|
|
||||||
public commands: string[] = new Array<string>();
|
|
||||||
public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
|
|
||||||
public name!: string;
|
|
||||||
public hook!: string[];
|
|
||||||
public step!: string[];
|
|
||||||
}
|
|
||||||
@@ -1,265 +0,0 @@
|
|||||||
import YAML from 'yaml';
|
|
||||||
import CloudRunner from '../../cloud-runner';
|
|
||||||
import { CustomWorkflow } from '../../workflows/custom-workflow';
|
|
||||||
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
|
|
||||||
import path from 'node:path';
|
|
||||||
import fs from 'node:fs';
|
|
||||||
import Input from '../../../input';
|
|
||||||
import CloudRunnerOptions from '../../options/cloud-runner-options';
|
|
||||||
import { ContainerHook as ContainerHook } from './container-hook';
|
|
||||||
import { CloudRunnerStepParameters } from '../../options/cloud-runner-step-parameters';
|
|
||||||
|
|
||||||
export class ContainerHookService {
|
|
||||||
static GetContainerHooksFromFiles(hookLifecycle: string): ContainerHook[] {
|
|
||||||
const results: ContainerHook[] = [];
|
|
||||||
try {
|
|
||||||
const gameCiCustomStepsPath = path.join(process.cwd(), `game-ci`, `container-hooks`);
|
|
||||||
const files = fs.readdirSync(gameCiCustomStepsPath);
|
|
||||||
for (const file of files) {
|
|
||||||
if (!CloudRunnerOptions.containerHookFiles.includes(file.replace(`.yaml`, ``))) {
|
|
||||||
// RemoteClientLogger.log(`Skipping CustomStepFile: ${file}`);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const fileContents = fs.readFileSync(path.join(gameCiCustomStepsPath, file), `utf8`);
|
|
||||||
const fileContentsObject = ContainerHookService.ParseContainerHooks(fileContents)[0];
|
|
||||||
if (fileContentsObject.hook === hookLifecycle) {
|
|
||||||
results.push(fileContentsObject);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
RemoteClientLogger.log(`Failed Getting: ${hookLifecycle} \n ${JSON.stringify(error, undefined, 4)}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// RemoteClientLogger.log(`Active Steps From Files: \n ${JSON.stringify(results, undefined, 4)}`);
|
|
||||||
|
|
||||||
const builtInContainerHooks: ContainerHook[] = ContainerHookService.ParseContainerHooks(
|
|
||||||
`- name: aws-s3-upload-build
|
|
||||||
image: amazon/aws-cli
|
|
||||||
hook: after
|
|
||||||
commands: |
|
|
||||||
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
|
|
||||||
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
|
|
||||||
aws configure set region $AWS_DEFAULT_REGION --profile default
|
|
||||||
aws s3 cp /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
|
|
||||||
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
|
||||||
} s3://${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${
|
|
||||||
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
|
||||||
}
|
|
||||||
rm /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
|
|
||||||
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
|
||||||
}
|
|
||||||
secrets:
|
|
||||||
- name: awsAccessKeyId
|
|
||||||
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
|
|
||||||
- name: awsSecretAccessKey
|
|
||||||
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
|
|
||||||
- name: awsDefaultRegion
|
|
||||||
value: ${process.env.AWS_REGION || ``}
|
|
||||||
- name: aws-s3-pull-build
|
|
||||||
image: amazon/aws-cli
|
|
||||||
commands: |
|
|
||||||
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
|
|
||||||
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
|
|
||||||
aws configure set region $AWS_DEFAULT_REGION --profile default
|
|
||||||
aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/ || true
|
|
||||||
aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/build || true
|
|
||||||
mkdir -p /data/cache/$CACHE_KEY/build/
|
|
||||||
aws s3 cp s3://${
|
|
||||||
CloudRunner.buildParameters.awsStackName
|
|
||||||
}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
|
||||||
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
|
||||||
} /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
|
||||||
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
|
|
||||||
}
|
|
||||||
secrets:
|
|
||||||
- name: AWS_ACCESS_KEY_ID
|
|
||||||
- name: AWS_SECRET_ACCESS_KEY
|
|
||||||
- name: AWS_DEFAULT_REGION
|
|
||||||
- name: BUILD_GUID_TARGET
|
|
||||||
- name: steam-deploy-client
|
|
||||||
image: steamcmd/steamcmd
|
|
||||||
commands: |
|
|
||||||
apt-get update
|
|
||||||
apt-get install -y curl tar coreutils git tree > /dev/null
|
|
||||||
curl -s https://gist.githubusercontent.com/frostebite/1d56f5505b36b403b64193b7a6e54cdc/raw/fa6639ed4ef750c4268ea319d63aa80f52712ffb/deploy-client-steam.sh | bash
|
|
||||||
secrets:
|
|
||||||
- name: STEAM_USERNAME
|
|
||||||
- name: STEAM_PASSWORD
|
|
||||||
- name: STEAM_APPID
|
|
||||||
- name: STEAM_SSFN_FILE_NAME
|
|
||||||
- name: STEAM_SSFN_FILE_CONTENTS
|
|
||||||
- name: STEAM_CONFIG_VDF_1
|
|
||||||
- name: STEAM_CONFIG_VDF_2
|
|
||||||
- name: STEAM_CONFIG_VDF_3
|
|
||||||
- name: STEAM_CONFIG_VDF_4
|
|
||||||
- name: BUILD_GUID_TARGET
|
|
||||||
- name: RELEASE_BRANCH
|
|
||||||
- name: steam-deploy-project
|
|
||||||
image: steamcmd/steamcmd
|
|
||||||
commands: |
|
|
||||||
apt-get update
|
|
||||||
apt-get install -y curl tar coreutils git tree > /dev/null
|
|
||||||
curl -s https://gist.githubusercontent.com/frostebite/969da6a41002a0e901174124b643709f/raw/02403e53fb292026cba81ddcf4ff35fc1eba111d/steam-deploy-project.sh | bash
|
|
||||||
secrets:
|
|
||||||
- name: STEAM_USERNAME
|
|
||||||
- name: STEAM_PASSWORD
|
|
||||||
- name: STEAM_APPID
|
|
||||||
- name: STEAM_SSFN_FILE_NAME
|
|
||||||
- name: STEAM_SSFN_FILE_CONTENTS
|
|
||||||
- name: STEAM_CONFIG_VDF_1
|
|
||||||
- name: STEAM_CONFIG_VDF_2
|
|
||||||
- name: STEAM_CONFIG_VDF_3
|
|
||||||
- name: STEAM_CONFIG_VDF_4
|
|
||||||
- name: BUILD_GUID_2
|
|
||||||
- name: RELEASE_BRANCH
|
|
||||||
- name: aws-s3-upload-cache
|
|
||||||
image: amazon/aws-cli
|
|
||||||
hook: after
|
|
||||||
commands: |
|
|
||||||
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
|
|
||||||
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
|
|
||||||
aws configure set region $AWS_DEFAULT_REGION --profile default
|
|
||||||
aws s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://${
|
|
||||||
CloudRunner.buildParameters.awsStackName
|
|
||||||
}/cloud-runner-cache/$CACHE_KEY/lfs
|
|
||||||
rm -r /data/cache/$CACHE_KEY/lfs
|
|
||||||
aws s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://${
|
|
||||||
CloudRunner.buildParameters.awsStackName
|
|
||||||
}/cloud-runner-cache/$CACHE_KEY/Library
|
|
||||||
rm -r /data/cache/$CACHE_KEY/Library
|
|
||||||
secrets:
|
|
||||||
- name: AWS_ACCESS_KEY_ID
|
|
||||||
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
|
|
||||||
- name: AWS_SECRET_ACCESS_KEY
|
|
||||||
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
|
|
||||||
- name: AWS_DEFAULT_REGION
|
|
||||||
value: ${process.env.AWS_REGION || ``}
|
|
||||||
- name: aws-s3-pull-cache
|
|
||||||
image: amazon/aws-cli
|
|
||||||
hook: before
|
|
||||||
commands: |
|
|
||||||
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
|
|
||||||
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
|
|
||||||
aws configure set region $AWS_DEFAULT_REGION --profile default
|
|
||||||
mkdir -p /data/cache/$CACHE_KEY/Library/
|
|
||||||
mkdir -p /data/cache/$CACHE_KEY/lfs/
|
|
||||||
aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/ || true
|
|
||||||
aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/ || true
|
|
||||||
BUCKET1="${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/Library/"
|
|
||||||
aws s3 ls $BUCKET1 || true
|
|
||||||
OBJECT1="$(aws s3 ls $BUCKET1 | sort | tail -n 1 | awk '{print $4}' || '')"
|
|
||||||
aws s3 cp s3://$BUCKET1$OBJECT1 /data/cache/$CACHE_KEY/Library/ || true
|
|
||||||
BUCKET2="${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/lfs/"
|
|
||||||
aws s3 ls $BUCKET2 || true
|
|
||||||
OBJECT2="$(aws s3 ls $BUCKET2 | sort | tail -n 1 | awk '{print $4}' || '')"
|
|
||||||
aws s3 cp s3://$BUCKET2$OBJECT2 /data/cache/$CACHE_KEY/lfs/ || true
|
|
||||||
secrets:
|
|
||||||
- name: AWS_ACCESS_KEY_ID
|
|
||||||
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
|
|
||||||
- name: AWS_SECRET_ACCESS_KEY
|
|
||||||
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
|
|
||||||
- name: AWS_DEFAULT_REGION
|
|
||||||
value: ${process.env.AWS_REGION || ``}
|
|
||||||
- name: debug-cache
|
|
||||||
image: ubuntu
|
|
||||||
hook: after
|
|
||||||
commands: |
|
|
||||||
apt-get update > /dev/null
|
|
||||||
${CloudRunnerOptions.cloudRunnerDebug ? `apt-get install -y tree > /dev/null` : `#`}
|
|
||||||
${CloudRunnerOptions.cloudRunnerDebug ? `tree -L 3 /data/cache` : `#`}
|
|
||||||
secrets:
|
|
||||||
- name: awsAccessKeyId
|
|
||||||
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
|
|
||||||
- name: awsSecretAccessKey
|
|
||||||
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
|
|
||||||
- name: awsDefaultRegion
|
|
||||||
value: ${process.env.AWS_REGION || ``}`,
|
|
||||||
).filter((x) => CloudRunnerOptions.containerHookFiles.includes(x.name) && x.hook === hookLifecycle);
|
|
||||||
if (builtInContainerHooks.length > 0) {
|
|
||||||
results.push(...builtInContainerHooks);
|
|
||||||
}
|
|
||||||
|
|
||||||
return results;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static ConvertYamlSecrets(object: ContainerHook) {
|
|
||||||
if (object.secrets === undefined) {
|
|
||||||
object.secrets = [];
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
object.secrets = object.secrets.map((x: { [key: string]: any }) => {
|
|
||||||
return {
|
|
||||||
ParameterKey: x.name,
|
|
||||||
EnvironmentVariable: Input.ToEnvVarFormat(x.name),
|
|
||||||
ParameterValue: x.value,
|
|
||||||
};
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
public static ParseContainerHooks(steps: string): ContainerHook[] {
|
|
||||||
if (steps === '') {
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
const isArray = steps.replace(/\s/g, ``)[0] === `-`;
|
|
||||||
const object: ContainerHook[] = isArray ? YAML.parse(steps) : [YAML.parse(steps)];
|
|
||||||
for (const step of object) {
|
|
||||||
ContainerHookService.ConvertYamlSecrets(step);
|
|
||||||
if (step.secrets === undefined) {
|
|
||||||
step.secrets = [];
|
|
||||||
} else {
|
|
||||||
for (const secret of step.secrets) {
|
|
||||||
if (secret.ParameterValue === undefined && process.env[secret.EnvironmentVariable] !== undefined) {
|
|
||||||
if (CloudRunner.buildParameters?.cloudRunnerDebug) {
|
|
||||||
// CloudRunnerLogger.log(`Injecting custom step ${step.name} from env var ${secret.ParameterKey}`);
|
|
||||||
}
|
|
||||||
secret.ParameterValue = process.env[secret.ParameterKey] || ``;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (step.image === undefined) {
|
|
||||||
step.image = `ubuntu`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (object === undefined) {
|
|
||||||
throw new Error(`Failed to parse ${steps}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return object;
|
|
||||||
}
|
|
||||||
|
|
||||||
static async RunPostBuildSteps(cloudRunnerStepState: CloudRunnerStepParameters) {
|
|
||||||
let output = ``;
|
|
||||||
const steps: ContainerHook[] = [
|
|
||||||
...ContainerHookService.ParseContainerHooks(CloudRunner.buildParameters.postBuildContainerHooks),
|
|
||||||
...ContainerHookService.GetContainerHooksFromFiles(`after`),
|
|
||||||
];
|
|
||||||
|
|
||||||
if (steps.length > 0) {
|
|
||||||
output += await CustomWorkflow.runContainerJob(
|
|
||||||
steps,
|
|
||||||
cloudRunnerStepState.environment,
|
|
||||||
cloudRunnerStepState.secrets,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return output;
|
|
||||||
}
|
|
||||||
static async RunPreBuildSteps(cloudRunnerStepState: CloudRunnerStepParameters) {
|
|
||||||
let output = ``;
|
|
||||||
const steps: ContainerHook[] = [
|
|
||||||
...ContainerHookService.ParseContainerHooks(CloudRunner.buildParameters.preBuildContainerHooks),
|
|
||||||
...ContainerHookService.GetContainerHooksFromFiles(`before`),
|
|
||||||
];
|
|
||||||
|
|
||||||
if (steps.length > 0) {
|
|
||||||
output += await CustomWorkflow.runContainerJob(
|
|
||||||
steps,
|
|
||||||
cloudRunnerStepState.environment,
|
|
||||||
cloudRunnerStepState.secrets,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return output;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
import CloudRunnerSecret from '../../options/cloud-runner-secret';
|
|
||||||
|
|
||||||
export class ContainerHook {
|
|
||||||
public commands!: string;
|
|
||||||
public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
|
|
||||||
public name!: string;
|
|
||||||
public image: string = `ubuntu`;
|
|
||||||
public hook!: string;
|
|
||||||
}
|
|
||||||
@@ -1,43 +0,0 @@
|
|||||||
import path from 'node:path';
|
|
||||||
import { CloudRunnerFolders } from '../../options/cloud-runner-folders';
|
|
||||||
import { CloudRunnerSystem } from '../core/cloud-runner-system';
|
|
||||||
import fs from 'node:fs';
|
|
||||||
import { Cli } from '../../../cli/cli';
|
|
||||||
import { CliFunction } from '../../../cli/cli-functions-repository';
|
|
||||||
|
|
||||||
export class LfsHashing {
|
|
||||||
public static async createLFSHashFiles() {
|
|
||||||
await CloudRunnerSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`);
|
|
||||||
await CloudRunnerSystem.Run(`md5sum .lfs-assets-guid > .lfs-assets-guid-sum`);
|
|
||||||
const lfsHashes = {
|
|
||||||
lfsGuid: fs
|
|
||||||
.readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid`)}`, 'utf8')
|
|
||||||
.replace(/\n/g, ``),
|
|
||||||
lfsGuidSum: fs
|
|
||||||
.readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid-sum`)}`, 'utf8')
|
|
||||||
.replace(' .lfs-assets-guid', '')
|
|
||||||
.replace(/\n/g, ``),
|
|
||||||
};
|
|
||||||
|
|
||||||
return lfsHashes;
|
|
||||||
}
|
|
||||||
public static async hashAllFiles(folder: string) {
|
|
||||||
const startPath = process.cwd();
|
|
||||||
process.chdir(folder);
|
|
||||||
const result = await (await CloudRunnerSystem.Run(`find -type f -exec md5sum "{}" + | sort | md5sum`))
|
|
||||||
.replace(/\n/g, '')
|
|
||||||
.split(` `)[0];
|
|
||||||
process.chdir(startPath);
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
@CliFunction(`hash`, `hash all folder contents`)
|
|
||||||
static async hash() {
|
|
||||||
if (!Cli.options) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const folder = Cli.options['cachePushFrom'];
|
|
||||||
LfsHashing.hashAllFiles(folder);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,40 +0,0 @@
|
|||||||
import { BuildParameters, ImageTag } from '../..';
|
|
||||||
import CloudRunner from '../cloud-runner';
|
|
||||||
import UnityVersioning from '../../unity-versioning';
|
|
||||||
import { Cli } from '../../cli/cli';
|
|
||||||
import CloudRunnerOptions from '../options/cloud-runner-options';
|
|
||||||
import setups from './cloud-runner-suite.test';
|
|
||||||
import { OptionValues } from 'commander';
|
|
||||||
|
|
||||||
async function CreateParameters(overrides: OptionValues | undefined) {
|
|
||||||
if (overrides) Cli.options = overrides;
|
|
||||||
|
|
||||||
return BuildParameters.create();
|
|
||||||
}
|
|
||||||
describe('Cloud Runner Async Workflows', () => {
|
|
||||||
setups();
|
|
||||||
it('Responds', () => {});
|
|
||||||
|
|
||||||
if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.providerStrategy !== `local-docker`) {
|
|
||||||
it('Async Workflows', async () => {
|
|
||||||
// Setup parameters
|
|
||||||
const buildParameter = await CreateParameters({
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.read('test-project'),
|
|
||||||
asyncCloudRunner: `true`,
|
|
||||||
githubChecks: `true`,
|
|
||||||
providerStrategy: 'k8s',
|
|
||||||
buildPlatform: 'linux',
|
|
||||||
targetPlatform: 'StandaloneLinux64',
|
|
||||||
});
|
|
||||||
const baseImage = new ImageTag(buildParameter);
|
|
||||||
|
|
||||||
// Run the job
|
|
||||||
await CloudRunner.run(buildParameter, baseImage.toString());
|
|
||||||
|
|
||||||
// wait for 15 seconds
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 1000 * 60 * 12));
|
|
||||||
}, 1_000_000_000);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
@@ -1,59 +0,0 @@
|
|||||||
import fs from 'node:fs';
|
|
||||||
import path from 'node:path';
|
|
||||||
import BuildParameters from '../../build-parameters';
|
|
||||||
import { Cli } from '../../cli/cli';
|
|
||||||
import UnityVersioning from '../../unity-versioning';
|
|
||||||
import CloudRunner from '../cloud-runner';
|
|
||||||
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
|
|
||||||
import { Caching } from '../remote-client/caching';
|
|
||||||
import { v4 as uuidv4 } from 'uuid';
|
|
||||||
import GitHub from '../../github';
|
|
||||||
import CloudRunnerOptions from '../options/cloud-runner-options';
|
|
||||||
describe('Cloud Runner (Remote Client) Caching', () => {
|
|
||||||
it('responds', () => {});
|
|
||||||
if (CloudRunnerOptions.providerStrategy === `local-docker`) {
|
|
||||||
it('Simple caching works', async () => {
|
|
||||||
Cli.options = {
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.read('test-project'),
|
|
||||||
targetPlatform: 'StandaloneLinux64',
|
|
||||||
cacheKey: `test-case-${uuidv4()}`,
|
|
||||||
};
|
|
||||||
GitHub.githubInputEnabled = false;
|
|
||||||
const buildParameter = await BuildParameters.create();
|
|
||||||
CloudRunner.buildParameters = buildParameter;
|
|
||||||
|
|
||||||
// Create test folder
|
|
||||||
const testFolder = path.resolve(__dirname, Cli.options.cacheKey);
|
|
||||||
fs.mkdirSync(testFolder);
|
|
||||||
|
|
||||||
// Create cache folder
|
|
||||||
const cacheFolder = path.resolve(__dirname, `cache-${Cli.options.cacheKey}`);
|
|
||||||
fs.mkdirSync(cacheFolder);
|
|
||||||
|
|
||||||
// Add test file to test folders
|
|
||||||
fs.writeFileSync(path.resolve(testFolder, 'test.txt'), Cli.options.cacheKey);
|
|
||||||
await Caching.PushToCache(cacheFolder, testFolder, `${Cli.options.cacheKey}`);
|
|
||||||
|
|
||||||
// Delete test folder
|
|
||||||
fs.rmdirSync(testFolder, { recursive: true });
|
|
||||||
await Caching.PullFromCache(
|
|
||||||
cacheFolder.replace(/\\/g, `/`),
|
|
||||||
testFolder.replace(/\\/g, `/`),
|
|
||||||
`${Cli.options.cacheKey}`,
|
|
||||||
);
|
|
||||||
await CloudRunnerSystem.Run(`du -h ${__dirname}`);
|
|
||||||
|
|
||||||
// Compare validity to original hash
|
|
||||||
expect(fs.readFileSync(path.resolve(testFolder, 'test.txt'), { encoding: 'utf8' }).toString()).toContain(
|
|
||||||
Cli.options.cacheKey,
|
|
||||||
);
|
|
||||||
fs.rmdirSync(testFolder, { recursive: true });
|
|
||||||
fs.rmdirSync(cacheFolder, { recursive: true });
|
|
||||||
|
|
||||||
GitHub.githubInputEnabled = true;
|
|
||||||
delete Cli.options;
|
|
||||||
}, 1000000);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
@@ -1,115 +0,0 @@
|
|||||||
import { BuildParameters, CloudRunner, ImageTag, Input } from '../..';
|
|
||||||
import { TaskParameterSerializer } from '../services/core/task-parameter-serializer';
|
|
||||||
import UnityVersioning from '../../unity-versioning';
|
|
||||||
import { Cli } from '../../cli/cli';
|
|
||||||
import GitHub from '../../github';
|
|
||||||
import setups from './cloud-runner-suite.test';
|
|
||||||
import { CloudRunnerStatics } from '../options/cloud-runner-statics';
|
|
||||||
import CloudRunnerOptions from '../options/cloud-runner-options';
|
|
||||||
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
|
||||||
|
|
||||||
async function CreateParameters(overrides: any) {
|
|
||||||
if (overrides) {
|
|
||||||
Cli.options = overrides;
|
|
||||||
}
|
|
||||||
const originalValue = GitHub.githubInputEnabled;
|
|
||||||
GitHub.githubInputEnabled = false;
|
|
||||||
const results = await BuildParameters.create();
|
|
||||||
GitHub.githubInputEnabled = originalValue;
|
|
||||||
delete Cli.options;
|
|
||||||
|
|
||||||
return results;
|
|
||||||
}
|
|
||||||
|
|
||||||
describe('Cloud Runner Sync Environments', () => {
|
|
||||||
setups();
|
|
||||||
const testSecretName = 'testSecretName';
|
|
||||||
const testSecretValue = 'testSecretValue';
|
|
||||||
it('Responds', () => {});
|
|
||||||
|
|
||||||
if (CloudRunnerOptions.cloudRunnerDebug) {
|
|
||||||
it('All build parameters sent to cloud runner as env vars', async () => {
|
|
||||||
// Setup parameters
|
|
||||||
const buildParameter = await CreateParameters({
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.read('test-project'),
|
|
||||||
targetPlatform: 'StandaloneWindows64',
|
|
||||||
customJob: `
|
|
||||||
- name: 'step 1'
|
|
||||||
image: 'ubuntu'
|
|
||||||
commands: 'printenv'
|
|
||||||
secrets:
|
|
||||||
- name: '${testSecretName}'
|
|
||||||
value: '${testSecretValue}'
|
|
||||||
`,
|
|
||||||
});
|
|
||||||
const baseImage = new ImageTag(buildParameter);
|
|
||||||
if (baseImage.toString().includes('undefined')) {
|
|
||||||
throw new Error(`Base image is undefined`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run the job
|
|
||||||
const file = (await CloudRunner.run(buildParameter, baseImage.toString())).BuildResults;
|
|
||||||
|
|
||||||
// Assert results
|
|
||||||
// expect(file).toContain(JSON.stringify(buildParameter));
|
|
||||||
expect(file).toContain(`${Input.ToEnvVarFormat(testSecretName)}=${testSecretValue}`);
|
|
||||||
const environmentVariables = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
|
|
||||||
const secrets = TaskParameterSerializer.readDefaultSecrets().map((x) => {
|
|
||||||
return {
|
|
||||||
name: x.EnvironmentVariable,
|
|
||||||
value: x.ParameterValue,
|
|
||||||
};
|
|
||||||
});
|
|
||||||
const combined = [...environmentVariables, ...secrets]
|
|
||||||
.filter((element) => element.value !== undefined && element.value !== '' && typeof element.value !== 'function')
|
|
||||||
.map((x) => {
|
|
||||||
if (typeof x.value === `string`) {
|
|
||||||
x.value = x.value.replace(/\s+/g, '');
|
|
||||||
}
|
|
||||||
|
|
||||||
return x;
|
|
||||||
})
|
|
||||||
.filter((element) => {
|
|
||||||
return !['UNITY_LICENSE', 'UNITY_LICENSE', 'CUSTOM_JOB', 'CUSTOM_JOB'].includes(element.name);
|
|
||||||
});
|
|
||||||
const newLinePurgedFile = file
|
|
||||||
.replace(/\s+/g, '')
|
|
||||||
.replace(new RegExp(`\\[${CloudRunnerStatics.logPrefix}\\]`, 'g'), '');
|
|
||||||
for (const element of combined) {
|
|
||||||
expect(newLinePurgedFile).toContain(`${element.name}`);
|
|
||||||
CloudRunnerLogger.log(`Contains ${element.name}`);
|
|
||||||
const fullNameEqualValue = `${element.name}=${element.value}`;
|
|
||||||
expect(newLinePurgedFile).toContain(fullNameEqualValue);
|
|
||||||
}
|
|
||||||
}, 1_000_000_000);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('Cloud Runner Environment Serializer', () => {
|
|
||||||
setups();
|
|
||||||
const testSecretName = 'testSecretName';
|
|
||||||
const testSecretValue = 'testSecretValue';
|
|
||||||
it('Cloud Runner Parameter Serialization', async () => {
|
|
||||||
// Setup parameters
|
|
||||||
const buildParameter = await CreateParameters({
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.read('test-project'),
|
|
||||||
customJob: `
|
|
||||||
- name: 'step 1'
|
|
||||||
image: 'alpine'
|
|
||||||
commands: 'printenv'
|
|
||||||
secrets:
|
|
||||||
- name: '${testSecretName}'
|
|
||||||
value: '${testSecretValue}'
|
|
||||||
`,
|
|
||||||
});
|
|
||||||
|
|
||||||
const result = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
|
|
||||||
expect(result.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
|
|
||||||
const result2 = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
|
|
||||||
expect(result2.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,65 +0,0 @@
|
|||||||
import CloudRunner from '../cloud-runner';
|
|
||||||
import UnityVersioning from '../../unity-versioning';
|
|
||||||
import setups from './cloud-runner-suite.test';
|
|
||||||
import GitHub from '../../github';
|
|
||||||
import { TIMEOUT_INFINITE, createParameters } from '../../../test-utils/cloud-runner-test-helpers';
|
|
||||||
describe('Cloud Runner Github Checks', () => {
|
|
||||||
setups();
|
|
||||||
it('Responds', () => {});
|
|
||||||
|
|
||||||
beforeEach(() => {
|
|
||||||
// Mock GitHub API requests to avoid real network calls
|
|
||||||
jest.spyOn(GitHub as any, 'createGitHubCheckRequest').mockResolvedValue({
|
|
||||||
status: 201,
|
|
||||||
data: { id: '1' },
|
|
||||||
});
|
|
||||||
jest.spyOn(GitHub as any, 'updateGitHubCheckRequest').mockResolvedValue({
|
|
||||||
status: 200,
|
|
||||||
data: {},
|
|
||||||
});
|
|
||||||
jest.spyOn(GitHub as any, 'runUpdateAsyncChecksWorkflow').mockResolvedValue(undefined);
|
|
||||||
});
|
|
||||||
|
|
||||||
afterEach(() => {
|
|
||||||
jest.restoreAllMocks();
|
|
||||||
});
|
|
||||||
|
|
||||||
it(
|
|
||||||
'Check Handling Direct',
|
|
||||||
async () => {
|
|
||||||
// Setup parameters
|
|
||||||
const buildParameter = await createParameters({
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.read('test-project'),
|
|
||||||
asyncCloudRunner: `true`,
|
|
||||||
githubChecks: `true`,
|
|
||||||
});
|
|
||||||
await CloudRunner.setup(buildParameter);
|
|
||||||
CloudRunner.buildParameters.githubCheckId = await GitHub.createGitHubCheck(`direct create`);
|
|
||||||
await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `direct`);
|
|
||||||
await GitHub.updateGitHubCheck(`2 ${new Date().toISOString()}`, `direct`, `success`, `completed`);
|
|
||||||
},
|
|
||||||
TIMEOUT_INFINITE,
|
|
||||||
);
|
|
||||||
it(
|
|
||||||
'Check Handling Via Async Workflow',
|
|
||||||
async () => {
|
|
||||||
// Setup parameters
|
|
||||||
const buildParameter = await createParameters({
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.read('test-project'),
|
|
||||||
asyncCloudRunner: `true`,
|
|
||||||
githubChecks: `true`,
|
|
||||||
});
|
|
||||||
GitHub.forceAsyncTest = true;
|
|
||||||
await CloudRunner.setup(buildParameter);
|
|
||||||
CloudRunner.buildParameters.githubCheckId = await GitHub.createGitHubCheck(`async create`);
|
|
||||||
await GitHub.updateGitHubCheck(`1 ${new Date().toISOString()}`, `async`);
|
|
||||||
await GitHub.updateGitHubCheck(`2 ${new Date().toISOString()}`, `async`, `success`, `completed`);
|
|
||||||
GitHub.forceAsyncTest = false;
|
|
||||||
},
|
|
||||||
TIMEOUT_INFINITE,
|
|
||||||
);
|
|
||||||
});
|
|
||||||
@@ -1,118 +0,0 @@
|
|||||||
import CloudRunner from '../cloud-runner';
|
|
||||||
import { BuildParameters, ImageTag } from '../..';
|
|
||||||
import UnityVersioning from '../../unity-versioning';
|
|
||||||
import { Cli } from '../../cli/cli';
|
|
||||||
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
|
||||||
import { v4 as uuidv4 } from 'uuid';
|
|
||||||
import CloudRunnerOptions from '../options/cloud-runner-options';
|
|
||||||
import setups from './cloud-runner-suite.test';
|
|
||||||
import { ContainerHookService } from '../services/hooks/container-hook-service';
|
|
||||||
import { CommandHookService } from '../services/hooks/command-hook-service';
|
|
||||||
|
|
||||||
async function CreateParameters(overrides: any) {
|
|
||||||
if (overrides) {
|
|
||||||
Cli.options = overrides;
|
|
||||||
}
|
|
||||||
|
|
||||||
return await BuildParameters.create();
|
|
||||||
}
|
|
||||||
|
|
||||||
describe('Cloud Runner Custom Hooks And Steps', () => {
|
|
||||||
it('Responds', () => {});
|
|
||||||
setups();
|
|
||||||
it('Check parsing and reading of steps', async () => {
|
|
||||||
const yamlString = `hook: before
|
|
||||||
commands: echo "test"`;
|
|
||||||
const yamlString2 = `- hook: before
|
|
||||||
commands: echo "test"`;
|
|
||||||
const overrides = {
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
|
||||||
targetPlatform: 'StandaloneLinux64',
|
|
||||||
image: 'ubuntu',
|
|
||||||
cacheKey: `test-case-${uuidv4()}`,
|
|
||||||
};
|
|
||||||
CloudRunner.setup(await CreateParameters(overrides));
|
|
||||||
const stringObject = ContainerHookService.ParseContainerHooks(yamlString);
|
|
||||||
const stringObject2 = ContainerHookService.ParseContainerHooks(yamlString2);
|
|
||||||
|
|
||||||
CloudRunnerLogger.log(yamlString);
|
|
||||||
CloudRunnerLogger.log(JSON.stringify(stringObject, undefined, 4));
|
|
||||||
|
|
||||||
expect(stringObject.length).toBe(1);
|
|
||||||
expect(stringObject[0].hook).toBe(`before`);
|
|
||||||
expect(stringObject2.length).toBe(1);
|
|
||||||
expect(stringObject2[0].hook).toBe(`before`);
|
|
||||||
|
|
||||||
const getCustomStepsFromFiles = ContainerHookService.GetContainerHooksFromFiles(`before`);
|
|
||||||
CloudRunnerLogger.log(JSON.stringify(getCustomStepsFromFiles, undefined, 4));
|
|
||||||
});
|
|
||||||
if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.providerStrategy !== `k8s`) {
|
|
||||||
it('Should be 1 before and 1 after hook', async () => {
|
|
||||||
const overrides = {
|
|
||||||
versioning: 'None',
|
|
||||||
image: 'ubuntu',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
|
||||||
targetPlatform: 'StandaloneLinux64',
|
|
||||||
cacheKey: `test-case-${uuidv4()}`,
|
|
||||||
containerHookFiles: `my-test-step-pre-build,my-test-step-post-build`,
|
|
||||||
commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
|
|
||||||
};
|
|
||||||
const buildParameter2 = await CreateParameters(overrides);
|
|
||||||
await CloudRunner.setup(buildParameter2);
|
|
||||||
const beforeHooks = CommandHookService.GetCustomHooksFromFiles(`before`);
|
|
||||||
const afterHooks = CommandHookService.GetCustomHooksFromFiles(`after`);
|
|
||||||
expect(beforeHooks).toHaveLength(1);
|
|
||||||
expect(afterHooks).toHaveLength(1);
|
|
||||||
});
|
|
||||||
it('Should be 1 before and 1 after step', async () => {
|
|
||||||
const overrides = {
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
|
||||||
targetPlatform: 'StandaloneLinux64',
|
|
||||||
cacheKey: `test-case-${uuidv4()}`,
|
|
||||||
image: 'ubuntu',
|
|
||||||
containerHookFiles: `my-test-step-pre-build,my-test-step-post-build`,
|
|
||||||
commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
|
|
||||||
};
|
|
||||||
const buildParameter2 = await CreateParameters(overrides);
|
|
||||||
await CloudRunner.setup(buildParameter2);
|
|
||||||
const beforeSteps = ContainerHookService.GetContainerHooksFromFiles(`before`);
|
|
||||||
const afterSteps = ContainerHookService.GetContainerHooksFromFiles(`after`);
|
|
||||||
expect(beforeSteps).toHaveLength(1);
|
|
||||||
expect(afterSteps).toHaveLength(1);
|
|
||||||
});
|
|
||||||
it('Run build once - check for pre and post custom hooks run contents', async () => {
|
|
||||||
const overrides = {
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
|
||||||
targetPlatform: 'StandaloneLinux64',
|
|
||||||
cacheKey: `test-case-${uuidv4()}`,
|
|
||||||
containerHookFiles: `my-test-step-pre-build,my-test-step-post-build`,
|
|
||||||
commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
|
|
||||||
};
|
|
||||||
const buildParameter2 = await CreateParameters(overrides);
|
|
||||||
const baseImage2 = new ImageTag(buildParameter2);
|
|
||||||
const results2Object = await CloudRunner.run(buildParameter2, baseImage2.toString());
|
|
||||||
const results2 = results2Object.BuildResults;
|
|
||||||
CloudRunnerLogger.log(`run 2 succeeded`);
|
|
||||||
|
|
||||||
const buildContainsBuildSucceeded = results2.includes('Build succeeded');
|
|
||||||
const buildContainsPreBuildHookRunMessage = results2.includes('before-build hook test!');
|
|
||||||
const buildContainsPostBuildHookRunMessage = results2.includes('after-build hook test!');
|
|
||||||
|
|
||||||
const buildContainsPreBuildStepMessage = results2.includes('before-build step test!');
|
|
||||||
const buildContainsPostBuildStepMessage = results2.includes('after-build step test!');
|
|
||||||
|
|
||||||
expect(buildContainsBuildSucceeded).toBeTruthy();
|
|
||||||
expect(buildContainsPreBuildHookRunMessage).toBeTruthy();
|
|
||||||
expect(buildContainsPostBuildHookRunMessage).toBeTruthy();
|
|
||||||
expect(buildContainsPreBuildStepMessage).toBeTruthy();
|
|
||||||
expect(buildContainsPostBuildStepMessage).toBeTruthy();
|
|
||||||
}, 1_000_000_000);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
import { BuildParameters, ImageTag } from '../..';
|
|
||||||
import UnityVersioning from '../../unity-versioning';
|
|
||||||
import { Cli } from '../../cli/cli';
|
|
||||||
import GitHub from '../../github';
|
|
||||||
import setups from './cloud-runner-suite.test';
|
|
||||||
|
|
||||||
async function CreateParameters(overrides: any) {
|
|
||||||
if (overrides) {
|
|
||||||
Cli.options = overrides;
|
|
||||||
}
|
|
||||||
const originalValue = GitHub.githubInputEnabled;
|
|
||||||
GitHub.githubInputEnabled = false;
|
|
||||||
const results = await BuildParameters.create();
|
|
||||||
GitHub.githubInputEnabled = originalValue;
|
|
||||||
delete Cli.options;
|
|
||||||
|
|
||||||
return results;
|
|
||||||
}
|
|
||||||
|
|
||||||
describe('Cloud Runner Image', () => {
|
|
||||||
setups();
|
|
||||||
const testSecretName = 'testSecretName';
|
|
||||||
const testSecretValue = 'testSecretValue';
|
|
||||||
it('Can create valid image from normal config', async () => {
|
|
||||||
// Setup parameters
|
|
||||||
const buildParameter = await CreateParameters({
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.read('test-project'),
|
|
||||||
targetPlatform: 'StandaloneWindows64',
|
|
||||||
customJob: `
|
|
||||||
- name: 'step 1'
|
|
||||||
image: 'ubuntu'
|
|
||||||
commands: 'printenv'
|
|
||||||
secrets:
|
|
||||||
- name: '${testSecretName}'
|
|
||||||
value: '${testSecretValue}'
|
|
||||||
`,
|
|
||||||
});
|
|
||||||
const baseImage = new ImageTag(buildParameter);
|
|
||||||
if (buildParameter.targetPlatform === undefined) {
|
|
||||||
throw new Error(`target platform includes undefined`);
|
|
||||||
}
|
|
||||||
if (baseImage.toString().includes('undefined')) {
|
|
||||||
throw new Error(`Base image ${baseImage.toString()} includes undefined`);
|
|
||||||
}
|
|
||||||
if (baseImage.toString().includes('NaN')) {
|
|
||||||
throw new Error(`Base image ${baseImage.toString()} includes nan`);
|
|
||||||
}
|
|
||||||
}, 1_000_000_000);
|
|
||||||
});
|
|
||||||
@@ -1,53 +0,0 @@
|
|||||||
import { ImageTag } from '../..';
|
|
||||||
import CloudRunner from '../cloud-runner';
|
|
||||||
import UnityVersioning from '../../unity-versioning';
|
|
||||||
import CloudRunnerOptions from '../options/cloud-runner-options';
|
|
||||||
import setups from './cloud-runner-suite.test';
|
|
||||||
import fs from 'node:fs';
|
|
||||||
import { CreateParameters } from './create-test-parameter';
|
|
||||||
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
|
||||||
|
|
||||||
describe('Cloud Runner Local Docker Workflows', () => {
|
|
||||||
setups();
|
|
||||||
it('Responds', () => {});
|
|
||||||
|
|
||||||
if (CloudRunnerOptions.providerStrategy === `local-docker`) {
|
|
||||||
it('inspect stateful folder of workflows', async () => {
|
|
||||||
const testValue = `the state in a job exits in the expected local-docker folder`;
|
|
||||||
|
|
||||||
// Setup parameters
|
|
||||||
const buildParameter = await CreateParameters({
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.read('test-project'),
|
|
||||||
customJob: `
|
|
||||||
- name: 'step 1'
|
|
||||||
image: 'ubuntu'
|
|
||||||
commands: 'echo "${testValue}" >> /data/test-out-state.txt'
|
|
||||||
`,
|
|
||||||
});
|
|
||||||
const buildParameter2 = await CreateParameters({
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.read('test-project'),
|
|
||||||
customJob: `
|
|
||||||
- name: 'step 1'
|
|
||||||
image: 'ubuntu'
|
|
||||||
commands: 'cat /data/test-out-state.txt >> /data/test-out-state-2.txt'
|
|
||||||
`,
|
|
||||||
});
|
|
||||||
const baseImage = new ImageTag(buildParameter);
|
|
||||||
|
|
||||||
// Run the job
|
|
||||||
await CloudRunner.run(buildParameter, baseImage.toString());
|
|
||||||
await CloudRunner.run(buildParameter2, baseImage.toString());
|
|
||||||
|
|
||||||
const outputFile = fs.readFileSync(`./cloud-runner-cache/test-out-state.txt`, `utf-8`);
|
|
||||||
expect(outputFile).toMatch(testValue);
|
|
||||||
|
|
||||||
const outputFile2 = fs.readFileSync(`./cloud-runner-cache/test-out-state-2.txt`, `utf-8`);
|
|
||||||
expect(outputFile2).toMatch(testValue);
|
|
||||||
CloudRunnerLogger.log(outputFile);
|
|
||||||
}, 1_000_000_000);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
@@ -1,115 +0,0 @@
|
|||||||
import SharedWorkspaceLocking from '../services/core/shared-workspace-locking';
|
|
||||||
import { Cli } from '../../cli/cli';
|
|
||||||
import setups from './cloud-runner-suite.test';
|
|
||||||
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
|
||||||
import { v4 as uuidv4 } from 'uuid';
|
|
||||||
import CloudRunnerOptions from '../options/cloud-runner-options';
|
|
||||||
import UnityVersioning from '../../unity-versioning';
|
|
||||||
import BuildParameters from '../../build-parameters';
|
|
||||||
import CloudRunner from '../cloud-runner';
|
|
||||||
|
|
||||||
async function CreateParameters(overrides: any) {
|
|
||||||
if (overrides) {
|
|
||||||
Cli.options = overrides;
|
|
||||||
}
|
|
||||||
|
|
||||||
return await BuildParameters.create();
|
|
||||||
}
|
|
||||||
|
|
||||||
describe('Cloud Runner Locking Core', () => {
|
|
||||||
setups();
|
|
||||||
it('Responds', () => {});
|
|
||||||
if (CloudRunnerOptions.cloudRunnerDebug) {
|
|
||||||
it(`Create Workspace`, async () => {
|
|
||||||
const overrides: any = {
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
|
||||||
targetPlatform: 'StandaloneLinux64',
|
|
||||||
cacheKey: `test-case-${uuidv4()}`,
|
|
||||||
maxRetainedWorkspaces: 3,
|
|
||||||
};
|
|
||||||
const buildParameters = await CreateParameters(overrides);
|
|
||||||
CloudRunner.buildParameters = buildParameters;
|
|
||||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
|
||||||
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
|
||||||
}, 150000);
|
|
||||||
it(`Create Workspace And Lock Workspace`, async () => {
|
|
||||||
const overrides: any = {
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
|
||||||
targetPlatform: 'StandaloneLinux64',
|
|
||||||
cacheKey: `test-case-${uuidv4()}`,
|
|
||||||
maxRetainedWorkspaces: 3,
|
|
||||||
};
|
|
||||||
const runId = uuidv4();
|
|
||||||
const buildParameters = await CreateParameters(overrides);
|
|
||||||
CloudRunner.buildParameters = buildParameters;
|
|
||||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
|
||||||
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
|
||||||
}, 150000);
|
|
||||||
it(`0 free workspaces after locking`, async () => {
|
|
||||||
const overrides: any = {
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
|
||||||
targetPlatform: 'StandaloneLinux64',
|
|
||||||
cacheKey: `test-case-${uuidv4()}`,
|
|
||||||
maxRetainedWorkspaces: 3,
|
|
||||||
};
|
|
||||||
const buildParameters = await CreateParameters(overrides);
|
|
||||||
|
|
||||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
|
||||||
const runId = uuidv4();
|
|
||||||
CloudRunner.buildParameters = buildParameters;
|
|
||||||
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.HasWorkspaceLock(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.GetAllWorkspaces(buildParameters)).toHaveLength(1);
|
|
||||||
expect(await SharedWorkspaceLocking.GetAllLocksForWorkspace(newWorkspaceName, buildParameters)).toHaveLength(1);
|
|
||||||
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
|
|
||||||
|
|
||||||
const files = await SharedWorkspaceLocking.ReadLines(
|
|
||||||
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParameters.cacheKey}/`,
|
|
||||||
);
|
|
||||||
|
|
||||||
const lockFilesExist =
|
|
||||||
files.filter((x) => {
|
|
||||||
return x.includes(newWorkspaceName) && x.endsWith(`_lock`);
|
|
||||||
}).length > 0;
|
|
||||||
|
|
||||||
expect(files).toHaveLength(2);
|
|
||||||
expect(
|
|
||||||
files.filter((x) => {
|
|
||||||
return x.includes(newWorkspaceName) && x.endsWith(`_lock`);
|
|
||||||
}),
|
|
||||||
).toHaveLength(1);
|
|
||||||
expect(lockFilesExist).toBeTruthy();
|
|
||||||
const result: string[] = [];
|
|
||||||
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParameters);
|
|
||||||
for (const element of workspaces) {
|
|
||||||
expect((await SharedWorkspaceLocking.GetAllWorkspaces(buildParameters)).join()).toContain(element);
|
|
||||||
expect(await SharedWorkspaceLocking.GetAllWorkspaces(buildParameters)).toHaveLength(1);
|
|
||||||
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(element, buildParameters)).toBeTruthy();
|
|
||||||
await new Promise((promise) => setTimeout(promise, 1500));
|
|
||||||
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParameters);
|
|
||||||
const isBelowMax = await SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParameters);
|
|
||||||
CloudRunnerLogger.log(`workspace ${element} locked:${isLocked} below max:${isBelowMax}`);
|
|
||||||
const lock = files.find((x) => {
|
|
||||||
return x.endsWith(`_lock`);
|
|
||||||
});
|
|
||||||
expect(lock).toContain(element);
|
|
||||||
expect(isLocked).toBeTruthy();
|
|
||||||
expect(isBelowMax).toBeTruthy();
|
|
||||||
if (!isLocked && isBelowMax) {
|
|
||||||
result.push(element);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
expect(result).toHaveLength(0);
|
|
||||||
expect(await SharedWorkspaceLocking.GetFreeWorkspaces(buildParameters)).toHaveLength(0);
|
|
||||||
}, 300000);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
@@ -1,156 +0,0 @@
|
|||||||
import SharedWorkspaceLocking from '../services/core/shared-workspace-locking';
|
|
||||||
import { Cli } from '../../cli/cli';
|
|
||||||
import setups from './cloud-runner-suite.test';
|
|
||||||
import { v4 as uuidv4 } from 'uuid';
|
|
||||||
import CloudRunnerOptions from '../options/cloud-runner-options';
|
|
||||||
import UnityVersioning from '../../unity-versioning';
|
|
||||||
import BuildParameters from '../../build-parameters';
|
|
||||||
import CloudRunner from '../cloud-runner';
|
|
||||||
|
|
||||||
async function CreateParameters(overrides: any) {
|
|
||||||
if (overrides) {
|
|
||||||
Cli.options = overrides;
|
|
||||||
}
|
|
||||||
|
|
||||||
return await BuildParameters.create();
|
|
||||||
}
|
|
||||||
|
|
||||||
describe('Cloud Runner Locking Get Locked Workspace', () => {
|
|
||||||
setups();
|
|
||||||
it('Responds', () => {});
|
|
||||||
if (CloudRunnerOptions.cloudRunnerDebug) {
|
|
||||||
it(`Get locked workspace From No Workspace`, async () => {
|
|
||||||
const overrides: any = {
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
|
||||||
targetPlatform: 'StandaloneLinux64',
|
|
||||||
cacheKey: `test-case-${uuidv4()}`,
|
|
||||||
maxRetainedWorkspaces: 3,
|
|
||||||
};
|
|
||||||
const buildParameters = await CreateParameters(overrides);
|
|
||||||
|
|
||||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
|
||||||
const runId = uuidv4();
|
|
||||||
CloudRunner.buildParameters = buildParameters;
|
|
||||||
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
|
||||||
}, 150000);
|
|
||||||
it(`Get locked workspace from unlocked`, async () => {
|
|
||||||
const overrides: any = {
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
|
||||||
targetPlatform: 'StandaloneLinux64',
|
|
||||||
cacheKey: `test-case-${uuidv4()}`,
|
|
||||||
maxRetainedWorkspaces: 3,
|
|
||||||
};
|
|
||||||
const buildParameters = await CreateParameters(overrides);
|
|
||||||
|
|
||||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
|
||||||
const runId = uuidv4();
|
|
||||||
CloudRunner.buildParameters = buildParameters;
|
|
||||||
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
|
||||||
expect(CloudRunner.lockedWorkspace).toMatch(newWorkspaceName);
|
|
||||||
}, 300000);
|
|
||||||
it(`Get locked workspace from locked`, async () => {
|
|
||||||
const overrides: any = {
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
|
||||||
targetPlatform: 'StandaloneLinux64',
|
|
||||||
cacheKey: `test-case-${uuidv4()}`,
|
|
||||||
maxRetainedWorkspaces: 3,
|
|
||||||
};
|
|
||||||
const buildParameters = await CreateParameters(overrides);
|
|
||||||
|
|
||||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
|
||||||
const runId = uuidv4();
|
|
||||||
const runId2 = uuidv4();
|
|
||||||
CloudRunner.buildParameters = buildParameters;
|
|
||||||
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.HasWorkspaceLock(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.IsWorkspaceBelowMax(newWorkspaceName, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId2, buildParameters)).toBeTruthy();
|
|
||||||
expect(CloudRunner.lockedWorkspace).not.toMatch(newWorkspaceName);
|
|
||||||
}, 300000);
|
|
||||||
it(`Get locked workspace after double lock and one unlock`, async () => {
|
|
||||||
const overrides: any = {
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
|
||||||
targetPlatform: 'StandaloneLinux64',
|
|
||||||
cacheKey: `test-case-${uuidv4()}`,
|
|
||||||
maxRetainedWorkspaces: 3,
|
|
||||||
};
|
|
||||||
const buildParameters = await CreateParameters(overrides);
|
|
||||||
|
|
||||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
|
||||||
const runId = uuidv4();
|
|
||||||
const runId2 = uuidv4();
|
|
||||||
CloudRunner.buildParameters = buildParameters;
|
|
||||||
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeFalsy();
|
|
||||||
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.HasWorkspaceLock(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId2, buildParameters)).toBeTruthy();
|
|
||||||
expect(CloudRunner.lockedWorkspace).not.toContain(newWorkspaceName);
|
|
||||||
}, 300000);
|
|
||||||
it(`Get locked workspace after double lock and unlock`, async () => {
|
|
||||||
const overrides: any = {
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
|
||||||
targetPlatform: 'StandaloneLinux64',
|
|
||||||
cacheKey: `test-case-${uuidv4()}`,
|
|
||||||
maxRetainedWorkspaces: 3,
|
|
||||||
};
|
|
||||||
const buildParameters = await CreateParameters(overrides);
|
|
||||||
|
|
||||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
|
||||||
const runId = uuidv4();
|
|
||||||
const runId2 = uuidv4();
|
|
||||||
CloudRunner.buildParameters = buildParameters;
|
|
||||||
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeFalsy();
|
|
||||||
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.HasWorkspaceLock(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeFalsy();
|
|
||||||
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId2, buildParameters)).toBeTruthy();
|
|
||||||
expect(CloudRunner.lockedWorkspace).toContain(newWorkspaceName);
|
|
||||||
}, 300000);
|
|
||||||
it(`Get locked workspace from unlocked was locked`, async () => {
|
|
||||||
const overrides: any = {
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
|
||||||
targetPlatform: 'StandaloneLinux64',
|
|
||||||
cacheKey: `test-case-${uuidv4()}`,
|
|
||||||
maxRetainedWorkspaces: 3,
|
|
||||||
};
|
|
||||||
const buildParameters = await CreateParameters(overrides);
|
|
||||||
|
|
||||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
|
||||||
const runId = uuidv4();
|
|
||||||
CloudRunner.buildParameters = buildParameters;
|
|
||||||
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
|
||||||
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
|
|
||||||
expect(CloudRunner.lockedWorkspace).toMatch(newWorkspaceName);
|
|
||||||
}, 300000);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
@@ -1,48 +0,0 @@
|
|||||||
import CloudRunner from '../cloud-runner';
|
|
||||||
import { BuildParameters, ImageTag } from '../..';
|
|
||||||
import UnityVersioning from '../../unity-versioning';
|
|
||||||
import { Cli } from '../../cli/cli';
|
|
||||||
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
|
|
||||||
import { v4 as uuidv4 } from 'uuid';
|
|
||||||
import CloudRunnerOptions from '../options/cloud-runner-options';
|
|
||||||
import setups from './cloud-runner-suite.test';
|
|
||||||
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
|
|
||||||
import { OptionValues } from 'commander';
|
|
||||||
|
|
||||||
async function CreateParameters(overrides: OptionValues | undefined) {
|
|
||||||
if (overrides) {
|
|
||||||
Cli.options = overrides;
|
|
||||||
}
|
|
||||||
|
|
||||||
return await BuildParameters.create();
|
|
||||||
}
|
|
||||||
|
|
||||||
describe('Cloud Runner pre-built S3 steps', () => {
|
|
||||||
it('Responds', () => {});
|
|
||||||
setups();
|
|
||||||
if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.providerStrategy !== `local-docker`) {
|
|
||||||
it('Run build and prebuilt s3 cache pull, cache push and upload build', async () => {
|
|
||||||
const overrides = {
|
|
||||||
versioning: 'None',
|
|
||||||
projectPath: 'test-project',
|
|
||||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
|
||||||
targetPlatform: 'StandaloneLinux64',
|
|
||||||
cacheKey: `test-case-${uuidv4()}`,
|
|
||||||
containerHookFiles: `aws-s3-pull-cache,aws-s3-upload-cache,aws-s3-upload-build`,
|
|
||||||
};
|
|
||||||
const buildParameter2 = await CreateParameters(overrides);
|
|
||||||
const baseImage2 = new ImageTag(buildParameter2);
|
|
||||||
const results2Object = await CloudRunner.run(buildParameter2, baseImage2.toString());
|
|
||||||
const results2 = results2Object.BuildResults;
|
|
||||||
CloudRunnerLogger.log(`run 2 succeeded`);
|
|
||||||
|
|
||||||
const build2ContainsBuildSucceeded = results2.includes('Build succeeded');
|
|
||||||
expect(build2ContainsBuildSucceeded).toBeTruthy();
|
|
||||||
|
|
||||||
const results = await CloudRunnerSystem.RunAndReadLines(
|
|
||||||
`aws s3 ls s3://${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/`,
|
|
||||||
);
|
|
||||||
CloudRunnerLogger.log(results.join(`,`));
|
|
||||||
}, 1_000_000_000);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
@@ -1,25 +0,0 @@
|
|||||||
import { Cli } from '../../cli/cli';
|
|
||||||
import GitHub from '../../github';
|
|
||||||
|
|
||||||
describe('Cloud Runner', () => {
|
|
||||||
it('Responds', () => {});
|
|
||||||
});
|
|
||||||
|
|
||||||
const setups = () => {
|
|
||||||
beforeAll(() => {
|
|
||||||
GitHub.githubInputEnabled = false;
|
|
||||||
});
|
|
||||||
beforeEach(() => {
|
|
||||||
Cli.options = {};
|
|
||||||
});
|
|
||||||
afterEach(() => {
|
|
||||||
if (Cli.options !== undefined) {
|
|
||||||
delete Cli.options;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
afterAll(() => {
|
|
||||||
GitHub.githubInputEnabled = true;
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
export default setups;
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user