mirror of
https://github.com/game-ci/unity-builder.git
synced 2026-06-11 16:34:03 -07:00
Cloud runner develop - better parameterization of s3 usage, improved async workflow and GC, github checks early integration (#479)
* custom steps may leave value undefined, will be pulled from env vars * custom steps may leave value undefined, will be pulled from env vars * custom steps may leave value undefined, will be pulled from env vars * add 3 new premade steps, steam-deploy-client, steam-deploy-project, aws-s3-pull-build * fix * fix * fix * continue building async-workflow support * test checks * test checks * test checks * move github checks within build workflow * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test * async workflow test for aws only * async workflow test for aws only * async workflow test for aws only * async workflow test for aws only * cleanup logging * disable lz4 compression by default * disable lz4 compression by default * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * AWS BASE STACK for tests * disable lz4 compression by default * disable lz4 compression by default * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * Update github check with aws log * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * kinesis and subscription filter for logs creation skipped when watchToEnd false * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * cleanup local pipeline, log aws formation * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * async pipeline * workflow * workflow * workflow * workflow * workflow * workflow * workflow * workflow * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3 * parameterize s3
This commit is contained in:
Frostebite
@@ -1,5 +1,4 @@
|
||||
import YAML from 'yaml';
|
||||
import CloudRunnerSecret from './cloud-runner-secret';
|
||||
import CloudRunner from '../cloud-runner';
|
||||
import * as core from '@actions/core';
|
||||
import { CustomWorkflow } from '../workflows/custom-workflow';
|
||||
@@ -9,6 +8,7 @@ import * as fs from 'fs';
|
||||
import Input from '../../input';
|
||||
import CloudRunnerOptions from '../cloud-runner-options';
|
||||
import CloudRunnerLogger from './cloud-runner-logger';
|
||||
import { CustomStep } from './custom-step';
|
||||
|
||||
export class CloudRunnerCustomSteps {
|
||||
static GetCustomStepsFromFiles(hookLifecycle: string): CustomStep[] {
|
||||
@@ -43,8 +43,14 @@ export class CloudRunnerCustomSteps {
|
||||
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
|
||||
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
|
||||
aws configure set region $AWS_DEFAULT_REGION --profile default
|
||||
aws s3 cp /data/cache/$CACHE_KEY/build/build-$BUILD_GUID.tar.lz4 s3://game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar.lz4
|
||||
rm /data/cache/$CACHE_KEY/build/build-$BUILD_GUID.tar.lz4
|
||||
aws s3 cp /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
|
||||
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
|
||||
} s3://${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${
|
||||
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
|
||||
}
|
||||
rm /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
|
||||
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
|
||||
}
|
||||
secrets:
|
||||
- name: awsAccessKeyId
|
||||
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
|
||||
@@ -52,6 +58,62 @@ export class CloudRunnerCustomSteps {
|
||||
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
|
||||
- name: awsDefaultRegion
|
||||
value: ${process.env.AWS_REGION || ``}
|
||||
- name: aws-s3-pull-build
|
||||
image: amazon/aws-cli
|
||||
commands: |
|
||||
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
|
||||
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
|
||||
aws configure set region $AWS_DEFAULT_REGION --profile default
|
||||
aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/ || true
|
||||
aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/build || true
|
||||
aws s3 cp s3://${
|
||||
CloudRunner.buildParameters.awsBaseStackName
|
||||
}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
||||
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
|
||||
} /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
|
||||
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
|
||||
}
|
||||
secrets:
|
||||
- name: awsAccessKeyId
|
||||
- name: awsSecretAccessKey
|
||||
- name: awsDefaultRegion
|
||||
- name: BUILD_GUID_TARGET
|
||||
- name: steam-deploy-client
|
||||
image: steamcmd/steamcmd
|
||||
commands: |
|
||||
apt-get update
|
||||
apt-get install -y curl tar coreutils git tree > /dev/null
|
||||
curl -s https://gist.githubusercontent.com/frostebite/1d56f5505b36b403b64193b7a6e54cdc/raw/fa6639ed4ef750c4268ea319d63aa80f52712ffb/deploy-client-steam.sh | bash
|
||||
secrets:
|
||||
- name: STEAM_USERNAME
|
||||
- name: STEAM_PASSWORD
|
||||
- name: STEAM_APPID
|
||||
- name: STEAM_SSFN_FILE_NAME
|
||||
- name: STEAM_SSFN_FILE_CONTENTS
|
||||
- name: STEAM_CONFIG_VDF_1
|
||||
- name: STEAM_CONFIG_VDF_2
|
||||
- name: STEAM_CONFIG_VDF_3
|
||||
- name: STEAM_CONFIG_VDF_4
|
||||
- name: BUILD_GUID_TARGET
|
||||
- name: RELEASE_BRANCH
|
||||
- name: steam-deploy-project
|
||||
image: steamcmd/steamcmd
|
||||
commands: |
|
||||
apt-get update
|
||||
apt-get install -y curl tar coreutils git tree > /dev/null
|
||||
curl -s https://gist.githubusercontent.com/frostebite/969da6a41002a0e901174124b643709f/raw/02403e53fb292026cba81ddcf4ff35fc1eba111d/steam-deploy-project.sh | bash
|
||||
secrets:
|
||||
- name: STEAM_USERNAME
|
||||
- name: STEAM_PASSWORD
|
||||
- name: STEAM_APPID
|
||||
- name: STEAM_SSFN_FILE_NAME
|
||||
- name: STEAM_SSFN_FILE_CONTENTS
|
||||
- name: STEAM_CONFIG_VDF_1
|
||||
- name: STEAM_CONFIG_VDF_2
|
||||
- name: STEAM_CONFIG_VDF_3
|
||||
- name: STEAM_CONFIG_VDF_4
|
||||
- name: BUILD_GUID_2
|
||||
- name: RELEASE_BRANCH
|
||||
- name: aws-s3-upload-cache
|
||||
image: amazon/aws-cli
|
||||
hook: after
|
||||
@@ -59,9 +121,13 @@ export class CloudRunnerCustomSteps {
|
||||
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
|
||||
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
|
||||
aws configure set region $AWS_DEFAULT_REGION --profile default
|
||||
aws s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/lfs
|
||||
aws s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://${
|
||||
CloudRunner.buildParameters.awsBaseStackName
|
||||
}/cloud-runner-cache/$CACHE_KEY/lfs
|
||||
rm -r /data/cache/$CACHE_KEY/lfs
|
||||
aws s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/Library
|
||||
aws s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://${
|
||||
CloudRunner.buildParameters.awsBaseStackName
|
||||
}/cloud-runner-cache/$CACHE_KEY/Library
|
||||
rm -r /data/cache/$CACHE_KEY/Library
|
||||
secrets:
|
||||
- name: awsAccessKeyId
|
||||
@@ -77,13 +143,13 @@ export class CloudRunnerCustomSteps {
|
||||
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
|
||||
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
|
||||
aws configure set region $AWS_DEFAULT_REGION --profile default
|
||||
aws s3 ls game-ci-test-storage/cloud-runner-cache/ || true
|
||||
aws s3 ls game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/ || true
|
||||
BUCKET1="game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/Library/"
|
||||
aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/ || true
|
||||
aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/ || true
|
||||
BUCKET1="${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/Library/"
|
||||
aws s3 ls $BUCKET1 || true
|
||||
OBJECT1="$(aws s3 ls $BUCKET1 | sort | tail -n 1 | awk '{print $4}' || '')"
|
||||
aws s3 cp s3://$BUCKET1$OBJECT1 /data/cache/$CACHE_KEY/Library/ || true
|
||||
BUCKET2="game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/lfs/"
|
||||
BUCKET2="${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/lfs/"
|
||||
aws s3 ls $BUCKET2 || true
|
||||
OBJECT2="$(aws s3 ls $BUCKET2 | sort | tail -n 1 | awk '{print $4}' || '')"
|
||||
aws s3 cp s3://$BUCKET2$OBJECT2 /data/cache/$CACHE_KEY/lfs/ || true
|
||||
@@ -135,21 +201,21 @@ export class CloudRunnerCustomSteps {
|
||||
if (steps === '') {
|
||||
return [];
|
||||
}
|
||||
|
||||
// if (CloudRunner.buildParameters?.cloudRunnerIntegrationTests) {
|
||||
|
||||
// CloudRunnerLogger.log(`Parsing build steps: ${steps}`);
|
||||
|
||||
// }
|
||||
const isArray = steps.replace(/\s/g, ``)[0] === `-`;
|
||||
if (CloudRunner.buildParameters?.cloudRunnerDebug) {
|
||||
CloudRunnerLogger.log(`Parsing: ${steps}`);
|
||||
}
|
||||
const object: CustomStep[] = isArray ? YAML.parse(steps) : [YAML.parse(steps)];
|
||||
for (const step of object) {
|
||||
CloudRunnerCustomSteps.ConvertYamlSecrets(step);
|
||||
if (step.secrets === undefined) {
|
||||
step.secrets = [];
|
||||
} else {
|
||||
for (const secret of step.secrets) {
|
||||
if (secret.ParameterValue === undefined && process.env[secret.EnvironmentVariable] !== undefined) {
|
||||
if (CloudRunner.buildParameters?.cloudRunnerDebug) {
|
||||
CloudRunnerLogger.log(`Injecting custom step ${step.name} from env var ${secret.ParameterKey}`);
|
||||
}
|
||||
secret.ParameterValue = process.env[secret.ParameterKey] || ``;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (step.image === undefined) {
|
||||
step.image = `ubuntu`;
|
||||
@@ -201,10 +267,3 @@ export class CloudRunnerCustomSteps {
|
||||
return output;
|
||||
}
|
||||
}
|
||||
export class CustomStep {
|
||||
public commands;
|
||||
public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
|
||||
public name;
|
||||
public image: string = `ubuntu`;
|
||||
public hook!: string;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user