I have a workflow that uses 'strategy' = 'matrix'
along with a list of specific configurations to build.
Here is my workflow file:
#
# build-N-test-v2.1-Dev and build-N-test-v2.1-Release are nearly
# identical, but a few tests are commented out (to not needlessly stress CI system)
# for v2.1-Dev builds
#
# NOTE: I've tried many tricks - none which seem to work - to get this working on one file with one
# workflow and tests
# https://github.community/t/what-is-the-correct-if-condition-syntax-for-checking-matrix-os-version/16221
# https://github.community/t/how-to-conditionally-include-exclude-items-in-matrix-eg-based-on-branch/16853
#
# but none seem to work
#
name: build-N-test-v2.1-Dev
on:
push:
branches:
- v2.1-Dev
#- v2.1-Release
workflow_dispatch:
inputs:
ignored:
description: "ignored"
required: false
default: ""
## NB: JOBS section IDENTICAL between v2.1-Dev and 2.1-Release files EXCEPT that on v2.1-Dev file
## comment out all entries marked with includeInDevBranchBuilds: false
jobs:
build-n-test-Linux:
runs-on: ${{ matrix.runs_on }}
strategy:
#
# Configuration notes
# o --debug-symbols false to reduce build disk size (and we aren't debugging anyhow) in many debug configurations
#
matrix:
include:
# ## SADLY: Container operations are only supported on Linux runners
# - displayTargetName: windows-DBG
# os: windows
# compiler: g++-8
# runs_on: windows-latest
# container_image: sophistsolutionsinc/stroika-buildvm-windows-cygwin-vs2k19
# cpp_version: c++17
# config_name: Debug
# extra_config_args: --apply-default-debug-flags --trace2file enable
## centos 8
- displayTargetName: centos-8
os: unix
compiler: g++
runs_on: ubuntu-latest
container_image: sophistsolutionsinc/stroika-buildvm-centos-8-small
cpp_version: c++17
config_name: Release
extra_config_args: --apply-default-release-flags --trace2file enable
includeInDevBranchBuilds: true
## ubuntu 18.04
- displayTargetName: ubuntu-18.04-g++-8 (Debug)
os: unix
compiler: g++-8
runs_on: ubuntu-latest
container_image: sophistsolutionsinc/stroika-buildvm-ubuntu1804-regression-tests
cpp_version: c++17
config_name: Debug
extra_config_args: --apply-default-debug-flags --trace2file enable --debug-symbols false
includeInDevBranchBuilds: true
- displayTargetName: ubuntu-18.04-cross-compile-raspberrypi (Debug)
os: unix
compiler: g++-8
runs_on: ubuntu-latest
container_image: sophistsolutionsinc/stroika-buildvm-ubuntu1804-regression-tests
cpp_version: c++17
config_name: Debug
extra_config_args: --apply-default-release-flags --trace2file enable --compiler-driver arm-linux-gnueabihf-g++-8 --cross-compiling true
includeInDevBranchBuilds: true
# ubuntu 20.04
# - displayTargetName: ubuntu-20.04-g++-9 (Debug)
# os: unix
# compiler: g++-9
# runs_on: ubuntu-latest
# container_image: sophistsolutionsinc/stroika-buildvm-ubuntu2004-regression-tests
# cpp_version: c++17
# config_name: Debug
# extra_config_args: --apply-default-debug-flags --trace2file enable --debug-symbols false
# includeInDevBranchBuilds: false
# - displayTargetName: ubuntu-20.04-g++-10 (Debug)
# os: unix
# compiler: g++-10
# runs_on: ubuntu-latest
# container_image: sophistsolutionsinc/stroika-buildvm-ubuntu2004-regression-tests
# cpp_version: c++17
# config_name: Debug
# extra_config_args: --apply-default-debug-flags --trace2file enable --debug-symbols false
# includeInDevBranchBuilds: false
- displayTargetName: ubuntu-20.04-g++-10
os: unix
compiler: g++-10
runs_on: ubuntu-latest
container_image: sophistsolutionsinc/stroika-buildvm-ubuntu2004-regression-tests
cpp_version: c++17
config_name: Release
extra_config_args: --apply-default-release-flags --trace2file enable
includeInDevBranchBuilds: true
# - displayTargetName: ubuntu-20.04-g++-10-c++2a
# os: unix
# compiler: g++-10
# runs_on: ubuntu-latest
# container_image: sophistsolutionsinc/stroika-buildvm-ubuntu2004-regression-tests
# cpp_version: c++2a
# config_name: Release
# extra_config_args: --apply-default-release-flags --trace2file enable
# includeInDevBranchBuilds: false
# - displayTargetName: ubuntu-20.04-clang++-10
# os: unix
# compiler: clang++-10
# runs_on: ubuntu-latest
# container_image: sophistsolutionsinc/stroika-buildvm-ubuntu2004-regression-tests
# cpp_version: c++17
# config_name: Release
# extra_config_args: --apply-default-release-flags --trace2file enable
# includeInDevBranchBuilds: false
### ATTEMPT TO COMPRESS 2 workflow files into one, but so far not working
### SEE
### https://stackoverflow.com/questions/65384420/how-to-make-a-github-action-matrix-element-conditional/65385385#65385385
###
#if: github.ref == 'refs/heads/v2.1-Release' || matrix.includeInDevBranchBuilds
env:
# vm has 2 virtual CPUs, but 8GB ram, so jobs=5 (empirical), and QUICK_BUILD avoids some internal testing
MAKEFLAGS: "--jobs=3 QUICK_BUILD=1"
container: ${{ matrix.container_image }}
steps:
- uses: actions/checkout@v2
- name: Build System Info
if: ${{ matrix.os=='unix' }}
run: |
lsb_release -d 2>/dev/null || true
echo "CWD=" `pwd`
echo "nproc=" `nproc`
grep "model name" /proc/cpuinfo | head -1
grep processor /proc/cpuinfo | wc -l
grep MemTotal /proc/meminfo
df -h
- name: Build System Info (Windows)
if: ${{ matrix.os=='windows' }}
run: |
echo "CWD=" `pwd`
df -h
- name: Configure ${{ matrix.config_name }}
run: |
./configure ${{ matrix.config_name }} --compiler-driver ${{ matrix.compiler }} ${{ matrix.extra_config_args }} --cppstd-version ${{ matrix.cpp_version }}
cat ConfigurationFiles/${{ matrix.config_name }}.xml
# Break out third-party-components to do clean so we dont run out of disk space, and break out TPC AND library
# to show the summary time for each part
- name: Make third-party-components
run: |
make third-party-components
make clean
- name: Make libraries
run: make libraries
- name: Make all
run: make all
- name: Run Tests
run: make run-tests
- name: Archive Samples Results
uses: actions/upload-artifact@v2
with:
name: Sample apps (${{ matrix.displayTargetName }})
path: |
Builds/${{ matrix.config_name }}/Samples-*
- name: Archive Log Results
uses: actions/upload-artifact@v2
with:
name: Log Data (${{ matrix.displayTargetName }})
path: |
Builds/${{ matrix.config_name }}/PerformanceDump.txt
/tmp/Trace*.txt
build-n-test-MacOS:
runs-on: ${{ matrix.runs_on }}
strategy:
matrix:
# Add to extra_config_args for build speed: --Xerces no --OpenSSL no --lzma no --boost no
include:
- displayTargetName: MacOS-Debug
os: macos-10.15
runs_on: macos-10.15
config_name: Debug
extra_config_args: --apply-default-debug-flags --trace2file enable
includeInDevBranchBuilds: true
# - displayTargetName: MacOS
# os: macos-10.15
# runs_on: macos-10.15
# config_name: Release
# extra_config_args: --apply-default-release-flags --trace2file enable
# includeInDevBranchBuilds: false
env:
# vm has 2 virtual CPUs, but 8GB ram, so jobs=5 (empirical), and QUICK_BUILD avoids some internal testing
MAKEFLAGS: "--jobs=3 QUICK_BUILD=1"
steps:
- uses: actions/checkout@v2
- name: Build System Info
run: |
echo "CWD: `pwd`"
df -h
system_profiler SPSoftwareDataType
sw_vers
# If we had docker ability, most of these would be built into a docker file
- name: Install Basic Build requirements
run: |
brew install gnu-sed
brew install p7zip
brew install automake
make install-realpath
- name: Configure
run: |
./configure ${{ matrix.config_name }} ${{ matrix.extra_config_args }}
cat ConfigurationFiles/${{ matrix.config_name }}.xml
- name: Build third-party-components
run: |
make third-party-components
make clean
- name: Build Library
run: |
make libraries
- name: Build All
run: |
make all
- name: Run-Tests
run: |
make run-tests
- name: Workaround GitHub-Actions-MacOS Issue with env.TMPDIR
run: |
mkdir /tmp/LOGS-ARCHIVE
cp $TMPDIR/Trace*.txt /tmp/LOGS-ARCHIVE
- name: DEBUG Workaround GitHub-Actions-MacOS Issue with env.TMPDIR
run: |
echo "TMPDIR=$TMPDIR"
echo "TMPDIR using ENV.TMPDIR=${{ env.TMPDIR }}"
# Just the echo line above shows empty, and then the ls line causes exit 1/failure
#ls -l ${{ env.TMPDIR }}/Trace*.txt
#if this gets fixed, then lose Workaround GitHub-Actions-MacOS, and directly reference ${{ env.TMPDIR }}/Trace*.txt in Archive Log Results
- name: Build System Info
run: |
df -h
- name: Archive Log Results
uses: actions/upload-artifact@v2
with:
name: Log Results (${{ matrix.displayTargetName }})
path: |
Builds/${{ matrix.config_name }}/PerformanceDump.txt
/tmp/LOGS-ARCHIVE
#${{ env.TMPDIR }}/Trace*.txt
- name: Archive Sample Results
uses: actions/upload-artifact@v2
with:
name: Samples (${{ matrix.displayTargetName }})
path: |
Builds/${{ matrix.config_name }}/Samples-*
build-n-test-Windows:
runs-on: ${{ matrix.runs_on }}
strategy:
matrix:
# Add to extra_config_args for build speed: --Xerces no --OpenSSL no --lzma no --boost no
include:
- displayTargetName: windows-x86-Debug
os: windows
runs_on: windows-latest
container_image: sophistsolutionsinc/stroika-buildvm-windows-cygwin-vs2k19
config_name: Debug
extra_config_args: --arch x86 --apply-default-debug-flags --trace2file enable
includeInDevBranchBuilds: true
# - displayTargetName: windows-x86-Release
# os: windows
# runs_on: windows-latest
# container_image: sophistsolutionsinc/stroika-buildvm-windows-cygwin-vs2k19
# config_name: Release
# extra_config_args: --arch x86 --apply-default-release-flags --trace2file enable
# includeInDevBranchBuilds: false
# - displayTargetName: windows-x86_64-Debug
# os: windows
# runs_on: windows-latest
# container_image: sophistsolutionsinc/stroika-buildvm-windows-cygwin-vs2k19
# config_name: Debug
# extra_config_args: --arch x86_64 --apply-default-debug-flags --trace2file enable
# includeInDevBranchBuilds: false
# - displayTargetName: windows-x86_64-Release
# os: windows
# runs_on: windows-latest
# container_image: sophistsolutionsinc/stroika-buildvm-windows-cygwin-vs2k19
# config_name: Release
# extra_config_args: --arch x86 --apply-default-release-flags --trace2file enable
# includeInDevBranchBuilds: false
env:
# vm has 2 virtual CPUs, but 8GB ram, so jobs=5 (empirical), and QUICK_BUILD avoids some internal testing
MAKEFLAGS: "--jobs=3 QUICK_BUILD=1"
ARTIFACTS_DIR: "c:/Artifacts/"
steps:
- uses: actions/checkout@v2
# https://stackoverflow.com/questions/58033366/how-to-get-current-branch-within-github-actions
- name: Extract branch name
shell: bash
run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
id: extract_branch
- name: Build System Info
shell: "bash"
run: |
echo "CWD: `pwd`"
df -h
systeminfo
echo NUMBER_OF_PROCESSORS=$NUMBER_OF_PROCESSORS
- name: docker pull ${{ matrix.container_image }}
run: docker pull ${{ matrix.container_image }}
- name: Start docker build environment
run: |
docker run --tty --memory 5G --cpus 2 --storage-opt 'size=50GB' --detach --name buildContainer ${{ matrix.container_image }}
- name: Print Info about docker system
shell: "bash"
run: |
docker ps -a
docker exec buildContainer systeminfo
docker exec buildContainer df -h
- name: Git Checkout
shell: "bash"
run: |
docker exec buildContainer sh -c "git clone https://github.com/SophistSolutions/Stroika.git && cd Stroika && git checkout ${{ steps.extract_branch.outputs.branch }}"
- name: Configure
shell: "bash"
run: |
docker exec --workdir c:/Stroika buildContainer sh -c "./configure ${{ matrix.config_name }} ${{ matrix.extra_config_args }}"
docker exec --workdir c:/Stroika buildContainer cat ConfigurationFiles/${{ matrix.config_name }}.xml
- name: Build
shell: "bash"
run: |
docker exec --workdir c:/Stroika --env MAKEFLAGS="$MAKEFLAGS" buildContainer make all
- name: Run-Tests
shell: "bash"
run: |
docker exec --workdir c:/Stroika --env MAKEFLAGS="$MAKEFLAGS" buildContainer make run-tests
- name: Build System Info
shell: "bash"
run: |
df -h
docker exec buildContainer df -h
- name: Copy Build Artifacts
shell: "bash"
# due to flaws in docker (windows must stop) - and cp no wildcards
run: |
docker exec --workdir c:/Stroika buildContainer bash -c 'mkdir TRACE_LOGS && cp $TEMP/Trace*.txt TRACE_LOGS/'
docker stop buildContainer
docker cp buildContainer:Stroika/Builds/${{ matrix.config_name }}/ $ARTIFACTS_DIR 2> /dev/null
docker cp buildContainer:Stroika/TRACE_LOGS $ARTIFACTS_DIR 2> /dev/null
rm -rf $ARTIFACTS_DIR/{ThirdPartyComponents,Tests,*.lib}
- name: Archive Log Results
uses: actions/upload-artifact@v2
with:
name: Log Results (${{ matrix.displayTargetName }})
path: |
${{ env.ARTIFACTS_DIR }}PerformanceDump.txt
${{ env.ARTIFACTS_DIR }}TRACE_LOGS
- name: Archive Sample Results
uses: actions/upload-artifact@v2
with:
name: Samples (${{ matrix.displayTargetName }})
path: |
${{ env.ARTIFACTS_DIR }}Samples-*
However, I'd like to only build some of the configurations only when the branch is v2.1-Release
. That is - for the most part - just build one or two configurations but build a bunch more on release.
I've accomplished this by cloning the script (workflow) and renaming a few things and commenting things out, but it would be nice if the mechanism worked with matrix elements.
I realize there is an if
feature that can be added to each step, but that would create tons of jobs with disabled steps. What I want is to not spin up those jobs at all for each matrix element that has the if
part evaluates false.
TLDR: you can do what you want with one workflow by filtering the configurations you want to use in a prior build job/step, and using the result of that filtering as the matrix value in your build-n-test
job.
Longer version:
You can create a job
(i.e. build-n-test) where the value of strategy.matrix
is different based off of some criteria by setting the value of strategy.matrix
to the deserialized output
of a previous job (i.e. matrix_prep). This previous job would have the responsibility of constructing the matrix
value as per your custom criteria. The following yaml demonstrates this (a copy has been included later on with comments added in for explanation):
name: Configurable Build Matrix
on: push
jobs:
matrix_prep:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v2
- id: set-matrix
run: |
branchName=$(echo '${{ github.ref }}' | sed 's,refs/heads/,,g')
matrix=$(jq --arg branchName "$branchName" 'map(
. | select((.runOn==$branchName) or (.runOn=="always"))
)' matrix_includes.json)
echo "matrix={\"include\":$(echo $matrix)}" >> $GITHUB_OUTPUT
build-n-test:
needs: matrix_prep
runs-on: ${{ matrix.runs_on }}
strategy:
matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}}
steps:
- run: echo "Hello ${{ matrix.someValue }}"
The contents of the matrix_includes.json
file used in the set-matrix task can be found after this paragraph. To see what the matrix configuration from the question would look like as JSON, please look near the bottom of this answer. I went the route of having a JSON file separate from the workflow definition itself because I found that including the raw JSON in the workflow itself was very messy (especially if the JSON file was large).
[
{
"runs_on":"ubuntu-16.04",
"someValue":"Foo",
"runOn":"always"
},
{
"runs_on":"ubuntu-18.04",
"someValue":"Bar",
"runOn":"v2.1-Release"
},
{
"runs_on":"ubuntu-20.04",
"someValue":"Hello again",
"runOn":"v2.1-Release"
}
]
Using the setup above, one configuration will be included for all builds, and two will only be included if the branch name matches v2.1-Release. With some tweaks to the sed
and jq
options in the Workflow file, the branch name restrictions could be made looser such that you could have configurations run for all branches that include -Release
(instead of just for a single branch). I may include this in this answer if there is interest (as it does not necessarily match your current question).
set-matrix
Job ExplanationAs far as the set-matrix
task is concerned, please refer to the following notes:
# ${{ github.ref }} returns the full git ref. As such, 'refs/heads/`
# should be stripped for easier future use.
branchName=$(echo '${{ github.ref }}' | sed 's,refs/heads/,,g')
# Use jq to read in a json file that represents the matrix configuration. Each
# block has a 'runOn' property. The jq filter is setup to only output items that
# are set to 'always' or that have a branch name that matches the current branch.
matrix=$(jq --arg branchName "$branchName" 'map(
. | select((.runOn==$branchName) or (.runOn=="always"))
)' matrix_includes.json)
# This 'echo' uses a special syntax so that the output of this job
# is set correctly.
echo "matrix={\"include\":$(echo $matrix)}" >> $GITHUB_OUTPUT
The following yaml content should be the same as above with some additional comments to help explain things:
name: Configurable Build Matrix
on: push
jobs:
matrix_prep:
# Using a separate job and agent so as to be able to use tools
# like 'sed' and 'jq'.
runs-on: ubuntu-latest
# Defining outputs of a job allows for easier consumption and use.
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
# Checking out code as the set-matrix step utilizes a
# file named matrix_includes.json.
- name: Check out code into the Go module directory
uses: actions/checkout@v2
# This step is explained more in a following section
- id: set-matrix
run: |
branchName=$(echo '${{ github.ref }}' | sed 's,refs/heads/,,g')
matrix=$(jq --arg branchName "$branchName" 'map(
. | select((.runOn==$branchName) or (.runOn=="always"))
)' matrix_includes.json)
echo "matrix={\"include\":$(echo $matrix)}" >> $GITHUB_OUTPUT
build-n-test:
# By stating 'needs' here, the output of 'matrix_prep' is
# available to this job.
needs: matrix_prep
runs-on: ${{ matrix.runs_on }}
strategy:
# We need to convert the json string output into an object that the
# GitHub Workflow expects. Thankfully, the json-schema for Workflows
# allows 'matrix' to be set to an expression.
matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}}
steps:
# Output a configuration specific value as proof and as a sanity check
- run: echo "Hello ${{ matrix.someValue }}"
I put together two files for this demonstration:
The following two screenshots are from runs on different branches using the same workflow definition. Please note that the amount of build-n-test Jobs are different between the two:
Build for main branch Build for v2.1-Release branch
This is due to the first build occurring on the main
branch, and the second occurring on the v2.1-Release
branch. As can be seen by the included matrix_includes.json
file above, this is to be expected as two configurations are set to run only when the branch is v2.1-Release
, and only one configuration is set to run always.
The filtering is accomplished through using jq to select objects from the json array that either have their runOn
value set to always
or that match the current branchName
. This is the slight tweak to your logic that I mentioned earlier: instead of saying includeInDevBranchBuilds
, I am using runOn
as it seemed to work better for this specific example.
The set-matrix
step uses a value set from the previous line: branchName=$(echo '${{ github.ref }}' | sed 's,refs/heads/.*-,,g')
. This line will strip refs/heads/
from the branch ref and store the result in the value branchName
. For example, if your branch is 2.1-Release
, branchName
will be set to 2.1-Release
, and the filter from earlier will then match any objects that have "runOn":"2.1-Release"
or "runOn":"always"
.
The JSON file was created to simulate the content of the includes
statement from the workflow that you linked. JSON is used as GitHub Actions have builtin JSON functions. As a sample, the following is my take on converting your matrix:include
section to JSON. Please note that I've changed includeInDevBranchBuilds
to be runOn
, with the values set to either always
or v2.1-Release
.
[
{
"displayTargetName": "centos-8",
"os": "unix",
"compiler": "g++",
"runs_on": "ubuntu-latest",
"container_image": "sophistsolutionsinc/stroika-buildvm-centos-8-small",
"cpp_version": "c++17",
"config_name": "Release",
"extra_config_args": "--apply-default-release-flags --trace2file enable",
"runOn": "always"
},
{
"displayTargetName": "ubuntu-18.04-g++-8 (Debug)",
"os": "unix",
"compiler": "g++-8",
"runs_on": "ubuntu-latest",
"container_image":
"sophistsolutionsinc/stroika-buildvm-ubuntu1804-regression-tests",
"cpp_version": "c++17",
"config_name": "Debug",
"extra_config_args": "--apply-default-debug-flags --trace2file enable",
"runOn": "always"
},
{
"displayTargetName": "ubuntu-20.04-g++-9 (Debug)",
"os": "unix",
"compiler": "g++-9",
"runs_on": "ubuntu-latest",
"container_image":
"sophistsolutionsinc/stroika-buildvm-ubuntu2004-regression-tests",
"cpp_version": "c++17",
"config_name": "Debug",
"extra_config_args": "--apply-default-debug-flags --trace2file enable",
"runOn": "v2.1-Release"
},
{
"displayTargetName": "ubuntu-20.04-g++-10 (Debug)",
"os": "unix",
"compiler": "g++-10",
"runs_on": "ubuntu-latest",
"container_image":
"sophistsolutionsinc/stroika-buildvm-ubuntu2004-regression-tests",
"cpp_version": "c++17",
"config_name": "Debug",
"extra_config_args": "--apply-default-debug-flags --trace2file enable",
"runOn": "v2.1-Release"
},
{
"displayTargetName": "ubuntu-20.04-g++-10",
"os": "unix",
"compiler": "g++-10",
"runs_on": "ubuntu-latest",
"container_image":
"sophistsolutionsinc/stroika-buildvm-ubuntu2004-regression-tests",
"cpp_version": "c++17",
"config_name": "Release",
"extra_config_args": "--apply-default-release-flags --trace2file enable",
"runOn": "always"
},
{
"displayTargetName": "ubuntu-20.04-g++-10-c++2a",
"os": "unix",
"compiler": "g++-10",
"runs_on": "ubuntu-latest",
"container_image":
"sophistsolutionsinc/stroika-buildvm-ubuntu2004-regression-tests",
"cpp_version": "c++2a",
"config_name": "Release",
"extra_config_args": "--apply-default-release-flags --trace2file enable",
"runOn": "v2.1-Release"
},
{
"displayTargetName": "ubuntu-20.04-clang++-10",
"os": "unix",
"compiler": "clang++-10",
"runs_on": "ubuntu-latest",
"container_image":
"sophistsolutionsinc/stroika-buildvm-ubuntu2004-regression-tests",
"cpp_version": "c++17",
"config_name": "Release",
"extra_config_args": "--apply-default-release-flags --trace2file enable",
"runOn": "v2.1-Release"
}
]