Advanced CI/CD Techniques for HPC using GitHub Actions

spack:
  view: /opt/view
  specs:
    - openmpi@4.1.6 %gcc
    - boost@1.83.0 %gcc
  config:
    install_tree:
      root: /opt/spack
      padded_length: 128

spack env create my-hpc-env
spack env activate my-hpc-env

spack containerise -e my-hpc-env --image-type docker

spack:
  # Define package configuration: sets global package requirements.
  packages:
    # "all" applies to every package in the environment.
    all:
      # Specify available compilers: GCC (version 14 and above) and Clang (versions 14 up to 18).
      compiler: [gcc@14:, clang@14:18]
    # "mpi" section targets MPI-specific settings.
    mpi:
      # Require OpenMPI up to version 4 with automatic fabric selection.
      require: ['openmpi@:4 fabrics=auto']

  # Definitions allow you to create aliases for reuse in specs.
  definitions:
  - compilers: [gcc@14:]    # Define a group of compilers.
  - packages: [caliper, chai+openmp, raja+openmp, umpire+openmp, kokkos+openmp]  # Define a group of packages with OpenMP support.

  # Specs define the concrete packages to install.
  specs:
  - $compilers   # Include the compiler group defined above.
  - matrix:
    - [$packages]    # Use the package group defined in definitions.
    - [$%compilers]  # Combine with the specified compiler group.

  # Enable the creation of a view, which creates a unified directory of installed packages.
  view: true

  # Concretizer settings: unify ensures consistent dependency resolution.
  concretizer:
    unify: true

  # Configuration for the system environment.
  config:
    os: ubuntu22.04    # Target operating system.
    target: x86_64     # Target architecture.

docker build -t my-hpc-image .

docker images | grep my-hpc-image

apptainer build my-hpc.sif docker://my-hpc-image:latest

jobs:
  build:
    runs-on: ubuntu-22.04
    steps:
    - name: Checkout Repository
      uses: actions/checkout@v4
    - name: Set up Spack
      uses: spack/setup-spack@v2
      with:
        ref: develop
        buildcache: true
        color: true
        path: spack
    - name: Create and Activate Spack Environment
      run: |
        spack env create ci-env
        spack env activate ci-env
        spack install python
    - name: Run Application Tests
      run: |
        spack env activate ci-env
        python -c "import sys; print('Hello from CI!')"

- name: Cache CMake Build Directory (1)
    uses: actions/cache@v4 (2)
    with: (3)
        path: build (4)
        key: ${{ runner.os }}-cmake-${{ hashFiles('CMakeLists.txt') }} (5)

- name: Upload Build Artifact  (1)
    uses: actions/upload-artifact@v4  (2)
    with:
        name: build-tarball  (3)
        path: build/default/*.tar.gz  (4)

- name: Download Build Artifact                   (1)
    uses: actions/download-artifact@v4            (2)
    with:                                         (3)
        name: build-tarball                       (4)
        path: downloaded-artifacts/               (5)

#!/bin/bash
#SBATCH --job-name=apptainer
#SBATCH --output=apptainer.out
#SBATCH --error=apptainer.err
#SBATCH --time=00:10:00
#SBATCH --partition=qcpu
#SBATCH --ntasks=16
#SBATCH --cpus-per-task=1
#SBATCH --mem=1G
#SBATCH --account=dd-24-88

SIF=${1:-c3b-template.sif} (1)

# Declare the following variables to be used by apptainer to avoid warnings
export APPTAINER_BINDPATH=$SINGULARITY_BINDPATH
export APPTAINERENV_LD_PRELOAD=$SINGULARITYENV_LD_PRELOAD

module load Boost/1.83.0-GCC-13.2.0 Ninja/1.12.1-GCCcore-13.3.0 OpenMPI/4.1.6-GCC-13.2.0

srun apptainer exec --sharens $SIF myapp (2)

# Get SIF filename from command line
SIF=${1:-c3b-template.sif}
WAIT=10

# Submit the job and capture the sbatch output.
job_output=$(sbatch job_myapp.sh $SIF)
echo "sbatch output: $job_output"

# Extract the job ID (assumes output: "Submitted batch job <jobid>")
jobid=$(echo "$job_output" | awk '{print $4}')
echo "Submitted job with ID: $jobid"

job_state=""

# Poll the job state using sacct until it reaches a terminal state.
while true; do
    job_state=$(sacct -j "$jobid" --format=State --noheader | head -n 1 | tr -d ' ')
    echo "Job $jobid state: $job_state"

    # Check if the job state starts with any terminal keyword.
    if [[ "$job_state" == COMPLETED* || "$job_state" == FAILED* || "$job_state" == CANCELLED* || "$job_state" == TIMEOUT* ]]; then
        break
    fi

    echo "Job $jobid is still running... waiting $WAIT seconds."
    sleep $WAIT
done

echo "Job $jobid has finished with state: $job_state"

echo "Detailed job information:"
# Show additional job information.
sacct -j "$jobid" --format=JobID,JobName,Partition,Account,AllocCPUS,State,ExitCode,Elapsed,Start,End --noheader

echo "Displaying job output log:"
cat apptainer.out

- name: Run Container on Karolina
  if: matrix.runs-on == 'karolina'      (1)
  run: |
    bash job_monitor.sh $SIF_FILENAME   (2)