# Kokkos nightly tests at NERSC machine Perlmutter.
stages:
  - test
  - clean_up

default:
  tags:
    - perlmutter-login
  before_script:
    - export SOURCE_DIR=$SCRATCH/.nightly/testing/kokkos
    - if [ ! -d $SOURCE_DIR ]; then
    - git clone https://github.com/kokkos/kokkos.git ${SOURCE_DIR}
    - fi
    - cd ${SOURCE_DIR} && pwd && git pull && git branch
    - export CMAKE_BUILD_PARALLEL_LEVEL=32
    - export ENV_KOKKOS_OPTIONS=""
    - export ENV_KOKKOS_OPTIONS="${ENV_KOKKOS_OPTIONS};-DKokkos_ENABLE_TESTS=ON"
    - export ENV_KOKKOS_OPTIONS="${ENV_KOKKOS_OPTIONS};-DKokkos_ENABLE_COMPILER_WARNINGS=ON"
    - export ENV_KOKKOS_OPTIONS="${ENV_KOKKOS_OPTIONS};-DKokkos_ARCH_AMPERE80=ON"
    - export ENV_KOKKOS_OPTIONS="${ENV_KOKKOS_OPTIONS};-DKokkos_ARCH_ZEN3=ON"
    - export ENV_KOKKOS_OPTIONS="${ENV_KOKKOS_OPTIONS};-BUILD_SHARED_LIBS=ON"
    - export SCRIPT_FILE=${SOURCE_DIR}/scripts/CTestRun.cmake
    - export OMP_NUM_THREADS=4
    - export OMP_PROC_BIND=spread
    - export OMP_PLACES=cores

clang-cuda:
  stage: test
  script:
    - echo "Kokkos-OpenMP-CUDA tests by $GITLAB_USER_LOGIN using clang."
    - module load PrgEnv-llvm cudatoolkit
    - export BUILD_DIR=${SOURCE_DIR}/build_omp_cuda_clang
    - export ENV_CMAKE_OPTIONS="${ENV_KOKKOS_OPTIONS}"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-DCMAKE_CXX_COMPILER=clang++"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-DCMAKE_CUDA_COMPILER=clang++"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-DKokkos_ENABLE_CUDA=ON"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-DKokkos_ENABLE_OPENMP=ON"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-DCMAKE_CUDA_FLAGS=-Wno-pass-failed -Wno-unknown-cuda-version"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-DKokkos_ENABLE_COMPILE_AS_CMAKE_LANGUAGE=ON"
    - ctest -VV -D CDASH_MODEL="Nightly" -D CTEST_SITE="nersc" -D CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS}" -S ${SCRIPT_FILE} -D CTEST_BUILD_NAME="zen3-a100-omp-cuda13-clang21"

gcc-cuda:
  stage: test
  script:
    - echo "Kokkos-OpenMP-CUDA tests by $GITLAB_USER_LOGIN using gcc."
    - module load cudatoolkit/13.0 gcc-native/13.2
    - export BUILD_DIR=${SOURCE_DIR}/build_omp_cuda_gcc
    - export ENV_CMAKE_OPTIONS="${ENV_KOKKOS_OPTIONS}"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-DCMAKE_CXX_COMPILER=g++"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-D CMAKE_CXX_FLAGS='-Werror'"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-DKokkos_ENABLE_OPENMP=ON"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-DKokkos_ENABLE_CUDA=ON"
    - ctest -VV -D CDASH_MODEL="Nightly" -D CTEST_SITE="nersc" -D CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS}" -S ${SCRIPT_FILE} -D CTEST_BUILD_NAME="zen3-a100-omp-cuda13-gcc/13.2"

nvhpc-cuda:
  stage: test
  script:
    - echo "Kokkos-OpenMP-CUDA tests by $GITLAB_USER_LOGIN using nvhpc."
    - module load PrgEnv-nvidia cudatoolkit/13.0
    - export BUILD_DIR=${SOURCE_DIR}/build_omp_cuda_nvhpc
    - export ENV_CMAKE_OPTIONS="${ENV_KOKKOS_OPTIONS}"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-DCMAKE_CXX_COMPILER=nvc++"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-DKokkos_ENABLE_CUDA=ON"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-DKokkos_ENABLE_OPENMP=ON"
    - ctest -VV -D CDASH_MODEL="Nightly" -D CTEST_SITE="nersc" -D CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS}" -S ${SCRIPT_FILE} -D CTEST_BUILD_NAME="zen3-a100-omp-cuda13-nvhpc/24.5"

clang-cuda-cxx23:
  stage: test
  script:
    - echo "Kokkos-OpenMP-CUDA-C++23 tests by $GITLAB_USER_LOGIN using clang."
    - module load PrgEnv-llvm cudatoolkit
    - export BUILD_DIR=${SOURCE_DIR}/build_omp_cuda_clang_cxx23
    - export ENV_CMAKE_OPTIONS="${ENV_KOKKOS_OPTIONS}"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-DCMAKE_CXX_COMPILER=clang++"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-DCMAKE_CUDA_COMPILER=clang++"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-DKokkos_ENABLE_CUDA=ON"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-DKokkos_ENABLE_OPENMP=ON"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-DCMAKE_CUDA_FLAGS=-Wno-pass-failed -Wno-unknown-cuda-version"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-DKokkos_ENABLE_COMPILE_AS_CMAKE_LANGUAGE=ON"
    - export ENV_CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS};-DCMAKE_CXX_STANDARD=23"
    - ctest -VV -D CDASH_MODEL="Nightly" -D CTEST_SITE="nersc" -D CMAKE_OPTIONS="${ENV_CMAKE_OPTIONS}" -S ${SCRIPT_FILE} -D CTEST_BUILD_NAME="zen3-a100-omp-cuda13-clang21-cxx23"

clear-ci-builds:
  stage: clean_up
  script:
    - rm -rf ${SOURCE_DIR}