pipeline {
    agent none

    environment {
        CCACHE_DIR = '/tmp/ccache'
        CCACHE_MAXSIZE = '5G'
        CCACHE_CPP2 = 'true'
        GTEST_SHUFFLE = 1
    }

    options {
        disableConcurrentBuilds(abortPrevious: true)
        timeout(time: 6, unit: 'HOURS')
    }

    triggers {
        issueCommentTrigger('.*test this please.*')
    }

    stages {
        stage('Pre-Commit') {
            agent {
                docker {
                    image 'jfxs/pre-commit:4.4.0-002@sha256:40078d585cc17c502d8c2390b8d57e7ecb028d75dcc821f2f75ac8e9c485bf84'
                    label 'nvidia-docker || docker'
                    registryCredentialsId 'dockerhub'
                    args '--env NODE_NAME=${env.NODE_NAME} --env STAGE_NAME=${env.STAGE_NAME}'
                }
            }
            steps {
                sh '''#!/bin/bash
                      exec > >(awk '{ print "[" ENVIRON["STAGE_NAME"] "]", $0 }') 2>&1 && \
                      echo "Hostname: ${NODE_NAME}" && \
                      pre-commit run --all-files'''
            }
            post {
                always {
                    cleanWs()
                       }
                 }
        }
        stage('Build-1') {
            parallel {
                stage('C++20-Modules-GCC-16') {
                    agent {
                         dockerfile {
                             filename 'Dockerfile.gcc-16'
                             dir 'scripts/docker'
                             label 'nvidia-docker || docker'
                             registryCredentialsId 'dockerhub'
                             args '--env NODE_NAME=${env.NODE_NAME} --env STAGE_NAME=${env.STAGE_NAME}'
                         }
                     }
                    environment {
                        OMP_NUM_THREADS = 8
                        OMP_NESTED = 'true'
                        OMP_MAX_ACTIVE_LEVELS = 3
                        OMP_PROC_BIND = 'true'
                    }
                    steps {
                        sh '''#!/bin/bash
                              exec > >(awk '{ print "[" ENVIRON["STAGE_NAME"] "]", $0 }') 2>&1 && \
                              echo "Hostname: ${NODE_NAME}" && \
                              rm -rf build && \
                              set -x && \
                              cmake \
                                -B build \
                                -GNinja \
                                -DCMAKE_CXX_COMPILER=g++-16 \
                                -DCMAKE_CXX_FLAGS=-Werror \
                                -DKokkos_ARCH_NATIVE=ON \
                                -DKokkos_ENABLE_COMPILER_WARNINGS=ON \
                                -DKokkos_ENABLE_EXPERIMENTAL_CXX20_MODULES=ON \
                                -DKokkos_ENABLE_OPENMP=ON \
                                -DKokkos_ENABLE_SERIAL=ON && \
                              set +x && \
                              cmake --build build --target kokkoscore &&
                              cd example/build_cmake_installed_modules && \
                              rm -rf build && \
                              set -x && \
                              cmake \
                                -B build \
                                -GNinja \
                                -DCMAKE_CXX_COMPILER=g++-16 \
                                -DCMAKE_CXX_FLAGS="-Werror" \
                                -DKokkos_ROOT=../../build && \
                              set +x && \
                              cmake --build build && \
                              ctest --test-dir build --verbose'''
                    }
                    post {
                        always {
                            cleanWs()
                               }
                         }
                }

                stage('GCC-10.5.0') {
                    agent {
                         dockerfile {
                             filename 'Dockerfile.gcc-10'
                             dir 'scripts/docker'
                             label 'nvidia-docker || docker'
                             registryCredentialsId 'dockerhub'
                             args '--env NODE_NAME=${env.NODE_NAME} --env STAGE_NAME=${env.STAGE_NAME}'
                         }
                     }
                    environment {
                        OMP_NUM_THREADS = 8
                        OMP_NESTED = 'true'
                        OMP_MAX_ACTIVE_LEVELS = 3
                        OMP_PROC_BIND = 'true'
                    }
                    steps {
                        sh '''#!/bin/bash
                              exec > >(awk '{ print "[" ENVIRON["STAGE_NAME"] "]", $0 }') 2>&1 && \
                              echo "Hostname: ${NODE_NAME}" && \
                              rm -rf build && mkdir -p build && cd build && \
                              set -x && \
                              cmake \
                                -DCMAKE_BUILD_TYPE=Release \
                                -DCMAKE_CXX_STANDARD=20 \
                                -DCMAKE_CXX_FLAGS=-Werror \
                                -DKokkos_ARCH_NATIVE=ON \
                                -DKokkos_ENABLE_COMPILER_WARNINGS=ON \
                                -DKokkos_ENABLE_DEPRECATED_CODE_5=ON \
                                -DKokkos_ENABLE_TESTS=ON \
                                -DKokkos_ENABLE_OPENMP=ON \
                                -DKokkos_ENABLE_LIBDL=OFF \
                                -DKokkos_ENABLE_LIBQUADMATH=ON \
                                -DKokkos_ENABLE_SERIAL=ON \
                              .. && \
                              set +x && \
                              ulimit -c 0 && \
                              make -j8 && ctest --no-compress-output -T Test --verbose && gcc -I$PWD/../core/src/ ../core/unit_test/tools/TestCInterface.c'''
                    }
                    post {
                        always {
                            xunit([CTest(deleteOutputFiles: true, failIfNotNew: true, pattern: 'build/Testing/**/Test.xml', skipNoTestFiles: false, stopProcessingIfError: true)])
                            cleanWs()
                        }
                    }
                }
                stage('HIP-ROCm-6.2-CXX20') {
                    agent {
                        dockerfile {
                            filename 'Dockerfile.hipcc'
                            dir 'scripts/docker'
                            additionalBuildArgs '--build-arg BASE=rocm/dev-ubuntu-22.04:6.2.4-complete@sha256:6604a97283a218fc62ab59e23c54ec34ad634be9201b001435844a59ba1b8eb5'
                            label 'rocm-docker'
                            registryCredentialsId 'dockerhub'
                            args '-v /tmp/ccache.kokkos:/tmp/ccache --device=/dev/kfd --device=/dev/dri --security-opt seccomp=unconfined --group-add video --env HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES --env NODE_NAME=${env.NODE_NAME} --env STAGE_NAME=${env.STAGE_NAME}'
                        }
                    }
                    steps {
                        sh 'ccache --zero-stats'
                        sh '''#!/bin/bash
                              exec > >(awk '{ print "[" ENVIRON["STAGE_NAME"] "]", $0 }') 2>&1 && \
                              echo "Hostname: ${NODE_NAME}" && \
                              rm -rf build && mkdir -p build && cd build && \
                              set -x && \
                              cmake \
                                -DBUILD_SHARED_LIBS=ON \
                                -DCMAKE_BUILD_TYPE=RelWithDebInfo \
                                -DCMAKE_CXX_COMPILER=hipcc \
                                -DCMAKE_CXX_FLAGS="-Werror -Wno-unused-command-line-argument" \
                                -DCMAKE_CXX_STANDARD=20 \
                                -DKokkos_ARCH_NATIVE=ON \
                                -DKokkos_ENABLE_COMPILER_WARNINGS=ON \
                                -DKokkos_ENABLE_DEPRECATED_CODE_5=ON \
                                -DKokkos_ENABLE_TESTS=ON \
                                -DKokkos_ENABLE_HIP=ON \
                                -DKokkos_ENABLE_ROCTHRUST=OFF \
                                -DKokkos_ENABLE_MULTIPLE_CMAKE_LANGUAGES=ON \
                                -DCMAKE_INSTALL_PREFIX=${PWD}/../install \
                              .. && \
                              set +x && \
                              ulimit -c 0 && \
                              make -j16 install && ctest --no-compress-output -T Test --verbose && \
                              cd .. && \
                              export CMAKE_PREFIX_PATH=${PWD}/install && \
                              cd example/build_cmake_installed_multilanguage && \
                              rm -rf build && mkdir -p build && cd build && \
                              set -x && \
                              cmake \
                                -DCMAKE_CXX_COMPILER=hipcc \
                                -DCMAKE_CXX_FLAGS=-Werror \
                                -DCMAKE_CXX_STANDARD=20 \
                              .. && \
                              set +x && \
                              make -j8 && ctest --verbose'''
                    }
                    post {
                        always {
                            sh 'ccache --show-stats'
                            xunit([CTest(deleteOutputFiles: true, failIfNotNew: true, pattern: 'build/Testing/**/Test.xml', skipNoTestFiles: false, stopProcessingIfError: true)])
                            cleanWs()
                        }
                    }
                }

                stage('CUDA-12.2-NVCC-RDC') {
                    agent {
                        dockerfile {
                            filename 'Dockerfile.nvcc'
                            dir 'scripts/docker'
                            additionalBuildArgs '--build-arg BASE=nvcr.io/nvidia/cuda:12.2.2-devel-ubuntu22.04@sha256:5f603101462baa721ff6ddc44af82f6e9ba7cbd92a424c9f9f348e6e9d6d64c3 --build-arg ADDITIONAL_PACKAGES="gfortran clang" --build-arg CMAKE_VERSION=3.25.3'
                            label 'nvidia-docker && (volta || ampere)'
                            args '-v /tmp/ccache.kokkos:/tmp/ccache --env NVIDIA_VISIBLE_DEVICES=$NVIDIA_VISIBLE_DEVICES --env NODE_NAME=${env.NODE_NAME} --env STAGE_NAME=${env.STAGE_NAME}'
                        }
                    }
                    environment {
                        CTEST_OUTPUT_ON_FAILURE = 1
                        OMP_NUM_THREADS = 8
                        // Nested OpenMP does not work for this configuration,
                        // so disabling it
                        OMP_MAX_ACTIVE_LEVELS = 1
                        OMP_PLACES = 'threads'
                        OMP_PROC_BIND = 'spread'
                        NVCC_WRAPPER_DEFAULT_COMPILER = 'g++-11'
                    }
                    steps {
                        sh 'ccache --zero-stats'
                        sh '''#!/bin/bash
                              exec > >(awk '{ print "[" ENVIRON["STAGE_NAME"] "]", $0 }') 2>&1 && \
                              echo "Hostname: ${NODE_NAME}" && \
                              rm -rf install && mkdir -p install && \
                              rm -rf build && mkdir -p build && cd build && \
                              set -x && \
                              cmake \
                                -DCMAKE_BUILD_TYPE=Release \
                                -DCMAKE_CXX_COMPILER=g++-11 \
                                -DCMAKE_CXX_FLAGS=-Werror \
                                -DCMAKE_CXX_STANDARD=20 \
                                -DKokkos_ARCH_NATIVE=ON \
                                -DKokkos_ENABLE_COMPILER_WARNINGS=ON \
                                -DKokkos_ENABLE_OPENMP=OFF \
                                -DKokkos_ENABLE_CUDA=ON \
                                -DKokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE=ON \
                                -DKokkos_ENABLE_DEPRECATED_CODE_5=ON \
                                -DKokkos_ENABLE_MULTIPLE_CMAKE_LANGUAGES=ON \
                                \
                                -DCMAKE_INSTALL_PREFIX=${PWD}/../install \
                              .. && \
                              set +x && \
                              ulimit -c 0 && \
                              make -j8 install && \
                              cd .. && \
                              rm -rf build-tests && mkdir -p build-tests && cd build-tests && \
                              export CMAKE_PREFIX_PATH=${PWD}/../install && \
                              set -x && \
                              cmake \
                                -DCMAKE_BUILD_TYPE=Release \
                                -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
                                -DCMAKE_CXX_COMPILER=$WORKSPACE/bin/nvcc_wrapper \
                                -DCMAKE_CXX_FLAGS="-Werror --Werror=all-warnings -Xcudafe --diag_suppress=940" \
                                -DCMAKE_EXE_LINKER_FLAGS="-Xnvlink -suppress-stack-size-warning" \
                                -DCMAKE_CXX_STANDARD=20 \
                                -DKokkos_INSTALL_TESTING=ON \
                              .. && \
                              set +x && \
                              make -j8 && ctest --no-compress-output -T Test --verbose && \
                              cd ../example/build_cmake_installed && \
                              rm -rf build && mkdir -p build && cd build && \
                              set -x && \
                              cmake \
                                -DCMAKE_CXX_COMPILER=g++-11 \
                                -DCMAKE_CXX_FLAGS=-Werror \
                                -DCMAKE_CXX_STANDARD=20 \
                              .. && \
                              set +x && \
                              make -j8 && ctest --verbose && \
                              cd ../../build_cmake_installed_multilanguage && \
                              rm -rf build && mkdir -p build && cd build && \
                              set -x && \
                              cmake \
                                -DCMAKE_CXX_COMPILER=g++-11 \
                                -DCMAKE_CXX_FLAGS=-Werror \
                                -DCMAKE_CXX_STANDARD=20 \
                              .. && \
                              set +x && \
                              make -j8 && ctest --verbose && \
                              cd ../.. && \
                              cmake -B build_cmake_installed_different_compiler/build -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_CXX_FLAGS=-Werror -DCMAKE_CXX_STANDARD=20 build_cmake_installed_different_compiler && \
                              cmake --build build_cmake_installed_different_compiler/build --target all && \
                              cmake --build build_cmake_installed_different_compiler/build --target test'''
                    }
                    post {
                        always {
                            sh 'ccache --show-stats'
                            xunit([CTest(deleteOutputFiles: true, failIfNotNew: true, pattern: 'build-tests/Testing/**/Test.xml', skipNoTestFiles: false, stopProcessingIfError: true)])
                            cleanWs()
                        }
                    }
                }

            }
        }
        stage('Build-2') {
            parallel {
                stage('C++20-Modules-Clang-19') {
                    agent {
                        dockerfile {
                            filename 'Dockerfile.modules'
                            dir 'scripts/docker'
                            label 'nvidia-docker || docker'
                            registryCredentialsId 'dockerhub'
                            args '--env NODE_NAME=${env.NODE_NAME} --env STAGE_NAME=${env.STAGE_NAME}'
                        }
                    }
                    steps {
                        sh '''#!/bin/bash
                              exec > >(awk '{ print "[" ENVIRON["STAGE_NAME"] "]", $0 }') 2>&1 && \
                              echo "Hostname: ${NODE_NAME}" && \
                              rm -rf build && \
                              set -x && \
                              cmake \
                                -B build \
                                -GNinja \
                                -DCMAKE_CXX_COMPILER=clang++-19 \
                                -DCMAKE_CXX_FLAGS="-Werror" \
                                -DCMAKE_CXX_STANDARD=20 \
                                -DKokkos_ENABLE_COMPILER_WARNINGS=ON \
                                -DKokkos_ENABLE_EXPERIMENTAL_CXX20_MODULES=ON \
                                -DKokkos_ENABLE_DEPRECATED_CODE_5=OFF \
                                -DKokkos_ENABLE_TESTS=ON \
                                -DKokkos_ENABLE_EXAMPLES=ON \
                                -DKokkos_ENABLE_SERIAL=ON && \
                              set +x && \
                              ulimit -c 0 && \
                              cmake --build build --target install -j 8 && \
                              ctest --test-dir build --no-compress-output -T Test --verbose && \
                              cd example/build_cmake_installed_modules && \
                              rm -rf build && \
                              set -x && \
                              cmake \
                                -B build \
                                -GNinja \
                                -DCMAKE_CXX_COMPILER=clang++-19 \
                                -DCMAKE_CXX_FLAGS="-Werror" && \
                              set +x && \
                              cmake --build build -j 8 && \
                              ctest --test-dir build --verbose'''
                    }
                    post {
                        always {
                            xunit([CTest(deleteOutputFiles: true, failIfNotNew: true, pattern: 'build/Testing/**/Test.xml', skipNoTestFiles: false, stopProcessingIfError: true)])
                            cleanWs()
                        }
                    }
                }

                stage('OPENACC-NVHPC-CUDA-12.2') {
                    agent {
                        dockerfile {
                            filename 'Dockerfile.nvhpc'
                            dir 'scripts/docker'
                            label 'nvidia-docker && volta && large_images'
                            args '--env NVIDIA_VISIBLE_DEVICES=$NVIDIA_VISIBLE_DEVICES --env NODE_NAME=${env.NODE_NAME} --env STAGE_NAME=${env.STAGE_NAME}'
                        }
                    }
                    environment {
                        NVHPC_CUDA_HOME = '/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/cuda/12.2'
                    }
                    steps {
                        sh '''#!/bin/bash
                              exec > >(awk '{ print "[" ENVIRON["STAGE_NAME"] "]", $0 }') 2>&1 && \
                              echo "Hostname: ${NODE_NAME}" && \
                              rm -rf build && mkdir -p build && cd build && \
                              set -x && \
                              /opt/cmake/bin/cmake \
                                -DCMAKE_CXX_COMPILER=nvc++ \
                                -DCMAKE_CXX_STANDARD=20 \
                                -DCMAKE_CXX_FLAGS=-Werror \
                                -DKokkos_ARCH_NATIVE=ON \
                                -DKokkos_ENABLE_COMPILER_WARNINGS=ON \
                                -DKokkos_ENABLE_TESTS=ON \
                                -DKokkos_ENABLE_OPENACC=ON \
                                -DKokkos_ARCH_VOLTA70=ON \
                              .. && \
                              set +x && \
                              ulimit -c 0 && \
                              make -j8 && ctest --no-compress-output -T Test --verbose'''
                    }
                    post {
                        always {
                            xunit([CTest(deleteOutputFiles: true, failIfNotNew: true, pattern: 'build/Testing/**/Test.xml', skipNoTestFiles: false, stopProcessingIfError: true)])
                            cleanWs()
                        }
                    }

                }
                stage('CUDA-12.2-NVHPC-AS-HOST-COMPILER') {
                    agent {
                        dockerfile {
                            filename 'Dockerfile.nvhpc'
                            dir 'scripts/docker'
                            label 'nvidia-docker && large_images && volta'
                            args '-v /tmp/ccache.kokkos:/tmp/ccache --env NVIDIA_VISIBLE_DEVICES=$NVIDIA_VISIBLE_DEVICES --env NODE_NAME=${env.NODE_NAME} --env STAGE_NAME=${env.STAGE_NAME}'
                        }
                    }
                    environment {
                        OMP_NUM_THREADS = 8
                        // Nested OpenMP does not work for this configuration,
                        // so disabling it
                        OMP_MAX_ACTIVE_LEVELS = 1
                        OMP_PLACES = 'threads'
                        OMP_PROC_BIND = 'spread'
                        NVHPC_CUDA_HOME = '/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/cuda/12.2'
                    }
                    steps {
                        sh '''#!/bin/bash
                              exec > >(awk '{ print "[" ENVIRON["STAGE_NAME"] "]", $0 }') 2>&1 && \
                              echo "Hostname: ${NODE_NAME}" && \
                              rm -rf build && mkdir -p build && cd build && \
                              set -x && \
                              /opt/cmake/bin/cmake \
                                -DCMAKE_BUILD_TYPE=RelWithDebInfo \
                                -DCMAKE_CXX_COMPILER=nvc++ \
                                -DCMAKE_CXX_STANDARD=20 \
                                -DCMAKE_CXX_FLAGS="-Werror -Werror=all-warnings --diag_suppress=implicit_return_from_non_void_function" \
                                -DKokkos_ARCH_NATIVE=ON \
                                -DKokkos_ENABLE_COMPILER_WARNINGS=ON \
                                -DKokkos_ENABLE_DEPRECATED_CODE_5=OFF \
                                -DKokkos_ENABLE_TESTS=ON \
                                -DKokkos_ENABLE_CUDA=ON \
                                -DKokkos_ENABLE_OPENMP=ON \
                              .. && \
                              set +x && \
                              ulimit -c 0 && \
                              make -j8 && ctest --no-compress-output -T Test --verbose'''
                    }
                    post {
                        always {
                            xunit([CTest(deleteOutputFiles: true, failIfNotNew: true, pattern: 'build/Testing/**/Test.xml', skipNoTestFiles: false, stopProcessingIfError: true)])
                            cleanWs()
                        }
                    }

                }
                stage('SYCL-OneAPI') {
                    agent {
                        dockerfile {
                            filename 'Dockerfile.sycl'
                            dir 'scripts/docker'
                            label 'nvidia-docker && ampere'
                            args '-v /tmp/ccache.kokkos:/tmp/ccache --env NODE_NAME=${env.NODE_NAME} --env STAGE_NAME=${env.STAGE_NAME}'
                        }
                    }
                    steps {
                        sh 'ccache --zero-stats'
                        sh '''#!/bin/bash
                              exec > >(awk '{ print "[" ENVIRON["STAGE_NAME"] "]", $0 }') 2>&1 && \
                              echo "Hostname: ${NODE_NAME}" && \
                              rm -rf build && mkdir -p build && cd build && \
                              set -x && \
                              cmake \
                                -DCMAKE_BUILD_TYPE=Release \
                                -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
                                -DCMAKE_CXX_COMPILER=icpx \
                                -DCMAKE_CXX_FLAGS="-fsycl-device-code-split=per_kernel -fp-model=precise -Wno-deprecated-declarations -Werror -Wno-gnu-zero-variadic-macro-arguments -Wno-unknown-cuda-version -Wno-sycl-target" \
                                -DKokkos_ARCH_NATIVE=ON \
                                -DKokkos_ARCH_AMPERE80=ON \
                                -DKokkos_ENABLE_COMPILER_WARNINGS=ON \
                                -DKokkos_ENABLE_DEPRECATED_CODE_5=OFF \
                                -DKokkos_ENABLE_EXAMPLES=ON \
                                -DKokkos_ENABLE_TESTS=ON \
                                -DoneDPL_ROOT=/opt/intel/oneapi/dpl/2022.9 \
                                -DKokkos_ENABLE_SYCL=ON \
                                -DKokkos_ENABLE_SYCL_RELOCATABLE_DEVICE_CODE=ON \
                                -DKokkos_ENABLE_UNSUPPORTED_ARCHS=ON \
                                -DCMAKE_CXX_STANDARD=20 \
                              .. && \
                              set +x && \
                              ulimit -c 0 && \
                              make -j8 && ctest --no-compress-output -T Test --verbose'''
                    }
                    post {
                        always {
                            sh 'ccache --show-stats'
                            xunit([CTest(deleteOutputFiles: true, failIfNotNew: true, pattern: 'build/Testing/**/Test.xml', skipNoTestFiles: false, stopProcessingIfError: true)])
                            cleanWs()
                        }
                    }
                }
                stage('HIP-ROCm-6.3') {
                    agent {
                        dockerfile {
                            filename 'Dockerfile.hipcc'
                            dir 'scripts/docker'
                            additionalBuildArgs '--build-arg BASE=rocm/dev-ubuntu-24.04:6.3.4-complete@sha256:76e99e263ef6ce69ba5d32905623c801fff3f85a6108e931820f6eb1d13eac67'
                            label 'rocm-docker'
                            registryCredentialsId 'dockerhub'
                            args '-v /tmp/ccache.kokkos:/tmp/ccache --device=/dev/kfd --device=/dev/dri --security-opt seccomp=unconfined --group-add video --env HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES --env NODE_NAME=${env.NODE_NAME} --env STAGE_NAME=${env.STAGE_NAME}'
                        }
                    }
                    environment {
                        OMP_NUM_THREADS = 8
                        OMP_MAX_ACTIVE_LEVELS = 3
                        OMP_PLACES = 'threads'
                        OMP_PROC_BIND = 'spread'
                    }
                    steps {
                        sh 'ccache --zero-stats'
                        sh 'echo "/opt/rocm/llvm/lib" > /etc/ld.so.conf.d/llvm.conf && ldconfig'
                        sh '''#!/bin/bash
                              exec > >(awk '{ print "[" ENVIRON["STAGE_NAME"] "]", $0 }') 2>&1 && \
                              echo "Hostname: ${NODE_NAME}" && \
                              rm -rf build && mkdir -p build && cd build && \
                              set -x && \
                              cmake \
                                -DCMAKE_BUILD_TYPE=Debug \
                                -DCMAKE_CXX_COMPILER=hipcc \
                                -DCMAKE_CXX_FLAGS="-Werror -Wno-unused-command-line-argument -DNDEBUG" \
                                -DCMAKE_CXX_STANDARD=20 \
                                -DKokkos_ARCH_NATIVE=ON \
                                -DKokkos_ENABLE_COMPILER_WARNINGS=ON \
                                -DKokkos_ENABLE_DEPRECATED_CODE_5=OFF \
                                -DKokkos_ENABLE_TESTS=ON \
                                -DKokkos_ENABLE_HIP=ON \
                                -DKokkos_ENABLE_OPENMP=ON \
                                -DKokkos_ENABLE_HIP_MULTIPLE_KERNEL_INSTANTIATIONS=ON \
                              .. && \
                              set +x && \
                              ulimit -c 0 && \
                              make -j16 && ctest --no-compress-output -T Test --verbose'''
                    }
                    post {
                        always {
                            sh 'ccache --show-stats'
                            xunit([CTest(deleteOutputFiles: true, failIfNotNew: true, pattern: 'build/Testing/**/Test.xml', skipNoTestFiles: false, stopProcessingIfError: true)])
                            cleanWs()
                        }
                    }
                }
                stage('HIP-ROCm-6.2-amdclang-CXX20') {
                    agent {
                        dockerfile {
                            filename 'Dockerfile.hipcc'
                            dir 'scripts/docker'
                            additionalBuildArgs '--build-arg BASE=rocm/dev-ubuntu-24.04:6.2-complete@sha256:c7049ac3ae8516c7b230deec6dc6dd678a0b3f7215d5a7f7fe2f2b71880b62f8 --build-arg ADDITIONAL_PACKAGES="clang-tidy"'
                            label 'rocm-docker'
                            registryCredentialsId 'dockerhub'
                            args '-v /tmp/ccache.kokkos:/tmp/ccache --device=/dev/kfd --device=/dev/dri --security-opt seccomp=unconfined --group-add video --env HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES --env NODE_NAME=${env.NODE_NAME} --env STAGE_NAME=${env.STAGE_NAME}'
                        }
                    }
                    environment {
                        // FIXME hip_hostpinned.view_allocation_large_rank test returns a wrong value
                        GTEST_FILTER = '-hip_hostpinned.view_allocation_large_rank'
                    }
                    steps {
                        sh 'ccache --zero-stats'
                        sh '''#!/bin/bash
                              exec > >(awk '{ print "[" ENVIRON["STAGE_NAME"] "]", $0 }') 2>&1 && \
                              echo "Hostname: ${NODE_NAME}" && \
                              rm -rf build && mkdir -p build && cd build && \
                              set -x && \
                              cmake \
                                -DBUILD_SHARED_LIBS=ON \
                                -DCMAKE_BUILD_TYPE=RelWithDebInfo \
                                -DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/amdclang++ \
                                -DCMAKE_CXX_CLANG_TIDY="clang-tidy;-warnings-as-errors=*" \
                                -DCMAKE_PREFIX_PATH=/opt/rocm/lib \
                                -DCMAKE_CXX_FLAGS="-Werror -Wno-unused-command-line-argument" \
                                -DCMAKE_CXX_STANDARD=20 \
                                -DKokkos_ARCH_NATIVE=ON \
                                -DKokkos_ENABLE_COMPILER_WARNINGS=ON \
                                -DKokkos_ENABLE_DEPRECATED_CODE_5=ON \
                                -DKokkos_ENABLE_TESTS=ON \
                                -DKokkos_ENABLE_HIP=ON \
                              .. && \
                              set +x && \
                              ulimit -c 0 && \
                              make -j16 && ctest --no-compress-output -T Test --verbose'''
                    }
                    post {
                        always {
                            sh 'ccache --show-stats'
                            xunit([CTest(deleteOutputFiles: true, failIfNotNew: true, pattern: 'build/Testing/**/Test.xml', skipNoTestFiles: false, stopProcessingIfError: true)])
                            cleanWs()
                        }
                    }
                }
                stage('CUDA-11.8-Clang-15') {
                    agent {
                        dockerfile {
                            filename 'Dockerfile.nvcc'
                            dir 'scripts/docker'
                            label 'nvidia-docker && volta'
                            args '-v /tmp/ccache.kokkos:/tmp/ccache --env NVIDIA_VISIBLE_DEVICES=$NVIDIA_VISIBLE_DEVICES --env NODE_NAME=${env.NODE_NAME} --env STAGE_NAME=${env.STAGE_NAME}'
                            additionalBuildArgs '--build-arg BASE=nvcr.io/nvidia/cuda:11.8.0-devel-ubuntu22.04 --build-arg ADDITIONAL_PACKAGES="clang-15 clang-tidy-15"'
                        }
                    }
                    steps {
                        sh 'ccache --zero-stats'
                        sh '''#!/bin/bash
                              exec > >(awk '{ print "[" ENVIRON["STAGE_NAME"] "]", $0 }') 2>&1 && \
                              echo "Hostname: ${NODE_NAME}" && \
                              rm -rf build && mkdir -p build && cd build && \
                              set -x && \
                              cmake \
                                -DCMAKE_BUILD_TYPE=RelWithDebInfo \
                                -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
                                -DCMAKE_CXX_COMPILER=clang++-15 \
                                -DCMAKE_CXX_CLANG_TIDY="clang-tidy-15;-warnings-as-errors=*" \
                                -DCMAKE_CXX_FLAGS="-Werror -Wno-unknown-cuda-version -Wno-pass-failed" \
                                -DCMAKE_CXX_STANDARD=20 \
                                -DKokkos_ARCH_NATIVE=ON \
                                -DKokkos_ENABLE_COMPILER_WARNINGS=ON \
                                -DKokkos_ENABLE_DEPRECATED_CODE_5=ON \
                                -DKokkos_ENABLE_TESTS=ON \
                                -DKokkos_ENABLE_CUDA=ON \
                                -DKokkos_ENABLE_TUNING=ON \
                                -DKokkos_ARCH_VOLTA70=ON \
                              .. && \
                              set +x && \
                              ulimit -c 0 && \
                              make -j8 && ctest --no-compress-output -T Test --verbose'''
                    }
                    post {
                        always {
                            sh 'ccache --show-stats'
                            xunit([CTest(deleteOutputFiles: true, failIfNotNew: true, pattern: 'build/Testing/**/Test.xml', skipNoTestFiles: false, stopProcessingIfError: true)])
                            cleanWs()
                        }
                    }
                }
                stage('CUDA-12.5.1-Clang-17-RDC') {
                    agent {
                        dockerfile {
                            filename 'Dockerfile.nvcc'
                            dir 'scripts/docker'
                            label 'nvidia-docker && volta'
                            args '-v /tmp/ccache.kokkos:/tmp/ccache --env NVIDIA_VISIBLE_DEVICES=$NVIDIA_VISIBLE_DEVICES --env NODE_NAME=${env.NODE_NAME} --env STAGE_NAME=${env.STAGE_NAME}'
                            additionalBuildArgs '--build-arg BASE=nvcr.io/nvidia/cuda:12.5.1-devel-ubuntu24.04 --build-arg ADDITIONAL_PACKAGES="clang-17 clang-tidy-17"'
                        }
                    }
                    steps {
                        sh 'ccache --zero-stats'
                        sh '''#!/bin/bash
                              exec > >(awk '{ print "[" ENVIRON["STAGE_NAME"] "]", $0 }') 2>&1 && \
                              echo "Hostname: ${NODE_NAME}" && \
                              rm -rf build && mkdir -p build && cd build && \
                              set -x && \
                              cmake \
                                -DCMAKE_BUILD_TYPE=RelWithDebInfo \
                                -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
                                -DCMAKE_CXX_COMPILER=clang++-17 \
                                -DCMAKE_CXX_CLANG_TIDY="clang-tidy-17;-warnings-as-errors=*" \
                                -DCMAKE_CXX_FLAGS="-Werror -Wno-unknown-cuda-version -Wno-pass-failed" \
                                -DCMAKE_CXX_STANDARD=20 \
                                -DKokkos_ARCH_NATIVE=ON \
                                -DKokkos_ENABLE_COMPILER_WARNINGS=ON \
                                -DKokkos_ENABLE_DEPRECATED_CODE_5=ON \
                                -DKokkos_ENABLE_TESTS=ON \
                                -DKokkos_ENABLE_CUDA=ON \
                                -DKokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE=ON \
                                -DKokkos_ENABLE_TUNING=ON \
                                -DKokkos_ARCH_VOLTA70=ON \
                              .. && \
                              set +x && \
                              ulimit -c 0 && \
                              make -j8 && ctest --no-compress-output -T Test --verbose'''
                    }
                    post {
                        always {
                            sh 'ccache --show-stats'
                            xunit([CTest(deleteOutputFiles: true, failIfNotNew: true, pattern: 'build/Testing/**/Test.xml', skipNoTestFiles: false, stopProcessingIfError: true)])
                            cleanWs()
                        }
                    }
                }
                stage('CUDA-13.0-NVCC-DEBUG') {
                    agent {
                        dockerfile {
                            filename 'Dockerfile.nvcc'
                            dir 'scripts/docker'
                            additionalBuildArgs '--build-arg BASE=nvcr.io/nvidia/cuda:13.0.0-devel-ubuntu24.04@sha256:435220c0fef35cbf712e11999f8670a83835ef3cdd18564e5e8122f83078c88c --build-arg CMAKE_VERSION=3.22.6'
                            label 'nvidia-docker && ampere && cuda-13-driver'
                            args '-v /tmp/ccache.kokkos:/tmp/ccache --env NVIDIA_VISIBLE_DEVICES=$NVIDIA_VISIBLE_DEVICES --env NODE_NAME=${env.NODE_NAME} --env STAGE_NAME=${env.STAGE_NAME}'
                        }
                    }
                    environment {
                        KOKKOS_NUM_THREADS = 8
                    }
                    steps {
                        sh 'ccache --zero-stats'
                        sh '''#!/bin/bash
                              exec > >(awk '{ print "[" ENVIRON["STAGE_NAME"] "]", $0 }') 2>&1 && \
                              echo "Hostname: ${NODE_NAME}" && \
                              rm -rf build && mkdir -p build && cd build && \
                              set -x && \
                              cmake \
                                -DBUILD_SHARED_LIBS=ON \
                                -DCMAKE_BUILD_TYPE=Debug \
                                -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
                                -DCMAKE_CXX_COMPILER=$WORKSPACE/bin/nvcc_wrapper \
                                -DCMAKE_CXX_FLAGS="-Werror -Werror=all-warnings" \
                                -DCMAKE_CXX_STANDARD=20 \
                                -DKokkos_ARCH_NATIVE=ON \
                                -DKokkos_ENABLE_COMPILER_WARNINGS=ON \
                                -DKokkos_ENABLE_DEBUG=ON \
                                -DKokkos_ENABLE_DEBUG_BOUNDS_CHECK=ON \
                                -DKokkos_ENABLE_DEPRECATED_CODE_5=OFF \
                                -DKokkos_ENABLE_TESTS=ON \
                                -DKokkos_ENABLE_CUDA=ON \
                                -DKokkos_ENABLE_LIBDL=OFF \
                                -DKokkos_ENABLE_THREADS=ON \
                                -DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=ON \
                              .. && \
                              set +x && \
                              ulimit -c 0 && \
                              make -j8 && ctest --no-compress-output -T Test --verbose && \
                              cd ../example/build_cmake_in_tree && \
                              rm -rf build && mkdir -p build && cd build && \
                              set -x && cmake -DCMAKE_CXX_STANDARD=20 .. && set +x && make -j8 && ctest --verbose'''
                    }
                    post {
                        always {
                            sh 'ccache --show-stats'
                            xunit([CTest(deleteOutputFiles: true, failIfNotNew: true, pattern: 'build/Testing/**/Test.xml', skipNoTestFiles: false, stopProcessingIfError: true)])
                            cleanWs()
                        }
                    }
                }
            }
        }
    }
}