Re-add all compile and arch dependent flags

Fix ACC not linking on CMake < 3.16
Fix CUDA warnings for CMP0104 and avoid repeated -O[n] flags
Fix ComputeCpp not picking up custom flags
[CI] Highlight compiler warnings
[CI] Don't skip remaining tests when one fails
[CI] Add CMake 3.13, 3.15, 3.18 checks

.github/workflows/main.yaml

@@ -21,19 +21,78 @@ jobs:
 
       - name: Setup test environment
         run: source ./ci-prepare-bionic.sh ./compilers VARS false || true
-      - name: Test compile gcc
+
-        run: ./ci-test-compile.sh ./build gcc all
+      - name: Test compile gcc     @ CMake 3.13
-      - name: Test compile clang
+        if: ${{ ! cancelled() }}
-        run: ./ci-test-compile.sh ./build clang all
+        run: ./ci-test-compile.sh ./build gcc all ${{ env.CMAKE_3_13_BIN }}
-      - name: Test compile nvhpc
+      - name: Test compile clang   @ CMake 3.13
-        run: ./ci-test-compile.sh ./build nvhpc all
+        if: ${{ ! cancelled() }}
-      - name: Test compile aocc
+        run: ./ci-test-compile.sh ./build clang all ${{ env.CMAKE_3_13_BIN }}
-        run: ./ci-test-compile.sh ./build aocc all
+      - name: Test compile nvhpc   @ CMake 3.13
-      - name: Test compile aomp
+        if: ${{ ! cancelled() }}
-        run: ./ci-test-compile.sh ./build aomp all
+        run: ./ci-test-compile.sh ./build nvhpc all ${{ env.CMAKE_3_13_BIN }}
-      - name: Test compile hip
+      - name: Test compile aocc    @ CMake 3.13
-        run: ./ci-test-compile.sh ./build hip all
+        if: ${{ ! cancelled() }}
-      - name: Test compile dpcpp
+        run: ./ci-test-compile.sh ./build aocc all ${{ env.CMAKE_3_13_BIN }}
-        run: ./ci-test-compile.sh ./build dpcpp all
+      - name: Test compile aomp    @ CMake 3.13
-      - name: Test compile hipsycl
+        if: ${{ ! cancelled() }}
-        run: ./ci-test-compile.sh ./build hipsycl all
+        run: ./ci-test-compile.sh ./build aomp all ${{ env.CMAKE_3_13_BIN }}
+      - name: Test compile hip     @ CMake 3.13
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build hip all ${{ env.CMAKE_3_13_BIN }}
+      - name: Test compile dpcpp   @ CMake 3.13
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build dpcpp all ${{ env.CMAKE_3_13_BIN }}
+      - name: Test compile hipsycl @ CMake 3.13
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build hipsycl all ${{ env.CMAKE_3_13_BIN }}
+
+      - name: Test compile gcc     @ CMake 3.15
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build gcc all ${{ env.CMAKE_3_15_BIN }}
+      - name: Test compile clang   @ CMake 3.15
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build clang all ${{ env.CMAKE_3_15_BIN }}
+      - name: Test compile nvhpc   @ CMake 3.15
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build nvhpc all ${{ env.CMAKE_3_15_BIN }}
+      - name: Test compile aocc    @ CMake 3.15
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build aocc all ${{ env.CMAKE_3_15_BIN }}
+      - name: Test compile aomp    @ CMake 3.15
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build aomp all ${{ env.CMAKE_3_15_BIN }}
+      - name: Test compile hip     @ CMake 3.15
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build hip all ${{ env.CMAKE_3_15_BIN }}
+      - name: Test compile dpcpp   @ CMake 3.15
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build dpcpp all ${{ env.CMAKE_3_15_BIN }}
+      - name: Test compile hipsycl @ CMake 3.15
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build hipsycl all ${{ env.CMAKE_3_15_BIN }}
+
+      - name: Test compile gcc     @ CMake 3.18
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build gcc all ${{ env.CMAKE_3_18_BIN }}
+      - name: Test compile clang   @ CMake 3.18
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build clang all ${{ env.CMAKE_3_18_BIN }}
+      - name: Test compile nvhpc   @ CMake 3.18
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build nvhpc all ${{ env.CMAKE_3_18_BIN }}
+      - name: Test compile aocc    @ CMake 3.18
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build aocc all ${{ env.CMAKE_3_18_BIN }}
+      - name: Test compile aomp    @ CMake 3.18
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build aomp all ${{ env.CMAKE_3_18_BIN }}
+      - name: Test compile hip     @ CMake 3.18
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build hip all ${{ env.CMAKE_3_18_BIN }}
+      - name: Test compile dpcpp   @ CMake 3.18
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build dpcpp all ${{ env.CMAKE_3_18_BIN }}
+      - name: Test compile hipsycl @ CMake 3.18
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build hipsycl all ${{ env.CMAKE_3_18_BIN }}

ACC.cmake

@@ -45,7 +45,17 @@ register_flag_optional(TARGET_PROCESSOR
 
 macro(setup)
     find_package(OpenACC REQUIRED)
-    register_link_library(OpenACC::OpenACC_CXX)
+
+    if(${CMAKE_VERSION} VERSION_LESS "3.16.0")
+        # CMake didn't really implement ACC as a target before 3.16, so we append them manually
+        separate_arguments(OpenACC_CXX_FLAGS)
+        register_append_cxx_flags(ANY ${OpenACC_CXX_FLAGS})
+        register_append_link_flags(${OpenACC_CXX_FLAGS})
+    else()
+        register_link_library(OpenACC::OpenACC_CXX)
+    endif()
+
+
     register_definitions(restrict=__restrict)
     # XXX NVHPC is really new so older Cmake thinks it's PGI, which is true
     if ((CMAKE_CXX_COMPILER_ID STREQUAL PGI) OR (CMAKE_CXX_COMPILER_ID STREQUAL NVHPC))

CMakeLists.txt

@@ -64,18 +64,9 @@ if ((NOT BUILD_TYPE STREQUAL RELEASE) AND (NOT BUILD_TYPE STREQUAL DEBUG))
     message(FATAL_ERROR "Only Release or Debug is supported, got `${CMAKE_BUILD_TYPE}`")
 endif ()
 
-
+# setup some defaults flags for everything
-# setup common build flag defaults if there are no overrides
+set(DEFAULT_DEBUG_FLAGS -O2 -fno-omit-frame-pointer)
-if (NOT DEFINED DEBUG_FLAGS)
+set(DEFAULT_RELEASE_FLAGS -O3 -march=native)
-    # XXX switch on compiler type for the best default here, right now it's just GCC-compatible flags
-    set(DEBUG_FLAGS -O2 -fno-omit-frame-pointer)
-endif ()
-if (NOT DEFINED RELEASE_FLAGS)
-    # XXX switch on compiler type for the best default here, right now it's just GCC-compatible flags
-    set(RELEASE_FLAGS -O3 -march=native)
-endif ()
-
-message(STATUS "Common ${CMAKE_BUILD_TYPE} flags are `${${BUILD_TYPE}_FLAGS}`, set ${BUILD_TYPE}_FLAGS to override")
 
 macro(hint_flag FLAG DESCRIPTION)
     if (NOT DEFINED ${FLAG})
@@ -146,13 +137,28 @@ setup()
 
 # CMake insists that -O2 (or equivalent) is the universally accepted optimisation level
 # we remove that here and use our own <BUILD_TYPE>_FLAGS
-string(REGEX REPLACE "([\\/\\-]O.)" ""
+wipe_gcc_style_optimisation_flags(CMAKE_CXX_FLAGS_${BUILD_TYPE})
-        CMAKE_CXX_FLAGS_${BUILD_TYPE} "${CMAKE_CXX_FLAGS_${BUILD_TYPE}}")
+
+message(STATUS "Default ${CMAKE_BUILD_TYPE} flags are `${DEFAULT_${BUILD_TYPE}_FLAGS}`, set ${BUILD_TYPE}_FLAGS to override (CXX_EXTRA_* flags are not affected)")
+
+# setup common build flag defaults if there are no overrides
+if (NOT DEFINED ${BUILD_TYPE}_FLAGS)
+    set(ACTUAL_${BUILD_TYPE}_FLAGS ${DEFAULT_${BUILD_TYPE}_FLAGS})
+    elseif()
+    set(ACTUAL_${BUILD_TYPE}_FLAGS ${${BUILD_TYPE}_FLAGS})
+endif ()
+
 
 message(STATUS "CXX vendor  : ${CMAKE_CXX_COMPILER_ID} (${CMAKE_CXX_COMPILER})")
+message(STATUS "Platform    : ${CMAKE_SYSTEM_PROCESSOR}")
 message(STATUS "Sources     : ${IMPL_SOURCES}")
 message(STATUS "Libraries   : ${LINK_LIBRARIES}")
-message(STATUS "CXX Flags   : ${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${${BUILD_TYPE}_FLAGS} ${CXX_EXTRA_FLAGS} ")
+message(STATUS "CXX Flags   : ${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${ACTUAL_${BUILD_TYPE}_FLAGS} ${CXX_EXTRA_FLAGS}
+    CXX flags derived from (CMake + (Override ? Override : Default) + Extras), where:
+        CMake                    = `${CMAKE_CXX_FLAGS_${BUILD_TYPE}}`
+        Default                  = `${DEFAULT_${BUILD_TYPE}_FLAGS}`
+        Override (RELEASE_FLAGS) = `${${BUILD_TYPE}_FLAGS}`
+        Extras (CXX_EXTRA_FLAGS) = `${CXX_EXTRA_FLAGS}`")
 message(STATUS "Link Flags  : ${LINK_FLAGS} ${CXX_EXTRA_LINK_FLAGS}")
 message(STATUS "Linker Flags: ${CMAKE_EXE_LINKER_FLAGS} ${CXX_EXTRA_LINKER_FLAGS} ")
 message(STATUS "Defs        : ${IMPL_DEFINITIONS}")
@@ -168,8 +174,8 @@ if (CXX_EXTRA_LIBRARIES)
     target_link_libraries(${EXE_NAME} PUBLIC ${CXX_EXTRA_LIBRARIES})
 endif ()
 
-target_compile_options(${EXE_NAME} PUBLIC "$<$<CONFIG:Release>:${RELEASE_FLAGS};${CXX_EXTRA_FLAGS}>")
+target_compile_options(${EXE_NAME} PUBLIC "$<$<CONFIG:Release>:${ACTUAL_RELEASE_FLAGS};${CXX_EXTRA_FLAGS}>")
-target_compile_options(${EXE_NAME} PUBLIC "$<$<CONFIG:Debug>:${DEBUG_FLAGS};${CXX_EXTRA_FLAGS}>")
+target_compile_options(${EXE_NAME} PUBLIC "$<$<CONFIG:Debug>:${ACTUAL_DEBUG_FLAGS};${CXX_EXTRA_FLAGS}>")
 
 target_link_options(${EXE_NAME} PUBLIC LINKER:${CXX_EXTRA_LINKER_FLAGS})
 target_link_options(${EXE_NAME} PUBLIC ${LINK_FLAGS} ${CXX_EXTRA_LINK_FLAGS})

CUDA.cmake

@@ -12,7 +12,7 @@ register_flag_optional(MEM "Device memory mode:
 register_flag_required(CMAKE_CUDA_COMPILER
         "Path to the CUDA nvcc compiler")
 
-# XXX CMake 3.18 supports CMAKE_CUDA_ARCHITECTURES/CUDA_ARCHITECTURES but we support older CMakes
+# XXX we may want to drop this eventually and use CMAKE_CUDA_ARCHITECTURES directly
 register_flag_required(CUDA_ARCH
         "Nvidia architecture, will be passed in via `-arch=` (e.g `sm_70`) for nvcc")
 
@@ -22,9 +22,22 @@ register_flag_optional(CUDA_EXTRA_FLAGS
 
 
 macro(setup)
+
+    # XXX CMake 3.18 supports CMAKE_CUDA_ARCHITECTURES/CUDA_ARCHITECTURES but we support older CMakes
+    if(POLICY CMP0104)
+        cmake_policy(SET CMP0104 OLD)
+    endif()
+
     enable_language(CUDA)
     register_definitions(MEM=${MEM})
-    set(CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS} -arch=${CUDA_ARCH} ${CUDA_EXTRA_FLAGS})
+
-    message(STATUS "NVCC flags: ${CMAKE_CUDA_FLAGS}")
+    # add -forward-unknown-to-host-compiler for compatibility reasons
+    set(CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS} "-forward-unknown-to-host-compiler -arch=${CUDA_ARCH}" ${CUDA_EXTRA_FLAGS})
+
+    # CMake defaults to -O2 for CUDA at Release, let's wipe that and use the global RELEASE_FLAG
+    # appended later
+    wipe_gcc_style_optimisation_flags(CMAKE_CUDA_FLAGS_${BUILD_TYPE})
+
+    message(STATUS "NVCC flags: ${CMAKE_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS_${BUILD_TYPE}}")
 endmacro()
 

KOKKOS.cmake

@@ -12,6 +12,9 @@ register_flag_required(KOKKOS_IN_TREE
 
          See https://github.com/kokkos/kokkos/blob/master/BUILD.md for all available options")
 
+# compiler vendor and arch specific flags
+set(KOKKOS_FLAGS_CPU_INTEL -qopt-streaming-stores=always)
+
 macro(setup)
 
     cmake_policy(SET CMP0074 NEW) #see https://github.com/kokkos/kokkos/blob/master/BUILD.md
@@ -25,6 +28,12 @@ macro(setup)
         message(FATAL_ERROR "`${KOKKOS_IN_TREE}` does not exist")
     endif ()
 
+    register_append_compiler_and_arch_specific_cxx_flags(
+            KOKKOS_FLAGS_CPU
+            ${CMAKE_CXX_COMPILER_ID}
+            ${CMAKE_SYSTEM_PROCESSOR}
+    )
+
 endmacro()
 
 

OMP.cmake

@@ -32,21 +32,51 @@
 #    NVHPC = NVIDIA HPC SDK Compiler (nvidia.com)
 
 
+# CMAKE_SYSTEM_PROCESSOR is set via `uname -p`, we have:
+# Power9 = ppc64le
+# x64    = x86_64
+# arm64  = aarch64
+#
+
+
 #predefined offload flags based on compiler id
-set(OMP_FLAGS_INTEL_CPU -qopt-streaming-stores=always)
+
-set(OMP_FLAGS_OFFLOAD_INTEL -qnextgen -fiopenmp -fopenmp-targets=spir64)
+
-set(OMP_FLAGS_OFFLOAD_GNU_NVIDIA -foffload=nvptx-none)
+set(OMP_FLAGS_OFFLOAD_INTEL
-set(OMP_FLAGS_OFFLOAD_GNU_AMD -foffload=amdgcn-amdhsa)
+        -qnextgen -fiopenmp -fopenmp-targets=spir64)
+set(OMP_FLAGS_OFFLOAD_GNU_NVIDIA
+        -foffload=nvptx-none)
+set(OMP_FLAGS_OFFLOAD_GNU_AMD
+        -foffload=amdgcn-amdhsa)
 set(OMP_FLAGS_OFFLOAD_CLANG_NVIDIA
         -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target=nvptx64-nvidia-cuda)
 set(OMP_FLAGS_OFFLOAD_CLANG_AMD
         -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target=amdgcn-amd-amdhsa)
-set(OMP_FLAGS_OFFLOAD_CLANG_ARCH_FLAG -march=)
+set(OMP_FLAGS_OFFLOAD_CLANG_ARCH_FLAG
+        -march=) # prefix only, arch appended by the vendor:arch tuple
+
+
+set(OMP_FLAGS_CPU_INTEL
+        -qopt-streaming-stores=always)
+set(OMP_FLAGS_CPU_GNU_PPC64LE
+        -mcpu=native)
+set(OMP_FLAGS_CPU_XL
+        -O5 -qarch=auto -qtune=auto)
+
+# NEC
+set(OMP_FLAGS_CPU_NEC -O4 -finline)
 
 register_flag_optional(CMAKE_CXX_COMPILER
         "Any CXX compiler that supports OpenMP as per CMake detection (and offloading if enabled with `OFFLOAD`)"
         "c++")
 
+register_flag_optional(ARCH
+        "This overrides CMake's CMAKE_SYSTEM_PROCESSOR detection which uses (uname -p), this is mainly for use with
+         specialised accelerators only and not to be confused with offload which is is mutually exclusive with this.
+         Supported values are:
+          - NEC"
+        "")
+
 register_flag_optional(OFFLOAD
         "Whether to use OpenMP offload, the format is <VENDOR:ARCH?>|ON|OFF.
         We support a small set of known offload flags for clang, gcc, and icpx.
@@ -75,10 +105,25 @@ macro(setup)
     register_link_library(OpenMP::OpenMP_CXX)
 
     string(TOUPPER ${CMAKE_CXX_COMPILER_ID} COMPILER)
+    if(NOT ARCH)
+        string(TOUPPER ${CMAKE_SYSTEM_PROCESSOR} ARCH)
+    else()
+        message(STATUS "Using custom arch: ${ARCH}")
+    endif()
+
+
 
     if (("${OFFLOAD}" STREQUAL OFF) OR (NOT DEFINED OFFLOAD))
         # no offload
 
+        # resolve the CPU specific flags
+        # starting with ${COMPILER_VENDOR}_${PLATFORM_ARCH}, then try ${COMPILER_VENDOR}, and then give up
+        register_append_compiler_and_arch_specific_cxx_flags(
+                OMP_FLAGS_CPU
+                ${COMPILER}
+                ${ARCH}
+        )
+
     elseif ("${OFFLOAD}" STREQUAL ON)
         #  offload but with custom flags
         register_definitions(OMP_TARGET_GPU)

RAJA.cmake

@@ -30,6 +30,8 @@ register_flag_optional(CUDA_EXTRA_FLAGS
         "[TARGET==NVIDIA only] Additional CUDA flags passed to nvcc, this is appended after `CUDA_ARCH`"
         "")
 
+# compiler vendor and arch specific flags
+set(RAJA_FLAGS_CPU_INTEL -qopt-streaming-stores=always)
 
 macro(setup)
 
@@ -56,9 +58,15 @@ macro(setup)
         set(ENABLE_CUDA ${ENABLE_CUDA} CACHE BOOL "" FORCE)
 
         if (ENABLE_CUDA)
+
+            # XXX CMake 3.18 supports CMAKE_CUDA_ARCHITECTURES/CUDA_ARCHITECTURES but we support older CMakes
+            if(POLICY CMP0104)
+                cmake_policy(SET CMP0104 OLD)
+            endif()
+
             # RAJA needs all the cuda stuff setup before including!
             set(CMAKE_CUDA_COMPILER ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc)
-            set(CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS} "-extended-lambda -arch=${CUDA_ARCH}" ${CUDA_EXTRA_FLAGS})
+            set(CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS} "-forward-unknown-to-host-compiler -extended-lambda -arch=${CUDA_ARCH}" ${CUDA_EXTRA_FLAGS})
             list(APPEND CMAKE_CUDA_FLAGS)
 
             message(STATUS "NVCC flags: ${CMAKE_CUDA_FLAGS}")
@@ -81,4 +89,10 @@ macro(setup)
     endif ()
 
 
+    register_append_compiler_and_arch_specific_cxx_flags(
+            RAJA_FLAGS_CPU
+            ${CMAKE_CXX_COMPILER_ID}
+            ${CMAKE_SYSTEM_PROCESSOR}
+    )
+
 endmacro()

README.md

@@ -59,16 +59,19 @@ Drivers, compiler and software applicable to whichever implementation you would
 
 ### CMake
 
-The project supports building with CMake >= 3.13.0.
+The project supports building with CMake >= 3.13.0, it can be installed without root via the [official script](https://cmake.org/download/).
 As with any CMake project, first configure the project:
 
 ```shell
 > cd babelstream
-> cmake -Bbuild -H. -DMODEL=<model> <model_specific_flags...> # configure the build, build type defaults to Release 
+> cmake -Bbuild -H. -DMODEL=<model> <model specific flags prefixed with -D...> # configure the build, build type defaults to Release 
 > cmake --build build # compile it 
 > ./build/babelstream # executable available at ./build/
 ```
 
+By default, we have defined a set of optimal flags for known HPC compilers and assigned those to `RELEASE_FLAG`.
+You can override this if required.
+
 To find out what flag each model supports or requires, simply configure while only specifying the model.
 For example:
 ```shell

SYCL.cmake

@@ -46,7 +46,6 @@ macro(setup)
 
         list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/Modules)
         set(ComputeCpp_DIR ${SYCL_COMPILER_DIR})
-        set(COMPUTECPP_USER_FLAGS -O3 -no-serial-memop)
 
         # don't point to the CL dir as the imports already have the CL prefix
         set(OpenCL_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/CL")
@@ -55,6 +54,9 @@ macro(setup)
         # ComputeCpp needs OpenCL
         find_package(ComputeCpp REQUIRED)
 
+        # this must come after FindComputeCpp (!)
+        set(COMPUTECPP_USER_FLAGS -O3 -no-serial-memop)
+
     elseif (${SYCL_COMPILER} STREQUAL "DPCPP")
         set(CMAKE_CXX_COMPILER ${SYCL_COMPILER_DIR}/bin/clang++)
         include_directories(${SYCL_COMPILER_DIR}/include/sycl)

ci-prepare-bionic.sh

@@ -76,6 +76,17 @@ get_and_install_deb() {
 
 }
 
+get() {
+  local name="$1"
+  local pkg_url="$2"
+  if [ "$SETUP" = true ]; then
+    if [ ! -f "$name" ] || [ "$FORCE_DOWNLOAD" = true ]; then
+      echo "$name not found, downloading..."
+      wget -q --show-progress --progress=bar:force:noscroll "$pkg_url" -O "$name"
+    fi
+  fi
+}
+
 get_and_untar() {
   local name="$1"
   local pkg_url="$2"
@@ -306,9 +317,37 @@ else
   echo "Running locally, defaulting to standard export"
 fi
 
+setup_cmake() {
+
+  echo "Preparing CMake"
+
+  local cmake_release="https://github.com/Kitware/CMake/releases/download"
+
+  get "cmake-3.13.sh" "$cmake_release/v3.13.4/cmake-3.13.4-Linux-x86_64.sh"
+  chmod +x "./cmake-3.13.sh" && sh "./cmake-3.13.sh" --skip-license --include-subdir
+  export_var CMAKE_3_13_BIN "$PWD/cmake-3.13.4-Linux-x86_64/bin/cmake"
+  verify_bin_exists "$CMAKE_3_13_BIN"
+  "$CMAKE_3_13_BIN" --version
+
+  get "cmake-3.15.sh" "$cmake_release/v3.15.7/cmake-3.15.7-Linux-x86_64.sh"
+  chmod +x "./cmake-3.15.sh" && "./cmake-3.15.sh" --skip-license --include-subdir
+  export_var CMAKE_3_15_BIN "$PWD/cmake-3.15.7-Linux-x86_64/bin/cmake"
+  verify_bin_exists "$CMAKE_3_15_BIN"
+  "$CMAKE_3_15_BIN" --version
+
+  get "cmake-3.18.sh" "$cmake_release/v3.18.6/cmake-3.18.6-Linux-x86_64.sh"
+  chmod +x "./cmake-3.18.sh" && "./cmake-3.18.sh" --skip-license --include-subdir
+  export_var CMAKE_3_18_BIN "$PWD/cmake-3.18.6-Linux-x86_64/bin/cmake"
+  verify_bin_exists "$CMAKE_3_18_BIN"
+  "$CMAKE_3_18_BIN" --version
+
+  check_size
+
+}
 
 if [ "$PARALLEL" = true ]; then
   (setup_clang_gcc && setup_rocm && setup_hipsycl) &   # these need apt so run sequentially
+  setup_cmake &
   setup_oclcpu &
   setup_aocc &
   setup_nvhpc &
@@ -318,6 +357,7 @@ if [ "$PARALLEL" = true ]; then
   setup_raja &
   wait
 else
+  setup_cmake
   setup_aocc
   setup_oclcpu
   setup_nvhpc

ci-test-compile.sh

@@ -8,6 +8,7 @@ export CCACHE_DISABLE=1
 BUILD_DIR=${1:-build}
 COMPILER=${2:-all}
 MODEL=${3:-all}
+CMAKE_BIN=${4}
 
 LOG_DIR="$BUILD_DIR"
 
@@ -45,13 +46,13 @@ run_build() {
   set +e
 
   # shellcheck disable=SC2086
-  cmake -B"$build" -H. \
+  "$CMAKE_BIN" -B"$build" -H. \
     -DCMAKE_BUILD_TYPE=Release \
     -DCMAKE_VERBOSE_MAKEFILE=ON \
     -DMODEL="$model" $flags &>>"$log"
   local cmake_code=$?
 
-  cmake --build "$build" --target babelstream -j "$(nproc)" &>>"$log"
+  "$CMAKE_BIN" --build "$build" --target babelstream -j "$(nproc)" &>>"$log"
   local cmake_code=$?
   set -e
 
@@ -59,17 +60,19 @@ run_build() {
   if [[ -f "$bin" ]]; then
     echo "$(tput setaf 2)[PASS!]($model->$build)$(tput sgr0): -DMODEL=$model $flags"
     # shellcheck disable=SC2002
+    cat "$log" | sed '/^--/d' | grep -i "/bin/nvcc" | sed 's/^/    /'
     cat "$log" | sed '/^--/d' | grep -i "$grep_kw" | sed 's/^/    /'
+    cat "$log" | sed '/^--/d' | grep -i "warning" | sed "s/.*/    $(tput setaf 3)&$(tput sgr0)/"
   else
     echo "$(tput setaf 1)[FAIL!]($model->$build)$(tput sgr0): -DMODEL=$model $flags"
-    echo "      CMake exited with code $cmake_code, see full build log at $log, reproduced below:"
+    echo "      $(tput setaf 1)CMake exited with code $cmake_code, see full build log at $log, reproduced below:$(tput sgr0)"
     cat "$log"
     exit 1
   fi
   echo "    $(tput setaf 4)$(file "$bin")$(tput sgr0)"
 }
 
-##
+###
 #KOKKOS_SRC="/home/tom/Downloads/kokkos-3.3.00"
 #RAJA_SRC="/home/tom/Downloads/RAJA-v0.13.0"
 #
@@ -82,10 +85,6 @@ run_build() {
 #NVHPC_CUDA_DIR="$NVSDK/cuda/11.2"
 #"$NVSDK/compilers/bin/makelocalrc" "$NVSDK/compilers/bin/" -x
 #
-##NVHPC_NVCXX="/opt/nvidia/hpc_sdk/Linux_x86_64/21.1/compilers/bin/nvc++"
-##NVHPC_NVCC="/opt/nvidia/hpc_sdk/Linux_x86_64/21.1/cuda/11.2/bin/nvcc"
-##NVHPC_CUDA_DIR="/opt/nvidia/hpc_sdk/Linux_x86_64/21.1/cuda/11.2"
-#
 #AOCC_CXX="/opt/AMD/aocc-compiler-2.3.0/bin/clang++"
 #AOMP_CXX="/usr/lib/aomp/bin/clang++"
 #OCL_LIB="/home/tom/Downloads/oclcpuexp-2020.11.11.0.04_rel/x64/libOpenCL.so"
@@ -98,13 +97,15 @@ run_build() {
 #HIPSYCL_DIR="/opt/hipsycl/cff515c/"
 #
 #ICPX_CXX="/opt/intel/oneapi/compiler/2021.1.2/linux/bin/icpx"
+#ICPC_CXX="/opt/intel/oneapi/compiler/2021.1.2/linux/bin/intel64/icpc"
 #
-#GCC_STD_PAR_LIB=""
+#GCC_STD_PAR_LIB="tbb"
-#CLANG_STD_PAR_LIB=""
+#CLANG_STD_PAR_LIB="tbb"
 #GCC_OMP_OFFLOAD_AMD=false
 #GCC_OMP_OFFLOAD_NVIDIA=true
 #CLANG_OMP_OFFLOAD_AMD=false
 #CLANG_OMP_OFFLOAD_NVIDIA=false
+###
 
 AMD_ARCH="gfx_903"
 NV_ARCH="sm_70"
@@ -196,10 +197,26 @@ build_hip() {
 }
 
 build_icpx() {
-  source /opt/intel/oneapi/setvars.sh -force
+  # clang derived
+  set +u
+  source /opt/intel/oneapi/setvars.sh -force || true
+  set -u
   run_build intel_build "${ICPX_CXX:?}" OMP "-DCMAKE_CXX_COMPILER=${ICPX_CXX:?} -DOFFLOAD=INTEL"
 }
 
+build_icpc() {
+  # icc/icpc
+  set +u
+  source /opt/intel/oneapi/setvars.sh -force || true
+  set -u
+  local name="intel_build"
+  local cxx="-DCMAKE_CXX_COMPILER=${ICPC_CXX:?}"
+  run_build $name "${ICPC_CXX:?}" OMP "$cxx"
+  run_build $name "${ICPC_CXX:?}" OCL "$cxx -DOpenCL_LIBRARY=${OCL_LIB:?}"
+  run_build $name "${ICPC_CXX:?}" RAJA "$cxx -DRAJA_IN_TREE=${RAJA_SRC:?}"
+  run_build $name "${ICPC_CXX:?}" KOKKOS "$cxx -DKOKKOS_IN_TREE=${KOKKOS_SRC:?} -DKokkos_ENABLE_OPENMP=ON"
+}
+
 build_computecpp() {
   run_build computecpp_build "compute++" SYCL "-DCMAKE_CXX_COMPILER=${GCC_CXX:?} \
   -DSYCL_COMPILER=COMPUTECPP \
@@ -223,9 +240,8 @@ build_hipsycl() {
   -DSYCL_COMPILER_DIR=${HIPSYCL_DIR:?}"
 }
 
-# TODO tested locally but can't install compilers for these two remotely without registration/license:
+echo "Test compiling with ${COMPILER} CXX for ${MODEL} model"
-# build_icpx
+"$CMAKE_BIN" --version
-# build_computecpp
 
 case "$COMPILER" in
 gcc) build_gcc ;;
@@ -236,6 +252,12 @@ aomp) build_aomp ;;
 hip) build_hip ;;
 dpcpp) build_dpcpp ;;
 hipsycl) build_hipsycl ;;
+
+# XXX below are local only; licence or very large download required, candidate for local runner
+computecpp) build_computecpp ;;
+icpx) build_icpx ;;
+icpc) build_icpc ;;
+
 all)
   build_gcc
   build_clang
@@ -245,6 +267,11 @@ all)
   build_hip
   build_dpcpp
   build_hipsycl
+
+  build_computecpp
+  build_icpx
+  build_icpc
+
   ;;
 *)
   echo "Unknown $COMPILER, use ALL to compile with all supported compilers"

register_models.cmake

@@ -19,15 +19,19 @@
 #endfunction()
 #
 
+macro(wipe_gcc_style_optimisation_flags VAR)
+    string(REGEX REPLACE "([\\/\\-]O.)" "" ${VAR} ${${VAR}})
+endmacro()
+
 macro(register_link_library)
     list(APPEND LINK_LIBRARIES ${ARGN})
 endmacro()
 
 macro(register_append_cxx_flags CONFIG)
     if ("${CONFIG}" STREQUAL "RELEASE" OR "${CONFIG}" STREQUAL "ANY")
-        list(APPEND RELEASE_FLAGS ${ARGN})
+        list(APPEND DEFAULT_RELEASE_FLAGS ${ARGN})
     elseif ("${CONFIG}" STREQUAL "DEBUG" OR "${CONFIG}" STREQUAL "ANY")
-        list(APPEND DEBUG_FLAGS ${ARGN})
+        list(APPEND DEFAULT_DEBUG_FLAGS ${ARGN})
     else ()
         message(FATAL_ERROR "register_flags supports only RELEASE, DEBUG, or ANY for all configs, got `${CONFIG}`")
     endif ()
@@ -37,6 +41,19 @@ macro(register_append_link_flags)
     list(APPEND LINK_FLAGS ${ARGN})
 endmacro()
 
+macro(register_append_compiler_and_arch_specific_cxx_flags PREFIX CXX ARCH)
+    string(TOUPPER ${CXX} _CXX)
+    string(TOUPPER ${ARCH} _ARCH)
+    set(_CXX_ARCH_SPECIFIC_FLAGS "${${PREFIX}_${_CXX}_${_ARCH}}")
+    if (_CXX_ARCH_SPECIFIC_FLAGS)
+        register_append_cxx_flags(ANY ${_CXX_ARCH_SPECIFIC_FLAGS})
+    endif ()
+    set(_CXX_ARCH_SPECIFIC_FLAGS "${${PREFIX}_${_CXX}}")
+    if (_CXX_ARCH_SPECIFIC_FLAGS)
+        register_append_cxx_flags(ANY ${_CXX_ARCH_SPECIFIC_FLAGS})
+    endif ()
+endmacro()
+
 macro(register_definitions)
     list(APPEND IMPL_DEFINITIONS ${ARGN})
 endmacro()
@@ -100,25 +117,12 @@ function(registered_flags_action ACTION OUT)
 endfunction()
 
 
-
-
 macro(register_model NAME PREPROCESSOR_NAME)
     string(TOUPPER ${NAME} MODEL_UPPER)
     list(APPEND REGISTERED_MODELS "${NAME}")
 
     list(APPEND IMPL_${MODEL_UPPER}_SOURCES "${ARGN}")
     list(APPEND IMPL_${MODEL_UPPER}_DEFINITIONS "${PREPROCESSOR_NAME}")
-
-#    if ("${MODEL_UPPER}" STREQUAL ${MODEL})
-#
-#        set(MODEL_FILE ${CMAKE_CURRENT_SOURCE_DIR}/${NAME}.cmake)
-#
-#        if (NOT EXISTS ${MODEL_FILE})
-#            message(FATAL_ERROR "${MODEL_FILE} not found, perhaps it needs to be implemented?")
-#        endif ()
-#        include(${MODEL_FILE})
-#
-#    endif ()
 endmacro()