Re-add all compile and arch dependent flags

Fix ACC not linking on CMake < 3.16 Fix CUDA warnings for CMP0104 and avoid repeated -O[n] flags Fix ComputeCpp not picking up custom flags [CI] Highlight compiler warnings [CI] Don't skip remaining tests when one fails [CI] Add CMake 3.13, 3.15, 3.18 checks
2021-03-11 15:46:23 +00:00 · 2021-03-11 15:46:23 +00:00 · 14aefecc57
commit 14aefecc57
parent ec131e2cfc
12 changed files with 307 additions and 75 deletions
--- a/.github/workflows/main.yaml
+++ b/.github/workflows/main.yaml
@ -21,19 +21,78 @@ jobs:

      - name: Setup test environment
        run: source ./ci-prepare-bionic.sh ./compilers VARS false || true
-      - name: Test compile gcc
-        run: ./ci-test-compile.sh ./build gcc all
-      - name: Test compile clang
-        run: ./ci-test-compile.sh ./build clang all
-      - name: Test compile nvhpc
-        run: ./ci-test-compile.sh ./build nvhpc all
-      - name: Test compile aocc
-        run: ./ci-test-compile.sh ./build aocc all
-      - name: Test compile aomp
-        run: ./ci-test-compile.sh ./build aomp all
-      - name: Test compile hip
-        run: ./ci-test-compile.sh ./build hip all
-      - name: Test compile dpcpp
-        run: ./ci-test-compile.sh ./build dpcpp all
-      - name: Test compile hipsycl
-        run: ./ci-test-compile.sh ./build hipsycl all
+
+      - name: Test compile gcc     @ CMake 3.13
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build gcc all ${{ env.CMAKE_3_13_BIN }}
+      - name: Test compile clang   @ CMake 3.13
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build clang all ${{ env.CMAKE_3_13_BIN }}
+      - name: Test compile nvhpc   @ CMake 3.13
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build nvhpc all ${{ env.CMAKE_3_13_BIN }}
+      - name: Test compile aocc    @ CMake 3.13
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build aocc all ${{ env.CMAKE_3_13_BIN }}
+      - name: Test compile aomp    @ CMake 3.13
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build aomp all ${{ env.CMAKE_3_13_BIN }}
+      - name: Test compile hip     @ CMake 3.13
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build hip all ${{ env.CMAKE_3_13_BIN }}
+      - name: Test compile dpcpp   @ CMake 3.13
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build dpcpp all ${{ env.CMAKE_3_13_BIN }}
+      - name: Test compile hipsycl @ CMake 3.13
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build hipsycl all ${{ env.CMAKE_3_13_BIN }}
+
+      - name: Test compile gcc     @ CMake 3.15
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build gcc all ${{ env.CMAKE_3_15_BIN }}
+      - name: Test compile clang   @ CMake 3.15
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build clang all ${{ env.CMAKE_3_15_BIN }}
+      - name: Test compile nvhpc   @ CMake 3.15
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build nvhpc all ${{ env.CMAKE_3_15_BIN }}
+      - name: Test compile aocc    @ CMake 3.15
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build aocc all ${{ env.CMAKE_3_15_BIN }}
+      - name: Test compile aomp    @ CMake 3.15
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build aomp all ${{ env.CMAKE_3_15_BIN }}
+      - name: Test compile hip     @ CMake 3.15
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build hip all ${{ env.CMAKE_3_15_BIN }}
+      - name: Test compile dpcpp   @ CMake 3.15
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build dpcpp all ${{ env.CMAKE_3_15_BIN }}
+      - name: Test compile hipsycl @ CMake 3.15
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build hipsycl all ${{ env.CMAKE_3_15_BIN }}
+
+      - name: Test compile gcc     @ CMake 3.18
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build gcc all ${{ env.CMAKE_3_18_BIN }}
+      - name: Test compile clang   @ CMake 3.18
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build clang all ${{ env.CMAKE_3_18_BIN }}
+      - name: Test compile nvhpc   @ CMake 3.18
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build nvhpc all ${{ env.CMAKE_3_18_BIN }}
+      - name: Test compile aocc    @ CMake 3.18
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build aocc all ${{ env.CMAKE_3_18_BIN }}
+      - name: Test compile aomp    @ CMake 3.18
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build aomp all ${{ env.CMAKE_3_18_BIN }}
+      - name: Test compile hip     @ CMake 3.18
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build hip all ${{ env.CMAKE_3_18_BIN }}
+      - name: Test compile dpcpp   @ CMake 3.18
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build dpcpp all ${{ env.CMAKE_3_18_BIN }}
+      - name: Test compile hipsycl @ CMake 3.18
+        if: ${{ ! cancelled() }}
+        run: ./ci-test-compile.sh ./build hipsycl all ${{ env.CMAKE_3_18_BIN }}
--- a/ACC.cmake
+++ b/ACC.cmake
@ -45,7 +45,17 @@ register_flag_optional(TARGET_PROCESSOR

 macro(setup)
    find_package(OpenACC REQUIRED)
-    register_link_library(OpenACC::OpenACC_CXX)
+
+    if(${CMAKE_VERSION} VERSION_LESS "3.16.0")
+        # CMake didn't really implement ACC as a target before 3.16, so we append them manually
+        separate_arguments(OpenACC_CXX_FLAGS)
+        register_append_cxx_flags(ANY ${OpenACC_CXX_FLAGS})
+        register_append_link_flags(${OpenACC_CXX_FLAGS})
+    else()
+        register_link_library(OpenACC::OpenACC_CXX)
+    endif()
+
+
    register_definitions(restrict=__restrict)
    # XXX NVHPC is really new so older Cmake thinks it's PGI, which is true
    if ((CMAKE_CXX_COMPILER_ID STREQUAL PGI) OR (CMAKE_CXX_COMPILER_ID STREQUAL NVHPC))
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -64,18 +64,9 @@ if ((NOT BUILD_TYPE STREQUAL RELEASE) AND (NOT BUILD_TYPE STREQUAL DEBUG))
    message(FATAL_ERROR "Only Release or Debug is supported, got `${CMAKE_BUILD_TYPE}`")
 endif ()

-
-# setup common build flag defaults if there are no overrides
-if (NOT DEFINED DEBUG_FLAGS)
-    # XXX switch on compiler type for the best default here, right now it's just GCC-compatible flags
-    set(DEBUG_FLAGS -O2 -fno-omit-frame-pointer)
-endif ()
-if (NOT DEFINED RELEASE_FLAGS)
-    # XXX switch on compiler type for the best default here, right now it's just GCC-compatible flags
-    set(RELEASE_FLAGS -O3 -march=native)
-endif ()
-
-message(STATUS "Common ${CMAKE_BUILD_TYPE} flags are `${${BUILD_TYPE}_FLAGS}`, set ${BUILD_TYPE}_FLAGS to override")
+# setup some defaults flags for everything
+set(DEFAULT_DEBUG_FLAGS -O2 -fno-omit-frame-pointer)
+set(DEFAULT_RELEASE_FLAGS -O3 -march=native)

 macro(hint_flag FLAG DESCRIPTION)
    if (NOT DEFINED ${FLAG})
@ -146,13 +137,28 @@ setup()

 # CMake insists that -O2 (or equivalent) is the universally accepted optimisation level
 # we remove that here and use our own <BUILD_TYPE>_FLAGS
-string(REGEX REPLACE "([\\/\\-]O.)" ""
-        CMAKE_CXX_FLAGS_${BUILD_TYPE} "${CMAKE_CXX_FLAGS_${BUILD_TYPE}}")
+wipe_gcc_style_optimisation_flags(CMAKE_CXX_FLAGS_${BUILD_TYPE})
+
+message(STATUS "Default ${CMAKE_BUILD_TYPE} flags are `${DEFAULT_${BUILD_TYPE}_FLAGS}`, set ${BUILD_TYPE}_FLAGS to override (CXX_EXTRA_* flags are not affected)")
+
+# setup common build flag defaults if there are no overrides
+if (NOT DEFINED ${BUILD_TYPE}_FLAGS)
+    set(ACTUAL_${BUILD_TYPE}_FLAGS ${DEFAULT_${BUILD_TYPE}_FLAGS})
+    elseif()
+    set(ACTUAL_${BUILD_TYPE}_FLAGS ${${BUILD_TYPE}_FLAGS})
+endif ()
+

 message(STATUS "CXX vendor  : ${CMAKE_CXX_COMPILER_ID} (${CMAKE_CXX_COMPILER})")
+message(STATUS "Platform    : ${CMAKE_SYSTEM_PROCESSOR}")
 message(STATUS "Sources     : ${IMPL_SOURCES}")
 message(STATUS "Libraries   : ${LINK_LIBRARIES}")
-message(STATUS "CXX Flags   : ${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${${BUILD_TYPE}_FLAGS} ${CXX_EXTRA_FLAGS} ")
+message(STATUS "CXX Flags   : ${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${ACTUAL_${BUILD_TYPE}_FLAGS} ${CXX_EXTRA_FLAGS}
+    CXX flags derived from (CMake + (Override ? Override : Default) + Extras), where:
+        CMake                    = `${CMAKE_CXX_FLAGS_${BUILD_TYPE}}`
+        Default                  = `${DEFAULT_${BUILD_TYPE}_FLAGS}`
+        Override (RELEASE_FLAGS) = `${${BUILD_TYPE}_FLAGS}`
+        Extras (CXX_EXTRA_FLAGS) = `${CXX_EXTRA_FLAGS}`")
 message(STATUS "Link Flags  : ${LINK_FLAGS} ${CXX_EXTRA_LINK_FLAGS}")
 message(STATUS "Linker Flags: ${CMAKE_EXE_LINKER_FLAGS} ${CXX_EXTRA_LINKER_FLAGS} ")
 message(STATUS "Defs        : ${IMPL_DEFINITIONS}")
@ -168,8 +174,8 @@ if (CXX_EXTRA_LIBRARIES)
    target_link_libraries(${EXE_NAME} PUBLIC ${CXX_EXTRA_LIBRARIES})
 endif ()

-target_compile_options(${EXE_NAME} PUBLIC "$<$<CONFIG:Release>:${RELEASE_FLAGS};${CXX_EXTRA_FLAGS}>")
-target_compile_options(${EXE_NAME} PUBLIC "$<$<CONFIG:Debug>:${DEBUG_FLAGS};${CXX_EXTRA_FLAGS}>")
+target_compile_options(${EXE_NAME} PUBLIC "$<$<CONFIG:Release>:${ACTUAL_RELEASE_FLAGS};${CXX_EXTRA_FLAGS}>")
+target_compile_options(${EXE_NAME} PUBLIC "$<$<CONFIG:Debug>:${ACTUAL_DEBUG_FLAGS};${CXX_EXTRA_FLAGS}>")

 target_link_options(${EXE_NAME} PUBLIC LINKER:${CXX_EXTRA_LINKER_FLAGS})
 target_link_options(${EXE_NAME} PUBLIC ${LINK_FLAGS} ${CXX_EXTRA_LINK_FLAGS})
--- a/CUDA.cmake
+++ b/CUDA.cmake
@ -12,7 +12,7 @@ register_flag_optional(MEM "Device memory mode:
 register_flag_required(CMAKE_CUDA_COMPILER
        "Path to the CUDA nvcc compiler")

-# XXX CMake 3.18 supports CMAKE_CUDA_ARCHITECTURES/CUDA_ARCHITECTURES but we support older CMakes
+# XXX we may want to drop this eventually and use CMAKE_CUDA_ARCHITECTURES directly
 register_flag_required(CUDA_ARCH
        "Nvidia architecture, will be passed in via `-arch=` (e.g `sm_70`) for nvcc")

@ -22,9 +22,22 @@ register_flag_optional(CUDA_EXTRA_FLAGS


 macro(setup)
+
+    # XXX CMake 3.18 supports CMAKE_CUDA_ARCHITECTURES/CUDA_ARCHITECTURES but we support older CMakes
+    if(POLICY CMP0104)
+        cmake_policy(SET CMP0104 OLD)
+    endif()
+
    enable_language(CUDA)
    register_definitions(MEM=${MEM})
-    set(CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS} -arch=${CUDA_ARCH} ${CUDA_EXTRA_FLAGS})
-    message(STATUS "NVCC flags: ${CMAKE_CUDA_FLAGS}")
+
+    # add -forward-unknown-to-host-compiler for compatibility reasons
+    set(CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS} "-forward-unknown-to-host-compiler -arch=${CUDA_ARCH}" ${CUDA_EXTRA_FLAGS})
+
+    # CMake defaults to -O2 for CUDA at Release, let's wipe that and use the global RELEASE_FLAG
+    # appended later
+    wipe_gcc_style_optimisation_flags(CMAKE_CUDA_FLAGS_${BUILD_TYPE})
+
+    message(STATUS "NVCC flags: ${CMAKE_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS_${BUILD_TYPE}}")
 endmacro()

--- a/KOKKOS.cmake
+++ b/KOKKOS.cmake
@ -12,6 +12,9 @@ register_flag_required(KOKKOS_IN_TREE

         See https://github.com/kokkos/kokkos/blob/master/BUILD.md for all available options")

+# compiler vendor and arch specific flags
+set(KOKKOS_FLAGS_CPU_INTEL -qopt-streaming-stores=always)
+
 macro(setup)

    cmake_policy(SET CMP0074 NEW) #see https://github.com/kokkos/kokkos/blob/master/BUILD.md
@ -25,6 +28,12 @@ macro(setup)
        message(FATAL_ERROR "`${KOKKOS_IN_TREE}` does not exist")
    endif ()

+    register_append_compiler_and_arch_specific_cxx_flags(
+            KOKKOS_FLAGS_CPU
+            ${CMAKE_CXX_COMPILER_ID}
+            ${CMAKE_SYSTEM_PROCESSOR}
+    )
+
 endmacro()


--- a/OMP.cmake
+++ b/OMP.cmake
@ -32,21 +32,51 @@
 #    NVHPC = NVIDIA HPC SDK Compiler (nvidia.com)


+# CMAKE_SYSTEM_PROCESSOR is set via `uname -p`, we have:
+# Power9 = ppc64le
+# x64    = x86_64
+# arm64  = aarch64
+#
+
+
 #predefined offload flags based on compiler id
-set(OMP_FLAGS_INTEL_CPU -qopt-streaming-stores=always)
-set(OMP_FLAGS_OFFLOAD_INTEL -qnextgen -fiopenmp -fopenmp-targets=spir64)
-set(OMP_FLAGS_OFFLOAD_GNU_NVIDIA -foffload=nvptx-none)
-set(OMP_FLAGS_OFFLOAD_GNU_AMD -foffload=amdgcn-amdhsa)
+
+
+set(OMP_FLAGS_OFFLOAD_INTEL
+        -qnextgen -fiopenmp -fopenmp-targets=spir64)
+set(OMP_FLAGS_OFFLOAD_GNU_NVIDIA
+        -foffload=nvptx-none)
+set(OMP_FLAGS_OFFLOAD_GNU_AMD
+        -foffload=amdgcn-amdhsa)
 set(OMP_FLAGS_OFFLOAD_CLANG_NVIDIA
        -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target=nvptx64-nvidia-cuda)
 set(OMP_FLAGS_OFFLOAD_CLANG_AMD
        -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target=amdgcn-amd-amdhsa)
-set(OMP_FLAGS_OFFLOAD_CLANG_ARCH_FLAG -march=)
+set(OMP_FLAGS_OFFLOAD_CLANG_ARCH_FLAG
+        -march=) # prefix only, arch appended by the vendor:arch tuple
+
+
+set(OMP_FLAGS_CPU_INTEL
+        -qopt-streaming-stores=always)
+set(OMP_FLAGS_CPU_GNU_PPC64LE
+        -mcpu=native)
+set(OMP_FLAGS_CPU_XL
+        -O5 -qarch=auto -qtune=auto)
+
+# NEC
+set(OMP_FLAGS_CPU_NEC -O4 -finline)

 register_flag_optional(CMAKE_CXX_COMPILER
        "Any CXX compiler that supports OpenMP as per CMake detection (and offloading if enabled with `OFFLOAD`)"
        "c++")

+register_flag_optional(ARCH
+        "This overrides CMake's CMAKE_SYSTEM_PROCESSOR detection which uses (uname -p), this is mainly for use with
+         specialised accelerators only and not to be confused with offload which is is mutually exclusive with this.
+         Supported values are:
+          - NEC"
+        "")
+
 register_flag_optional(OFFLOAD
        "Whether to use OpenMP offload, the format is <VENDOR:ARCH?>|ON|OFF.
        We support a small set of known offload flags for clang, gcc, and icpx.
@ -75,10 +105,25 @@ macro(setup)
    register_link_library(OpenMP::OpenMP_CXX)

    string(TOUPPER ${CMAKE_CXX_COMPILER_ID} COMPILER)
+    if(NOT ARCH)
+        string(TOUPPER ${CMAKE_SYSTEM_PROCESSOR} ARCH)
+    else()
+        message(STATUS "Using custom arch: ${ARCH}")
+    endif()
+
+

    if (("${OFFLOAD}" STREQUAL OFF) OR (NOT DEFINED OFFLOAD))
        # no offload

+        # resolve the CPU specific flags
+        # starting with ${COMPILER_VENDOR}_${PLATFORM_ARCH}, then try ${COMPILER_VENDOR}, and then give up
+        register_append_compiler_and_arch_specific_cxx_flags(
+                OMP_FLAGS_CPU
+                ${COMPILER}
+                ${ARCH}
+        )
+
    elseif ("${OFFLOAD}" STREQUAL ON)
        #  offload but with custom flags
        register_definitions(OMP_TARGET_GPU)
--- a/RAJA.cmake
+++ b/RAJA.cmake
@ -30,6 +30,8 @@ register_flag_optional(CUDA_EXTRA_FLAGS
        "[TARGET==NVIDIA only] Additional CUDA flags passed to nvcc, this is appended after `CUDA_ARCH`"
        "")

+# compiler vendor and arch specific flags
+set(RAJA_FLAGS_CPU_INTEL -qopt-streaming-stores=always)

 macro(setup)

@ -56,9 +58,15 @@ macro(setup)
        set(ENABLE_CUDA ${ENABLE_CUDA} CACHE BOOL "" FORCE)

        if (ENABLE_CUDA)
+
+            # XXX CMake 3.18 supports CMAKE_CUDA_ARCHITECTURES/CUDA_ARCHITECTURES but we support older CMakes
+            if(POLICY CMP0104)
+                cmake_policy(SET CMP0104 OLD)
+            endif()
+
            # RAJA needs all the cuda stuff setup before including!
            set(CMAKE_CUDA_COMPILER ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc)
-            set(CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS} "-extended-lambda -arch=${CUDA_ARCH}" ${CUDA_EXTRA_FLAGS})
+            set(CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS} "-forward-unknown-to-host-compiler -extended-lambda -arch=${CUDA_ARCH}" ${CUDA_EXTRA_FLAGS})
            list(APPEND CMAKE_CUDA_FLAGS)

            message(STATUS "NVCC flags: ${CMAKE_CUDA_FLAGS}")
@ -81,4 +89,10 @@ macro(setup)
    endif ()


+    register_append_compiler_and_arch_specific_cxx_flags(
+            RAJA_FLAGS_CPU
+            ${CMAKE_CXX_COMPILER_ID}
+            ${CMAKE_SYSTEM_PROCESSOR}
+    )
+
 endmacro()
--- a/README.md
+++ b/README.md
@ -59,16 +59,19 @@ Drivers, compiler and software applicable to whichever implementation you would

 ### CMake

-The project supports building with CMake >= 3.13.0.
+The project supports building with CMake >= 3.13.0, it can be installed without root via the [official script](https://cmake.org/download/).
 As with any CMake project, first configure the project:

 ```shell
 > cd babelstream
-> cmake -Bbuild -H. -DMODEL=<model> <model_specific_flags...> # configure the build, build type defaults to Release 
+> cmake -Bbuild -H. -DMODEL=<model> <model specific flags prefixed with -D...> # configure the build, build type defaults to Release 
 > cmake --build build # compile it 
 > ./build/babelstream # executable available at ./build/
 ```

+By default, we have defined a set of optimal flags for known HPC compilers and assigned those to `RELEASE_FLAG`.
+You can override this if required.
+
 To find out what flag each model supports or requires, simply configure while only specifying the model.
 For example:
 ```shell
--- a/SYCL.cmake
+++ b/SYCL.cmake
@ -46,7 +46,6 @@ macro(setup)

        list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/Modules)
        set(ComputeCpp_DIR ${SYCL_COMPILER_DIR})
-        set(COMPUTECPP_USER_FLAGS -O3 -no-serial-memop)

        # don't point to the CL dir as the imports already have the CL prefix
        set(OpenCL_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/CL")
@ -55,6 +54,9 @@ macro(setup)
        # ComputeCpp needs OpenCL
        find_package(ComputeCpp REQUIRED)

+        # this must come after FindComputeCpp (!)
+        set(COMPUTECPP_USER_FLAGS -O3 -no-serial-memop)
+
    elseif (${SYCL_COMPILER} STREQUAL "DPCPP")
        set(CMAKE_CXX_COMPILER ${SYCL_COMPILER_DIR}/bin/clang++)
        include_directories(${SYCL_COMPILER_DIR}/include/sycl)
--- a/ci-prepare-bionic.sh
+++ b/ci-prepare-bionic.sh
@ -76,6 +76,17 @@ get_and_install_deb() {

 }

+get() {
+  local name="$1"
+  local pkg_url="$2"
+  if [ "$SETUP" = true ]; then
+    if [ ! -f "$name" ] || [ "$FORCE_DOWNLOAD" = true ]; then
+      echo "$name not found, downloading..."
+      wget -q --show-progress --progress=bar:force:noscroll "$pkg_url" -O "$name"
+    fi
+  fi
+}
+
 get_and_untar() {
  local name="$1"
  local pkg_url="$2"
@ -306,9 +317,37 @@ else
  echo "Running locally, defaulting to standard export"
 fi

+setup_cmake() {
+
+  echo "Preparing CMake"
+
+  local cmake_release="https://github.com/Kitware/CMake/releases/download"
+
+  get "cmake-3.13.sh" "$cmake_release/v3.13.4/cmake-3.13.4-Linux-x86_64.sh"
+  chmod +x "./cmake-3.13.sh" && sh "./cmake-3.13.sh" --skip-license --include-subdir
+  export_var CMAKE_3_13_BIN "$PWD/cmake-3.13.4-Linux-x86_64/bin/cmake"
+  verify_bin_exists "$CMAKE_3_13_BIN"
+  "$CMAKE_3_13_BIN" --version
+
+  get "cmake-3.15.sh" "$cmake_release/v3.15.7/cmake-3.15.7-Linux-x86_64.sh"
+  chmod +x "./cmake-3.15.sh" && "./cmake-3.15.sh" --skip-license --include-subdir
+  export_var CMAKE_3_15_BIN "$PWD/cmake-3.15.7-Linux-x86_64/bin/cmake"
+  verify_bin_exists "$CMAKE_3_15_BIN"
+  "$CMAKE_3_15_BIN" --version
+
+  get "cmake-3.18.sh" "$cmake_release/v3.18.6/cmake-3.18.6-Linux-x86_64.sh"
+  chmod +x "./cmake-3.18.sh" && "./cmake-3.18.sh" --skip-license --include-subdir
+  export_var CMAKE_3_18_BIN "$PWD/cmake-3.18.6-Linux-x86_64/bin/cmake"
+  verify_bin_exists "$CMAKE_3_18_BIN"
+  "$CMAKE_3_18_BIN" --version
+
+  check_size
+
+}

 if [ "$PARALLEL" = true ]; then
  (setup_clang_gcc && setup_rocm && setup_hipsycl) &   # these need apt so run sequentially
+  setup_cmake &
  setup_oclcpu &
  setup_aocc &
  setup_nvhpc &
@ -318,6 +357,7 @@ if [ "$PARALLEL" = true ]; then
  setup_raja &
  wait
 else
+  setup_cmake
  setup_aocc
  setup_oclcpu
  setup_nvhpc
--- a/ci-test-compile.sh
+++ b/ci-test-compile.sh
@ -8,6 +8,7 @@ export CCACHE_DISABLE=1
 BUILD_DIR=${1:-build}
 COMPILER=${2:-all}
 MODEL=${3:-all}
+CMAKE_BIN=${4}

 LOG_DIR="$BUILD_DIR"

@ -45,13 +46,13 @@ run_build() {
  set +e

  # shellcheck disable=SC2086
-  cmake -B"$build" -H. \
+  "$CMAKE_BIN" -B"$build" -H. \
    -DCMAKE_BUILD_TYPE=Release \
    -DCMAKE_VERBOSE_MAKEFILE=ON \
    -DMODEL="$model" $flags &>>"$log"
  local cmake_code=$?

-  cmake --build "$build" --target babelstream -j "$(nproc)" &>>"$log"
+  "$CMAKE_BIN" --build "$build" --target babelstream -j "$(nproc)" &>>"$log"
  local cmake_code=$?
  set -e

@ -59,17 +60,19 @@ run_build() {
  if [[ -f "$bin" ]]; then
    echo "$(tput setaf 2)[PASS!]($model->$build)$(tput sgr0): -DMODEL=$model $flags"
    # shellcheck disable=SC2002
+    cat "$log" | sed '/^--/d' | grep -i "/bin/nvcc" | sed 's/^/    /'
    cat "$log" | sed '/^--/d' | grep -i "$grep_kw" | sed 's/^/    /'
+    cat "$log" | sed '/^--/d' | grep -i "warning" | sed "s/.*/    $(tput setaf 3)&$(tput sgr0)/"
  else
    echo "$(tput setaf 1)[FAIL!]($model->$build)$(tput sgr0): -DMODEL=$model $flags"
-    echo "      CMake exited with code $cmake_code, see full build log at $log, reproduced below:"
+    echo "      $(tput setaf 1)CMake exited with code $cmake_code, see full build log at $log, reproduced below:$(tput sgr0)"
    cat "$log"
    exit 1
  fi
  echo "    $(tput setaf 4)$(file "$bin")$(tput sgr0)"
 }

-##
+###
 #KOKKOS_SRC="/home/tom/Downloads/kokkos-3.3.00"
 #RAJA_SRC="/home/tom/Downloads/RAJA-v0.13.0"
 #
@ -82,10 +85,6 @@ run_build() {
 #NVHPC_CUDA_DIR="$NVSDK/cuda/11.2"
 #"$NVSDK/compilers/bin/makelocalrc" "$NVSDK/compilers/bin/" -x
 #
-##NVHPC_NVCXX="/opt/nvidia/hpc_sdk/Linux_x86_64/21.1/compilers/bin/nvc++"
-##NVHPC_NVCC="/opt/nvidia/hpc_sdk/Linux_x86_64/21.1/cuda/11.2/bin/nvcc"
-##NVHPC_CUDA_DIR="/opt/nvidia/hpc_sdk/Linux_x86_64/21.1/cuda/11.2"
-#
 #AOCC_CXX="/opt/AMD/aocc-compiler-2.3.0/bin/clang++"
 #AOMP_CXX="/usr/lib/aomp/bin/clang++"
 #OCL_LIB="/home/tom/Downloads/oclcpuexp-2020.11.11.0.04_rel/x64/libOpenCL.so"
@ -98,13 +97,15 @@ run_build() {
 #HIPSYCL_DIR="/opt/hipsycl/cff515c/"
 #
 #ICPX_CXX="/opt/intel/oneapi/compiler/2021.1.2/linux/bin/icpx"
+#ICPC_CXX="/opt/intel/oneapi/compiler/2021.1.2/linux/bin/intel64/icpc"
 #
-#GCC_STD_PAR_LIB=""
-#CLANG_STD_PAR_LIB=""
+#GCC_STD_PAR_LIB="tbb"
+#CLANG_STD_PAR_LIB="tbb"
 #GCC_OMP_OFFLOAD_AMD=false
 #GCC_OMP_OFFLOAD_NVIDIA=true
 #CLANG_OMP_OFFLOAD_AMD=false
 #CLANG_OMP_OFFLOAD_NVIDIA=false
+###

 AMD_ARCH="gfx_903"
 NV_ARCH="sm_70"
@ -196,10 +197,26 @@ build_hip() {
 }

 build_icpx() {
-  source /opt/intel/oneapi/setvars.sh -force
+  # clang derived
+  set +u
+  source /opt/intel/oneapi/setvars.sh -force || true
+  set -u
  run_build intel_build "${ICPX_CXX:?}" OMP "-DCMAKE_CXX_COMPILER=${ICPX_CXX:?} -DOFFLOAD=INTEL"
 }

+build_icpc() {
+  # icc/icpc
+  set +u
+  source /opt/intel/oneapi/setvars.sh -force || true
+  set -u
+  local name="intel_build"
+  local cxx="-DCMAKE_CXX_COMPILER=${ICPC_CXX:?}"
+  run_build $name "${ICPC_CXX:?}" OMP "$cxx"
+  run_build $name "${ICPC_CXX:?}" OCL "$cxx -DOpenCL_LIBRARY=${OCL_LIB:?}"
+  run_build $name "${ICPC_CXX:?}" RAJA "$cxx -DRAJA_IN_TREE=${RAJA_SRC:?}"
+  run_build $name "${ICPC_CXX:?}" KOKKOS "$cxx -DKOKKOS_IN_TREE=${KOKKOS_SRC:?} -DKokkos_ENABLE_OPENMP=ON"
+}
+
 build_computecpp() {
  run_build computecpp_build "compute++" SYCL "-DCMAKE_CXX_COMPILER=${GCC_CXX:?} \
  -DSYCL_COMPILER=COMPUTECPP \
@ -223,9 +240,8 @@ build_hipsycl() {
  -DSYCL_COMPILER_DIR=${HIPSYCL_DIR:?}"
 }

-# TODO tested locally but can't install compilers for these two remotely without registration/license:
-# build_icpx
-# build_computecpp
+echo "Test compiling with ${COMPILER} CXX for ${MODEL} model"
+"$CMAKE_BIN" --version

 case "$COMPILER" in
 gcc) build_gcc ;;
@ -236,6 +252,12 @@ aomp) build_aomp ;;
 hip) build_hip ;;
 dpcpp) build_dpcpp ;;
 hipsycl) build_hipsycl ;;
+
+# XXX below are local only; licence or very large download required, candidate for local runner
+computecpp) build_computecpp ;;
+icpx) build_icpx ;;
+icpc) build_icpc ;;
+
 all)
  build_gcc
  build_clang
@ -245,6 +267,11 @@ all)
  build_hip
  build_dpcpp
  build_hipsycl
+
+  build_computecpp
+  build_icpx
+  build_icpc
+
  ;;
 *)
  echo "Unknown $COMPILER, use ALL to compile with all supported compilers"
--- a/register_models.cmake
+++ b/register_models.cmake
@ -19,15 +19,19 @@
 #endfunction()
 #

+macro(wipe_gcc_style_optimisation_flags VAR)
+    string(REGEX REPLACE "([\\/\\-]O.)" "" ${VAR} ${${VAR}})
+endmacro()
+
 macro(register_link_library)
    list(APPEND LINK_LIBRARIES ${ARGN})
 endmacro()

 macro(register_append_cxx_flags CONFIG)
    if ("${CONFIG}" STREQUAL "RELEASE" OR "${CONFIG}" STREQUAL "ANY")
-        list(APPEND RELEASE_FLAGS ${ARGN})
+        list(APPEND DEFAULT_RELEASE_FLAGS ${ARGN})
    elseif ("${CONFIG}" STREQUAL "DEBUG" OR "${CONFIG}" STREQUAL "ANY")
-        list(APPEND DEBUG_FLAGS ${ARGN})
+        list(APPEND DEFAULT_DEBUG_FLAGS ${ARGN})
    else ()
        message(FATAL_ERROR "register_flags supports only RELEASE, DEBUG, or ANY for all configs, got `${CONFIG}`")
    endif ()
@ -37,6 +41,19 @@ macro(register_append_link_flags)
    list(APPEND LINK_FLAGS ${ARGN})
 endmacro()

+macro(register_append_compiler_and_arch_specific_cxx_flags PREFIX CXX ARCH)
+    string(TOUPPER ${CXX} _CXX)
+    string(TOUPPER ${ARCH} _ARCH)
+    set(_CXX_ARCH_SPECIFIC_FLAGS "${${PREFIX}_${_CXX}_${_ARCH}}")
+    if (_CXX_ARCH_SPECIFIC_FLAGS)
+        register_append_cxx_flags(ANY ${_CXX_ARCH_SPECIFIC_FLAGS})
+    endif ()
+    set(_CXX_ARCH_SPECIFIC_FLAGS "${${PREFIX}_${_CXX}}")
+    if (_CXX_ARCH_SPECIFIC_FLAGS)
+        register_append_cxx_flags(ANY ${_CXX_ARCH_SPECIFIC_FLAGS})
+    endif ()
+endmacro()
+
 macro(register_definitions)
    list(APPEND IMPL_DEFINITIONS ${ARGN})
 endmacro()
@ -100,25 +117,12 @@ function(registered_flags_action ACTION OUT)
 endfunction()


-
-
 macro(register_model NAME PREPROCESSOR_NAME)
    string(TOUPPER ${NAME} MODEL_UPPER)
    list(APPEND REGISTERED_MODELS "${NAME}")

    list(APPEND IMPL_${MODEL_UPPER}_SOURCES "${ARGN}")
    list(APPEND IMPL_${MODEL_UPPER}_DEFINITIONS "${PREPROCESSOR_NAME}")
-
-#    if ("${MODEL_UPPER}" STREQUAL ${MODEL})
-#
-#        set(MODEL_FILE ${CMAKE_CURRENT_SOURCE_DIR}/${NAME}.cmake)
-#
-#        if (NOT EXISTS ${MODEL_FILE})
-#            message(FATAL_ERROR "${MODEL_FILE} not found, perhaps it needs to be implemented?")
-#        endif ()
-#        include(${MODEL_FILE})
-#
-#    endif ()
 endmacro()