cmake cleanup

CMakeLists.txt

@@ -1,17 +1,17 @@
-cmake_minimum_required(VERSION 3.17 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.24 FATAL_ERROR)
 
 project(cosma
   DESCRIPTION "Communication Optimal Matrix Multiplication"
   HOMEPAGE_URL "https://github.com/eth-cscs/COSMA"
   VERSION 2.6.6
   LANGUAGES CXX)
 
+include(FetchContent)
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules")
 include(cmake/build_type.cmake)
 include(cmake/adjust_mpiexec_flags.cmake)
-include(GitSubmodule)
 set(CMAKE_EXPORT_COMPILE_COMMANDS "YES") # always write compile_commands.json
 
 # Options
@@ -41,31 +41,37 @@ set_property(CACHE COSMA_BLAS PROPERTY STRINGS ${COSMA_BLAS_LIST})
 # implementation. We have to sort out what we should find for the FindBLAS and
 # GPU supports since they are treated as separate components
 
+if(COSMA_BLAS STREQUAL "OFF")
+    message(FATAL_ERROR "A Blas implementation is needed when running on CPU only: choices are : auto, MKL, OPENBLAS, CRAY_LIBSCI, NVPL, CUSTOM, BLIS, ATLAS, FLEXIBLAS, ARMPL, GenericBLAS, CUDA or ROCM")
+endif()
+
 if (COSMA_BLAS MATCHES "CUDA|ROCM")
   set(COSMA_GPU_BACKEND ${COSMA_BLAS})
 else()
-  if(COSMA_BLAS STREQUAL "OFF")
-    message(FATAL_ERROR "A Blas implementation is needed when running on CPU only: choices are : auto, MKL, OPENBLAS, CRAY_LIBSCI, NVPL, CUSTOM, BLIS, ATLAS, FLEXIBLAS, ARMPL, GenericBLAS")
-  else()
-    set(COSMA_BLAS_VENDOR ${COSMA_BLAS})
-  endif()
+  set(COSMA_BLAS_VENDOR ${COSMA_BLAS})
 endif()
 
-if (COSMA_WITH_NCCL AND NOT COSMA_GPU_BACKEND IN_LIST COSMA_GPU_BACKENDS_LIST)
-  message(FATAL_ERROR "NCCL can only be used with the GPU backend.")
+if ((COSMA_WITH_NCCL OR COSMA_WITH_RCCL) AND NOT COSMA_GPU_BACKEND IN_LIST COSMA_GPU_BACKENDS_LIST)
+  message(FATAL_ERROR "NCCL (RCCL) can only be used with the GPU backend set to CUDA (ROCM).")
 endif()
 
 if (COSMA_WITH_GPU_AWARE_MPI AND NOT COSMA_GPU_BACKEND IN_LIST COSMA_GPU_BACKENDS_LIST)
-  message(FATAL_ERROR "GPU-aware MPI can only be used with the GPU backend.")
+  message(FATAL_ERROR "GPU-aware MPI can only be used with the GPU backend set to CUDA or ROCM.")
 endif()
 
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE "Release")
 endif()
 
+# Dependencies
+# MPI
+set(MPI_DETERMINE_LIBRARY_VERSION TRUE)
+find_package(MPI COMPONENTS CXX REQUIRED)
+adjust_mpiexec_flags()
 
 # check if scalapack backend is valid
 message(STATUS "Selected SCALAPACK backend for COSMA: ${COSMA_SCALAPACK}")
+
 if(NOT COSMA_SCALAPACK IN_LIST COSMA_SCALAPACK_LIST)
   message(FATAL_ERROR "Invalid value for COSMA_SCALAPACK!")
 endif()
@@ -79,53 +85,42 @@ if (NOT COSMA_SCALAPACK MATCHES "OFF")
   endif()
 endif()
 
-if (NOT COSMA_BLAS_VENDOR MATCHES "OFF")
-  find_package(Blas REQUIRED)
-endif()
 
-# preserve rpaths when installing and make the install folder relocatable
-# use `CMAKE_SKIP_INSTALL_RPATH` to skip this
-# https://spack.readthedocs.io/en/latest/workflows.html#write-the-cmake-build
-list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES
-  "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}" isSystemDir)
-# skip RPATH if COSMA is installed to system directories
-if(isSystemDir STREQUAL "-1")
-  set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
-  if(APPLE)
-    set(basePoint @loader_path)
-  else()
-    set(basePoint $ORIGIN)
-  endif()
-  file(RELATIVE_PATH relDir ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}
-    ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR})
-  set(CMAKE_INSTALL_RPATH ${basePoint} ${basePoint}/${relDir})
+if (NOT COSMA_BLAS_VENDOR MATCHES "OFF|CUDA|ROCM")
+  find_package(Blas REQUIRED)
 endif()
 
-# Dependencies
-# MPI
-set(MPI_DETERMINE_LIBRARY_VERSION TRUE)
-find_package(MPI COMPONENTS CXX REQUIRED)
-adjust_mpiexec_flags()
-
 if (NOT COSMA_SCALAPACK MATCHES "OFF")
   find_package(SCALAPACK REQUIRED)
-endif()
+endif ()
 
-if (COSMA_WITH_PROFILING)
-  find_package(semiprof REQUIRED)
+set(COSTA_WITH_PROFILING ${COSMA_WITH_PROFILING} CACHE INTERNAL "")
+set(COSTA_SCALAPACK ${COSMA_SCALAPACK} CACHE INTERNAL "")
+
+FetchContent_Declare(
+  costa
+  GIT_REPOSITORY https://github.com/eth-cscs/costa.git
+  GIT_TAG        03847e66f05ad4a1eb371b85be628e218ce46f11 # v2.2.3
+  FIND_PACKAGE_ARGS NAMES costa
+)
+# the joy of fetch_content. if we build costa and cosma together
+# fetch_content will pick up the FindSCALAPACK from cosma NOT costa.
+if (NOT TARGET costa::scalapack::scalapack AND NOT COSMA_SCALAPACK MATCHES "OFF")
+  add_library(costa::scalapack::scalapack ALIAS cosma::scalapack::scalapack)
 endif ()
 
-if (COSMA_WITH_TESTS OR COSMA_WITH_APPS)
-  add_git_submodule_or_find_external(cxxopts libs/cxxopts)
-endif()
+FetchContent_MakeAvailable(costa)
 
 # these are only GPU-backends
 if (COSMA_GPU_BACKEND MATCHES "CUDA|ROCM")
-  set(TILEDMM_GPU_BACKEND ${COSMA_GPU_BACKEND} CACHE STRING "GPU backend" FORCE)
-  add_git_submodule_or_find_external(Tiled-MM libs/Tiled-MM)
-  if (NOT TARGET Tiled-MM::Tiled-MM AND TARGET Tiled-MM)
-    add_library(Tiled-MM::Tiled-MM ALIAS Tiled-MM)
-  endif()
+  set(TILEDMM_GPU_BACKEND ${COSMA_GPU_BACKEND} CACHE INTERNAL "")
+  FetchContent_Declare(
+    Tiled-MM
+    GIT_REPOSITORY https://github.com/eth-cscs/Tiled-MM.git
+    GIT_TAG      0eb75179e670a04c649b50ae5e91bb71b43e4d06 # v2.3.2
+    FIND_PACKAGE_ARGS NAMES tiled-MM
+  )
+  FetchContent_MakeAvailable(Tiled-MM)
 
   if (COSMA_WITH_NCCL)
     find_package(CUDAToolkit REQUIRED)
@@ -134,26 +129,51 @@ if (COSMA_GPU_BACKEND MATCHES "CUDA|ROCM")
     find_package(hip REQUIRED)
     find_package(rccl REQUIRED)
   endif()
-endif()
 
-set(COSTA_WITH_PROFILING ${COSMA_WITH_PROFILING} CACHE STRING "" FORCE)
-set(COSTA_SCALAPACK ${COSMA_SCALAPACK} CACHE STRING "" FORCE)
+  if (NOT TARGET Tiled-MM::Tiled-MM)
+      message("Tiled-mm target not found")
+  endif ()
 
-if (TARGET cosma::scalapack::scalapack AND NOT COSMA_SCALAPACK MATCHES "OFF")
-  add_library(costa::scalapack::scalapack ALIAS cosma::scalapack::scalapack)
 endif()
 
-add_git_submodule_or_find_external(costa libs/COSTA)
-
-# alias targets for add_subdirectory dependency
+if (COSMA_WITH_PROFILING)
+  FetchContent_Declare(
+    semiprof
+    GIT_REPOSITORY  https://github.com/bcumming/semiprof.git
+    GIT_TAG         f132142ff2215dfa073e416fa7911d8877d62752
+    FIND_PACKAGE_ARGS NAMES semiprof)
+  FetchContent_MakeAvailable(semiprof)
+endif ()
 
-if (NOT TARGET costa::costa)
-  add_library(costa::costa ALIAS costa)
+if (COSMA_WITH_TESTS OR COSMA_WITH_APPS)
+  FetchContent_Declare(
+    cxxopts
+    GIT_REPOSITORY https://github.com/jarro2783/cxxopts.git
+    GIT_TAG        4bf61f08697b110d9e3991864650a405b3dd515d # v3.2.1
+    FIND_PACKAGE_ARGS NAMES cxxopts
+  )
+  FetchContent_MakeAvailable(cxxopts)
 endif()
 
-if (TARGET costa_prefixed_scalapack AND (NOT TARGET costa::costa_prefixed_scalapack))
-  add_library(costa::costa_prefixed_scalapack ALIAS costa_prefixed_scalapack)
-  add_library(costa::costa_scalapack ALIAS costa_scalapack)
+if (NOT TARGET Tiled-MM::Tiled-MM)
+      message("Tiled-mm target not found")
+endif ()
+# preserve rpaths when installing and make the install folder relocatable
+# use `CMAKE_SKIP_INSTALL_RPATH` to skip this
+# https://spack.readthedocs.io/en/latest/workflows.html#write-the-cmake-build
+list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES
+  "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}" isSystemDir)
+# skip RPATH if COSMA is installed to system directories
+if(isSystemDir STREQUAL "-1")
+  set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+  if(APPLE)
+    set(basePoint @loader_path)
+  else()
+    set(basePoint $ORIGIN)
+  endif()
+  file(RELATIVE_PATH relDir ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}
+    ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR})
+  set(CMAKE_INSTALL_RPATH ${basePoint} ${basePoint}/${relDir})
 endif()
 
 # COSMA

ci/cscs.yml

@@ -15,7 +15,7 @@ build base image:
     WATCH_FILECHANGES: ci/baseimage.cuda.Dockerfile
     PERSIST_IMAGE_NAME: $CSCS_REGISTRY_PATH/base/cosma-ci
 
-build tiled-mm:
+build cosma:
   extends: .container-builder-cscs-gh200
   needs: ["build base image"]
   stage: build
@@ -27,7 +27,7 @@ build tiled-mm:
 
 .run_tests:
   extends: [.container-runner-daint-gh200]
-  needs: ["build tiled-mm"]
+  needs: ["build cosma"]
   stage: test
   image: $CSCS_REGISTRY_PATH/cosma/cosma-ci:$CI_COMMIT_SHA
   variables:

miniapp/CMakeLists.txt

@@ -6,7 +6,6 @@ set(executables "layout_miniapp" "cosma_miniapp" "cosma_statistics")
 foreach(exec ${executables})
     add_executable(${exec} "${exec}.cpp")
     target_link_libraries(${exec} PRIVATE cosma cxxopts::cxxopts)
-    target_include_directories(${exec} PUBLIC "${cosma_SOURCE_DIR}/libs/cxxopts/include/")
     install(TARGETS ${exec} RUNTIME DESTINATION "${CMAKE_INSTALL_PREFIX}/bin")
 endforeach()
 
@@ -15,7 +14,6 @@ if(NOT COSMA_SCALAPACK MATCHES "OFF")
     foreach(exec ${scalapack_executables})
         add_executable(${exec} "${exec}.cpp")
         target_link_libraries(${exec} PRIVATE cosma_pxgemm_cpp cxxopts::cxxopts)
-        target_include_directories(${exec} PUBLIC "${cosma_SOURCE_DIR}/libs/cxxopts/include/")
         install(TARGETS ${exec} RUNTIME DESTINATION "${CMAKE_INSTALL_PREFIX}/bin")
         if (COSMA_WITH_PROFILING)
             target_link_libraries(${exec} PRIVATE semiprof::semiprof)

src/cosma/CMakeLists.txt

@@ -39,84 +39,62 @@ target_link_libraries(cosma PUBLIC
   MPI::MPI_CXX
   costa::costa
   $<TARGET_NAME_IF_EXISTS:roc::rccl>
-  $<TARGET_NAME_IF_EXISTS:cosma::nccl>)
-
-if (NOT COSMA_BLAS_VENDOR MATCHES "OFF")
-  target_link_libraries(cosma PUBLIC cosma::BLAS::blas)
-endif()
-
-if (COSMA_GPU_BACKEND STREQUAL "OFF")
-  if ((NOT COSMA_BLAS_VENDOR MATCHES "MKL") AND (NOT COSMA_BLAS_VENDOR MATCHES "BLIS"))
-    target_compile_definitions(cosma PUBLIC COSMA_WITH_BLAS)
-  endif()
-
-  if (COSMA_GPU_BACKEND STREQUAL "OFF")
-    target_compile_definitions(cosma PUBLIC
-      $<$<STREQUAL:${COSMA_BLAS_VENDOR},MKL>:COSMA_WITH_MKL_BLAS>
-      $<$<STREQUAL:${COSMA_BLAS_VENDOR},BLIS>:COSMA_WITH_BLIS_BLAS>)
-  endif()
-else()
-  target_compile_definitions(cosma PUBLIC COSMA_HAVE_GPU)
-  target_link_libraries(cosma PUBLIC Tiled-MM::Tiled-MM)
-endif()
+  $<TARGET_NAME_IF_EXISTS:cosma::nccl>
+  $<$<NOT:$<STREQUAL:${COSMA_BLAS_VENDOR},"OFF">>:cosma::BLAS::blas>
+  $<TARGET_NAME_IF_EXISTS:Tiled-MM::Tiled-MM>
+  $<$<STREQUAL:${COSMA_GPU_BACKEND},CUDA>:Tiled-MM::Tiled-MM>
+  $<$<STREQUAL:${COSMA_GPU_BACKEND},ROCM>:Tiled-MM::Tiled-MM>
+  $<$<BOOL:${COSMA_WITH_PROFILING}>:semiprof::semiprof>
+  $<$<BOOL:${COSMA_SCALAPACK}>:cosma::scalapack::scalapack>)
 
 target_compile_definitions(cosma PUBLIC
-  $<$<BOOL:${COSMA_WITH_NCCL}>:COSMA_WITH_NCCL>
-  $<$<STREQUAL:${COSMA_GPU_BACKEND},"ROCM">:__HIP_PLATFORM_HCC__>
-  $<$<BOOL:${COSMA_WITH_GPU_AWARE_MPI}>:COSMA_WITH_GPU_AWARE_MPI>)
-
-target_compile_definitions(cosma PUBLIC
-  $<$<BOOL:${COSMA_WITH_NCCL}>:COSMA_WITH_NCCL>
-  $<$<BOOL:${COSMA_WITH_RCCL}>:COSMA_WITH_NCCL>
-)
-
-target_compile_definitions(cosma PRIVATE $<$<BOOL:${COSMA_WITH_PROFILING}>:COSMA_WITH_PROFILING>)
-
-if(COSMA_WITH_PROFILING)
-  target_link_libraries(cosma PRIVATE semiprof::semiprof)
-endif()
+                              $<$<BOOL:${COSMA_WITH_NCCL}>:COSMA_WITH_NCCL>
+                              $<$<STREQUAL:${COSMA_GPU_BACKEND},"ROCM">:__HIP_PLATFORM_HCC__>
+                              $<$<BOOL:${COSMA_WITH_GPU_AWARE_MPI}>:COSMA_WITH_GPU_AWARE_MPI>
+                              $<$<BOOL:${COSMA_WITH_NCCL}>:COSMA_WITH_NCCL>
+                              $<$<BOOL:${COSMA_WITH_RCCL}>:COSMA_WITH_NCCL>
+                              $<$<STREQUAL:${COSMA_BLAS_VENDOR},MKL>:COSMA_WITH_MKL_BLAS>
+                              $<$<STREQUAL:${COSMA_BLAS_VENDOR},BLIS>:COSMA_WITH_BLIS_BLAS>
+                              $<$<NOT:$<IN_LIST:${COSMA_BLAS_VENDOR},"MKL;BLIS">>:COSMA_WITH_BLAS>
+                              $<$<STREQUAL:${COSMA_GPU_BACKEND},CUDA>:COSMA_HAVE_GPU>
+                              $<$<STREQUAL:${COSMA_GPU_BACKEND},ROCM>:COSMA_HAVE_GPU>
+                              PRIVATE
+                              $<$<BOOL:${COSMA_WITH_PROFILING}>:COSMA_WITH_PROFILING>)
 
 list(APPEND INSTALLED_TARGETS_LIST "cosma")
 
 # if SCALAPACK is found and cosma_pxgemm library is not already created
 # then create it here and link it to the profiler if needed
 # build as a shared library is necessary here because of the function interposing
 if(COSMA_SCALAPACK)
-  target_link_libraries(cosma PUBLIC cosma::scalapack::scalapack)
   if (NOT TARGET cosma_pxgemm AND BUILD_SHARED_LIBS)
     add_library(cosma_pxgemm scalapack.cpp
       pxgemm_params.hpp
       cosma_pxgemm.cpp
       pxgemm.cpp
     )
 
-    target_link_libraries(cosma_pxgemm PUBLIC cosma)
+    target_link_libraries(cosma_pxgemm PUBLIC cosma
+    $<$<BOOL:${COSMA_WITH_PROFILING}>:semiprof::semiprof>)
 
-    if(COSMA_WITH_PROFILING)
-      target_link_libraries(cosma_pxgemm PRIVATE semiprof::semiprof)
-      target_compile_definitions(cosma_pxgemm PRIVATE COSMA_WITH_PROFILING)
-    endif()
+    target_compile_definitions(cosma_pxgemm PRIVATE $<$<BOOL:${COSMA_WITH_PROFILING}>:COSMA_WITH_PROFILING>)
     list(APPEND INSTALLED_TARGETS_LIST "cosma_pxgemm")
   endif()
 
   # this is a library exposing the prefixed scalapack API (with cosma/COSMA prefix)
   # it is aimed for users who don't want to overwrite the available scalapack API with cosma.
   # if SCALAPACK is found and cosma_prefixed_pxgemm library is not already created
   # then create it here and link it to the profiler if needed
-  if(NOT TARGET cosma_prefixed_pxgemm)
     add_library(cosma_prefixed_pxgemm scalapack.cpp
       pxgemm_params.hpp
       prefixed_pxgemm.cpp
       cosma_pxgemm.cpp
     )
-    target_link_libraries(cosma_prefixed_pxgemm PUBLIC cosma)
-
-    if(COSMA_WITH_PROFILING)
-      target_link_libraries(cosma_prefixed_pxgemm PRIVATE semiprof::semiprof)
-      target_compile_definitions(cosma_prefixed_pxgemm PRIVATE COSMA_WITH_PROFILING)
-    endif()
+    target_link_libraries(cosma_prefixed_pxgemm
+    PUBLIC cosma
+    PRIVATE $<$<BOOL:${COSMA_WITH_PROFILING}>:semiprof::semiprof>)
+    target_compile_definitions(cosma_prefixed_pxgemm PRIVATE $<$<BOOL:${COSMA_WITH_PROFILING}>:COSMA_WITH_PROFILING>)
     list(APPEND INSTALLED_TARGETS_LIST "cosma_prefixed_pxgemm")
-  endif()
 
   # the following library is aimed only for testing purposes
   # it provides templated cosma::pxgemm call without
@@ -127,12 +105,12 @@ if(COSMA_SCALAPACK)
       pxgemm_params.hpp
       cosma_pxgemm.cpp
     )
-    target_link_libraries(cosma_pxgemm_cpp PUBLIC cosma)
+    target_link_libraries(cosma_pxgemm_cpp
+                            PUBLIC cosma
+                            PRIVATE $<$<BOOL:${COSMA_WITH_PROFILING}>:semiprof::semiprof>)
 
-    if(COSMA_WITH_PROFILING)
-      target_link_libraries(cosma_pxgemm_cpp PRIVATE semiprof::semiprof)
-      target_compile_definitions(cosma_pxgemm_cpp PRIVATE COSMA_WITH_PROFILING)
-    endif()
+    target_compile_definitions(cosma_pxgemm_cpp
+                                  PRIVATE $<$<BOOL:${COSMA_WITH_PROFILING}>:COSMA_WITH_PROFILING>)
     list(APPEND INSTALLED_TARGETS_LIST "cosma_pxgemm_cpp")
   endif()
 endif()

src/cosma/pinned_buffers.cpp

@@ -1,3 +1,4 @@
+#include <complex>
 #include <cosma/pinned_buffers.hpp>
 
 // container of pinned buffers