Base for librom-mgmol interface development (#219)

* changed the Signal.h header file name.

* additional routine to find scalapack library, if nothing is found.

* ci workflow with test and format

* changed branch name

* ctest verbose

* added --oversubscribe to cmake MPIEXEC_PREFLAGS variable.

* parallel hdf5 package fix. also librom is included.

* ci workflow only checks the first 20 tests.

* minor fix

* rom-fpmd driver and librom dependency in cmake. librom must be compiled before mgmol cmake.

* cmake fix and toolchain file for LC quartz.

* bug fix on FindSCALAPACK.cmake

* librom-mgmol installation script for LC quartz.

.github/workflows/ci.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
     types: [opened, labeled, synchronize]
     branches:
-      - release
+      - 'ROMFPMD'
   # push:
   #   branches:
   #     - release
@@ -25,23 +25,48 @@ jobs:
         uses: styfle/cancel-workflow-action@0.11.0
         with:
           access_token: ${{ github.token }}
-      - name: Set Swap Space
-        uses: pierotofy/set-swap-space@master
-        with:
-          swap-size-gb: 10
-      - name: Check out pylibROM
+      # - name: Set Swap Space
+      #   uses: pierotofy/set-swap-space@master
+      #   with:
+      #     swap-size-gb: 10
+      - name: Check out mgmol
         uses: actions/checkout@v1
         with:
             submodules: 'true'
-      - name: Git Submodules status
-        run: |
-            git submodule status
       - name: cmake
         run: |
             mkdir build
             cd build
-            cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90
+            cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMPIEXEC_PREFLAGS="--oversubscribe"
       - name: make
         run: |
             cd build && make -j 4
+      - name: test
+        run: |
+            cd build && ctest --no-compress-output -V -T Test -I 1,20,1
+  # code-style:
+  #   runs-on: ubuntu-latest
+  #   needs: [docker-image]
+  #   container:
+  #     image: ghcr.io/llnl/mgmol/mgmol_env:latest
+  #     options: --user 1001 --privileged
+  #     volumes:
+  #       - /mnt:/mnt
+  #   steps:
+  #     - name: Cancel previous runs
+  #       uses: styfle/cancel-workflow-action@0.11.0
+  #       with:
+  #         access_token: ${{ github.token }}
+  #     - name: Check out mgmol
+  #       uses: actions/checkout@v1
+  #       with:
+  #           submodules: 'true'
+  #     - name: cmake
+  #       run: |
+  #           mkdir build
+  #           cd build
+  #           cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMGMOL_WITH_CLANG_FORMAT=ON
+  #     - name: make
+  #       run: |
+  #           cd build && make format
 

CMakeLists.txt

@@ -127,24 +127,12 @@ endif(${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT)
 set(USE_LIBROM False CACHE BOOL "Build with libROM")
 set(LIBROM_PATH "" CACHE STRING "Path of libROM")
 if(USE_LIBROM)
+  message(STATUS "LIBROM_PATH: ${LIBROM_PATH}")
   if(NOT LIBROM_PATH)
-    message(FATAL_ERROR "libROM PATH not specified.")
-  else(NOT LIBROM_PATH)
-    set(LIBROM_SCRIPTS_PATH ${LIBROM_PATH}/scripts)
-    message(STATUS "LIBROM_PATH: ${LIBROM_PATH}")
-    message(STATUS "LIBROM_SCRIPTS_PATH: ${LIBROM_SCRIPTS_PATH}")
-    include(ExternalProject)
-    ExternalProject_Add(
-      libROM
-      SOURCE_DIR ${LIBROM_SCRIPTS_PATH}
-      CONFIGURE_COMMAND ""
-      BINARY_DIR ${LIBROM_PATH}
-      BUILD_COMMAND ${LIBROM_SCRIPTS_PATH}/compile.sh -t ${LIBROM_PATH}/cmake/toolchains/simple.cmake
-      INSTALL_COMMAND ""
-    )
-    include_directories(${LIBROM_PATH}/lib)
-    link_directories(${LIBROM_PATH}/build/lib)
+    message(FATAL_ERROR "Cmake is asked to use libROM, but LIBROM_PATH not specified.")
   endif(NOT LIBROM_PATH)
+
+  find_package(libROM REQUIRED)
 endif(USE_LIBROM)
 
 # ARPACK (optional)

cmake_modules/FindSCALAPACK.cmake

@@ -7,7 +7,7 @@ if(DEFINED ENV{SCALAPACK_ROOT})
 endif(DEFINED ENV{SCALAPACK_ROOT})
 
 if(SCALAPACK_ROOT)
-  set(_SCALAPACK_SEARCH_DIR ${SCALAPACK_ROOT})
+  set(_SCALAPACK_SEARCH_DIR ${SCALAPACK_ROOT} ${SCALAPACK_ROOT}/lib/intel64)
   list(APPEND _SCALAPACK_SEARCHES ${_SCALAPACK_SEARCH_DIR})
 endif(SCALAPACK_ROOT)
 
@@ -29,13 +29,28 @@ if(NOT SCALAPACK_LIBRARY)
   endforeach()
 endif()
 
-unset(SCALAPACK_NAMES)
-
 mark_as_advanced(SCALAPACK_LIBRARY SCALAPACK_INCLUDE_DIR)
 
 include(FindPackageHandleStandardArgs)
 FIND_PACKAGE_HANDLE_STANDARD_ARGS(SCALAPACK REQUIRED_VARS SCALAPACK_LIBRARY)
 
+# Search for some default library paths
+if (NOT SCALAPACK_FOUND)
+  find_library(SCALAPACK_LIBRARY
+    NAMES ${SCALAPACK_NAMES}
+    PATHS /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib
+    /opt/local/lib /opt/sw/lib /sw/lib
+    ENV LD_LIBRARY_PATH
+    ENV DYLD_FALLBACK_LIBRARY_PATH
+    ENV DYLD_LIBRARY_PATH
+    ENV SCALAPACKDIR
+    ENV BLACSDIR)
+
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(SCALAPACK REQUIRED_VARS SCALAPACK_LIBRARY)
+endif()
+
+unset(SCALAPACK_NAMES)
+
 if(SCALAPACK_FOUND)
   # Only Intel's scalapack requires an include directory
   if(SCALAPACK_INCLUDE_DIR)

cmake_modules/FindlibROM.cmake

@@ -0,0 +1,11 @@
+if(NOT LIBROM_PATH)
+    message(FATAL_ERROR "LIBROM_PATH not specified.")
+endif(NOT LIBROM_PATH)
+
+find_library(LIBROM_LIB libROM.so HINTS "${LIBROM_PATH}/build/lib")
+find_path(LIBROM_INCLUDES librom.h HINTS "${LIBROM_PATH}/lib")
+
+mark_as_advanced(LIBROM_LIB LIBROM_INCLUDES)
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(libROM REQUIRED_VARS LIBROM_LIB LIBROM_INCLUDES)

cmake_toolchains/quartz.default.cmake

@@ -0,0 +1,6 @@
+set(CMAKE_C_COMPILER mpicc)
+set(CMAKE_CXX_COMPILER mpicxx)
+set(CMAKE_Fortran_COMPILER mpif90)
+
+set(SCALAPACK_ROOT $ENV{MKLROOT})
+set(SCALAPACK_BLACS_LIBRARY $ENV{MKLROOT}/lib/intel64/libmkl_blacs_intelmpi_lp64.so)

docker/Dockerfile

@@ -10,23 +10,35 @@ WORKDIR /$ENVDIR
 # install packages
 RUN sudo apt-get install -yq git
 RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake
-RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpich-dev
+RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpi-dev
 RUN sudo apt-get install -yq libboost-all-dev
 RUN sudo apt-get install -yq vim
 RUN sudo apt-get install -yq git-lfs
 RUN sudo apt-get install -yq valgrind
 RUN sudo apt-get install -yq wget
-RUN sudo apt-get install -yq astyle
+### clang-format seems to be updated to 14.0. Not using it for now.
+# RUN sudo apt-get install -yq clang-format
 
 # install lldb and gdb for debugging
 RUN sudo apt-get install -yq lldb gdb
 
 RUN sudo apt-get clean -q
 
-# NOTE: currently docker does not have a way to set environment variable with a command output.
-# The following environment variable should use $(uname -m) for different architecture.
-# ENV SCALAPACK_ROOT=/usr/lib/aarch64-linux-gnu/
-ENV SCALAPACK_ROOT=/usr/lib/x86_64-linux-gnu/
+ENV LIB_DIR=/$ENVDIR/dependencies
+WORKDIR $LIB_DIR
+
+# cmake toolchain file for librom
+RUN echo 'set(CMAKE_C_COMPILER mpicc)\n\
+set(CMAKE_CXX_COMPILER mpicxx)\n\
+set(CMAKE_Fortran_COMPILER mpif90)' > ./librom_env.cmake
+ENV TOOLCHAIN_FILE=$LIB_DIR/librom_env.cmake
+
+# install libROM for scaleupROM
+RUN sudo git clone https://github.com/LLNL/libROM.git
+WORKDIR ./libROM/build
+# libROM without MFEM.
+RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=Optimized -DUSE_MFEM=OFF
+RUN sudo make -j 16
 
 # create and switch to a user
 ENV USERNAME=test

scripts/build_quartz_libROM.sh

@@ -1,52 +1,61 @@
-##! /bin/csh -f
+#!/bin/bash
 ## An example script to build on LLNL Peloton systems.
 ## For now, this script assumes intel/ mkl libraries are being used.
 
-# load some modules
+## load some modules
 source scripts/modules.quartz
 
-# set some environment variables. Set them explicitly or use loaded module path (preferred)
-# Here we use an explicit path for scalapack to be consistent with the path for the blas libraries and avoid
-# benign cmake warnings
-#setenv SCALAPACK_ROOT /usr/tce/packages/mkl/mkl-2020.0/lib
-setenv SCALAPACK_ROOT ${MKLROOT}
-setenv HDF5_ROOT /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0
-
-# We need to define the cmake blas vendor option here to find the right one.
-set BLAS_VENDOR = Intel10_64lp
-
-# manually set the location of BLACS libraries for scalapack
-set BLACS_LIB = ${SCALAPACK_ROOT}/lib/intel64
-
-set MGMOL_ROOT = `pwd`
-
-set INSTALL_DIR = ${MGMOL_ROOT}/install_quartz
+## set some environment variables. Set them explicitly or use loaded module path (preferred)
+## Here we use an explicit path for scalapack to be consistent with the path for the blas libraries and avoid
+## benign cmake warnings
+##setenv SCALAPACK_ROOT /usr/tce/packages/mkl/mkl-2020.0/lib
+#setenv SCALAPACK_ROOT ${MKLROOT}
+#setenv HDF5_ROOT /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0
+#
+## We need to define the cmake blas vendor option here to find the right one.
+#set BLAS_VENDOR = Intel10_64lp
+#
+## manually set the location of BLACS libraries for scalapack
+#set BLACS_LIB = ${SCALAPACK_ROOT}/lib/intel64
+
+MGMOL_ROOT="$(pwd)"
+
+INSTALL_DIR=${MGMOL_ROOT}/install_quartz
 mkdir -p ${INSTALL_DIR}
 
-set BUILD_DIR = ${MGMOL_ROOT}/build_quartz
+BUILD_DIR=${MGMOL_ROOT}/build_quartz
 mkdir -p ${BUILD_DIR}
 cd ${BUILD_DIR}
 
 # clone the libROM GitHub repo in BUILD_DIR
-set USE_LIBROM="On"
-set LIBROM_PATH = ${BUILD_DIR}/libROM
+USE_LIBROM="On"
+LIBROM_PATH=${BUILD_DIR}/libROM
 git clone https://github.com/LLNL/libROM
+cd libROM
+#./scripts/compile.sh -t ./cmake/toolchains/default-toss_4_x86_64_ib-librom-dev.cmake
+./scripts/compile.sh
+cd ${BUILD_DIR}
 
 # call cmake
-cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
-      -DCMAKE_CXX_COMPILER=mpic++ \
-      -DCMAKE_Fortran_COMPILER=mpif77 \
-      -DMPIEXEC_NUMPROC_FLAG="-n" \
-      -DBLA_VENDOR=${BLAS_VENDOR} \
-      -DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_intelmpi_lp64.so \
+cmake -DCMAKE_TOOLCHAIN_FILE=${MGMOL_ROOT}/cmake_toolchains/quartz.default.cmake \
+      -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
       -DUSE_LIBROM=${USE_LIBROM} \
       -DLIBROM_PATH=${LIBROM_PATH} \
-      -DCMAKE_BUILD_TYPE=DEBUG \
       ..
 
+#      -DCMAKE_CXX_COMPILER=mpic++ \
+#      -DCMAKE_Fortran_COMPILER=mpif77 \
+#      -DMPIEXEC_NUMPROC_FLAG="-n" \
+#      -DBLA_VENDOR=${BLAS_VENDOR} \
+#      -DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_intelmpi_lp64.so \
+#      -DCMAKE_BUILD_TYPE=DEBUG \
+
 # call make install
-make -j 
-make install
+make -j 16
+### Currently libROM does not have the installation procedure,
+### so copying binary file to installation directory will disrupt the relative path to libROM.so,
+### causing a run-time error.
+#make install
 
 #      -DBLAS_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \
 #      -DLAPACK_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \  

scripts/build_ubuntu22_openmpi.sh

@@ -25,7 +25,7 @@ cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
       -DMPIEXEC_PREFLAGS="--oversubscribe" \
       -DMGMOL_WITH_CLANG_FORMAT=ON \
       -DCMAKE_PREFIX_PATH=${HOME}/bin \
-      -D CMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \
+      -DCMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \
       ..
 
 # call make install

scripts/modules.quartz

@@ -1,12 +1,19 @@
-module load intel/2022.1.0
+### choose either gcc or intel
+#module load intel/2022.1.0
+module load gcc/11.2.1
+
+module load cmake
 module load hdf5-parallel/1.14.0
-#module load hdf5-parallel
-#module load boost
-module load mkl
-module load cmake/3.14.5
+module load boost
+
+### choose either one
+module load mkl-interfaces
+#module load mkl
+
 module load python
-## manually add boost path
-setenv LD_LIBRARY_PATH /usr/tce/packages/boost/boost-1.80.0-mvapich2-2.3.6-gcc-10.3.1/lib:$LD_LIBRARY_PATH
+
+### manually add boost path
+#setenv LD_LIBRARY_PATH /usr/tce/packages/boost/boost-1.80.0-mvapich2-2.3.6-gcc-10.3.1/lib:$LD_LIBRARY_PATH
 
 #setenv MKLROOT $LIBRARY_PATH
 #setenv MKLROOT /usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0

src/CMakeLists.txt

@@ -193,3 +193,24 @@ endif (${MGMOL_WITH_LIBXC})
 
 install(TARGETS mgmol-opt DESTINATION bin)
 
+# build ROM executable
+if(USE_LIBROM)
+  add_executable(mgmol-rom rom_main.cc)
+  target_include_directories (mgmol-rom PRIVATE ${Boost_INCLUDE_DIRS} ${LIBROM_INCLUDES})
+
+  target_link_libraries(mgmol-rom mgmol_src ${link_libs})
+  target_link_libraries(mgmol-rom ${SCALAPACK_LIBRARIES})
+  target_link_libraries(mgmol-rom ${HDF5_LIBRARIES})
+  target_link_libraries(mgmol-rom ${HDF5_HL_LIBRARIES})
+  target_link_libraries(mgmol-rom ${BLAS_LIBRARIES})
+  target_link_libraries(mgmol-rom ${LAPACK_LIBRARIES})
+  target_link_libraries(mgmol-rom ${Boost_LIBRARIES})
+  target_link_libraries(mgmol-rom ${LIBROM_LIB})
+  if (${OPENMP_CXX_FOUND})
+    target_link_libraries(mgmol-rom OpenMP::OpenMP_CXX)
+  endif()
+  if(${MGMOL_WITH_LIBXC})
+    target_link_libraries(mgmol-rom ${LIBXC_DIR}/lib/libxc.a)
+  endif (${MGMOL_WITH_LIBXC})
+  install(TARGETS mgmol-rom DESTINATION bin)
+endif(USE_LIBROM)

src/MGmol.cc

@@ -106,7 +106,7 @@ extern Timer ions_setupInteractingIons_tm;
 extern Timer ions_setup_tm;
 extern Timer updateCenters_tm;
 
-#include "Signal.h"
+#include "Signal_mgmol.h"
 std::set<int> Signal::recv_;
 
 template <class OrbitalsType>

src/md.cc

@@ -31,7 +31,7 @@
 #include "ProjectedMatricesMehrstellen.h"
 #include "ProjectedMatricesSparse.h"
 #include "Rho.h"
-#include "Signal.h"
+#include "Signal_mgmol.h"
 #include "SpreadsAndCenters.h"
 #include "tools.h"