diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a88d7b48..05942636 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: pull_request: types: [opened, labeled, synchronize] branches: - - release + - 'ROMFPMD' # push: # branches: # - release @@ -25,23 +25,48 @@ jobs: uses: styfle/cancel-workflow-action@0.11.0 with: access_token: ${{ github.token }} - - name: Set Swap Space - uses: pierotofy/set-swap-space@master - with: - swap-size-gb: 10 - - name: Check out pylibROM + # - name: Set Swap Space + # uses: pierotofy/set-swap-space@master + # with: + # swap-size-gb: 10 + - name: Check out mgmol uses: actions/checkout@v1 with: submodules: 'true' - - name: Git Submodules status - run: | - git submodule status - name: cmake run: | mkdir build cd build - cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 + cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMPIEXEC_PREFLAGS="--oversubscribe" - name: make run: | cd build && make -j 4 + - name: test + run: | + cd build && ctest --no-compress-output -V -T Test -I 1,20,1 + # code-style: + # runs-on: ubuntu-latest + # needs: [docker-image] + # container: + # image: ghcr.io/llnl/mgmol/mgmol_env:latest + # options: --user 1001 --privileged + # volumes: + # - /mnt:/mnt + # steps: + # - name: Cancel previous runs + # uses: styfle/cancel-workflow-action@0.11.0 + # with: + # access_token: ${{ github.token }} + # - name: Check out mgmol + # uses: actions/checkout@v1 + # with: + # submodules: 'true' + # - name: cmake + # run: | + # mkdir build + # cd build + # cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMGMOL_WITH_CLANG_FORMAT=ON + # - name: make + # run: | + # cd build && make format diff --git a/CMakeLists.txt b/CMakeLists.txt index f1deb89c..6898f97b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -127,24 +127,12 @@ endif(${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT) set(USE_LIBROM False CACHE BOOL "Build with libROM") set(LIBROM_PATH "" CACHE STRING "Path of libROM") if(USE_LIBROM) + message(STATUS "LIBROM_PATH: ${LIBROM_PATH}") if(NOT LIBROM_PATH) - message(FATAL_ERROR "libROM PATH not specified.") - else(NOT LIBROM_PATH) - set(LIBROM_SCRIPTS_PATH ${LIBROM_PATH}/scripts) - message(STATUS "LIBROM_PATH: ${LIBROM_PATH}") - message(STATUS "LIBROM_SCRIPTS_PATH: ${LIBROM_SCRIPTS_PATH}") - include(ExternalProject) - ExternalProject_Add( - libROM - SOURCE_DIR ${LIBROM_SCRIPTS_PATH} - CONFIGURE_COMMAND "" - BINARY_DIR ${LIBROM_PATH} - BUILD_COMMAND ${LIBROM_SCRIPTS_PATH}/compile.sh -t ${LIBROM_PATH}/cmake/toolchains/simple.cmake - INSTALL_COMMAND "" - ) - include_directories(${LIBROM_PATH}/lib) - link_directories(${LIBROM_PATH}/build/lib) + message(FATAL_ERROR "Cmake is asked to use libROM, but LIBROM_PATH not specified.") endif(NOT LIBROM_PATH) + + find_package(libROM REQUIRED) endif(USE_LIBROM) # ARPACK (optional) diff --git a/cmake_modules/FindSCALAPACK.cmake b/cmake_modules/FindSCALAPACK.cmake index dde12c26..74e58e7d 100644 --- a/cmake_modules/FindSCALAPACK.cmake +++ b/cmake_modules/FindSCALAPACK.cmake @@ -7,7 +7,7 @@ if(DEFINED ENV{SCALAPACK_ROOT}) endif(DEFINED ENV{SCALAPACK_ROOT}) if(SCALAPACK_ROOT) - set(_SCALAPACK_SEARCH_DIR ${SCALAPACK_ROOT}) + set(_SCALAPACK_SEARCH_DIR ${SCALAPACK_ROOT} ${SCALAPACK_ROOT}/lib/intel64) list(APPEND _SCALAPACK_SEARCHES ${_SCALAPACK_SEARCH_DIR}) endif(SCALAPACK_ROOT) @@ -29,13 +29,28 @@ if(NOT SCALAPACK_LIBRARY) endforeach() endif() -unset(SCALAPACK_NAMES) - mark_as_advanced(SCALAPACK_LIBRARY SCALAPACK_INCLUDE_DIR) include(FindPackageHandleStandardArgs) FIND_PACKAGE_HANDLE_STANDARD_ARGS(SCALAPACK REQUIRED_VARS SCALAPACK_LIBRARY) +# Search for some default library paths +if (NOT SCALAPACK_FOUND) + find_library(SCALAPACK_LIBRARY + NAMES ${SCALAPACK_NAMES} + PATHS /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib + /opt/local/lib /opt/sw/lib /sw/lib + ENV LD_LIBRARY_PATH + ENV DYLD_FALLBACK_LIBRARY_PATH + ENV DYLD_LIBRARY_PATH + ENV SCALAPACKDIR + ENV BLACSDIR) + + FIND_PACKAGE_HANDLE_STANDARD_ARGS(SCALAPACK REQUIRED_VARS SCALAPACK_LIBRARY) +endif() + +unset(SCALAPACK_NAMES) + if(SCALAPACK_FOUND) # Only Intel's scalapack requires an include directory if(SCALAPACK_INCLUDE_DIR) diff --git a/cmake_modules/FindlibROM.cmake b/cmake_modules/FindlibROM.cmake new file mode 100644 index 00000000..9fed8fb6 --- /dev/null +++ b/cmake_modules/FindlibROM.cmake @@ -0,0 +1,11 @@ +if(NOT LIBROM_PATH) + message(FATAL_ERROR "LIBROM_PATH not specified.") +endif(NOT LIBROM_PATH) + +find_library(LIBROM_LIB libROM.so HINTS "${LIBROM_PATH}/build/lib") +find_path(LIBROM_INCLUDES librom.h HINTS "${LIBROM_PATH}/lib") + +mark_as_advanced(LIBROM_LIB LIBROM_INCLUDES) + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(libROM REQUIRED_VARS LIBROM_LIB LIBROM_INCLUDES) \ No newline at end of file diff --git a/cmake_toolchains/quartz.default.cmake b/cmake_toolchains/quartz.default.cmake new file mode 100644 index 00000000..9901dcd6 --- /dev/null +++ b/cmake_toolchains/quartz.default.cmake @@ -0,0 +1,6 @@ +set(CMAKE_C_COMPILER mpicc) +set(CMAKE_CXX_COMPILER mpicxx) +set(CMAKE_Fortran_COMPILER mpif90) + +set(SCALAPACK_ROOT $ENV{MKLROOT}) +set(SCALAPACK_BLACS_LIBRARY $ENV{MKLROOT}/lib/intel64/libmkl_blacs_intelmpi_lp64.so) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4ccd50ac..a097df9b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,23 +10,35 @@ WORKDIR /$ENVDIR # install packages RUN sudo apt-get install -yq git RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake -RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpich-dev +RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpi-dev RUN sudo apt-get install -yq libboost-all-dev RUN sudo apt-get install -yq vim RUN sudo apt-get install -yq git-lfs RUN sudo apt-get install -yq valgrind RUN sudo apt-get install -yq wget -RUN sudo apt-get install -yq astyle +### clang-format seems to be updated to 14.0. Not using it for now. +# RUN sudo apt-get install -yq clang-format # install lldb and gdb for debugging RUN sudo apt-get install -yq lldb gdb RUN sudo apt-get clean -q -# NOTE: currently docker does not have a way to set environment variable with a command output. -# The following environment variable should use $(uname -m) for different architecture. -# ENV SCALAPACK_ROOT=/usr/lib/aarch64-linux-gnu/ -ENV SCALAPACK_ROOT=/usr/lib/x86_64-linux-gnu/ +ENV LIB_DIR=/$ENVDIR/dependencies +WORKDIR $LIB_DIR + +# cmake toolchain file for librom +RUN echo 'set(CMAKE_C_COMPILER mpicc)\n\ +set(CMAKE_CXX_COMPILER mpicxx)\n\ +set(CMAKE_Fortran_COMPILER mpif90)' > ./librom_env.cmake +ENV TOOLCHAIN_FILE=$LIB_DIR/librom_env.cmake + +# install libROM for scaleupROM +RUN sudo git clone https://github.com/LLNL/libROM.git +WORKDIR ./libROM/build +# libROM without MFEM. +RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=Optimized -DUSE_MFEM=OFF +RUN sudo make -j 16 # create and switch to a user ENV USERNAME=test diff --git a/scripts/build_quartz_libROM.sh b/scripts/build_quartz_libROM.sh index dc2b689b..458dd662 100644 --- a/scripts/build_quartz_libROM.sh +++ b/scripts/build_quartz_libROM.sh @@ -1,52 +1,61 @@ -##! /bin/csh -f +#!/bin/bash ## An example script to build on LLNL Peloton systems. ## For now, this script assumes intel/ mkl libraries are being used. -# load some modules +## load some modules source scripts/modules.quartz -# set some environment variables. Set them explicitly or use loaded module path (preferred) -# Here we use an explicit path for scalapack to be consistent with the path for the blas libraries and avoid -# benign cmake warnings -#setenv SCALAPACK_ROOT /usr/tce/packages/mkl/mkl-2020.0/lib -setenv SCALAPACK_ROOT ${MKLROOT} -setenv HDF5_ROOT /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0 - -# We need to define the cmake blas vendor option here to find the right one. -set BLAS_VENDOR = Intel10_64lp - -# manually set the location of BLACS libraries for scalapack -set BLACS_LIB = ${SCALAPACK_ROOT}/lib/intel64 - -set MGMOL_ROOT = `pwd` - -set INSTALL_DIR = ${MGMOL_ROOT}/install_quartz +## set some environment variables. Set them explicitly or use loaded module path (preferred) +## Here we use an explicit path for scalapack to be consistent with the path for the blas libraries and avoid +## benign cmake warnings +##setenv SCALAPACK_ROOT /usr/tce/packages/mkl/mkl-2020.0/lib +#setenv SCALAPACK_ROOT ${MKLROOT} +#setenv HDF5_ROOT /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0 +# +## We need to define the cmake blas vendor option here to find the right one. +#set BLAS_VENDOR = Intel10_64lp +# +## manually set the location of BLACS libraries for scalapack +#set BLACS_LIB = ${SCALAPACK_ROOT}/lib/intel64 + +MGMOL_ROOT="$(pwd)" + +INSTALL_DIR=${MGMOL_ROOT}/install_quartz mkdir -p ${INSTALL_DIR} -set BUILD_DIR = ${MGMOL_ROOT}/build_quartz +BUILD_DIR=${MGMOL_ROOT}/build_quartz mkdir -p ${BUILD_DIR} cd ${BUILD_DIR} # clone the libROM GitHub repo in BUILD_DIR -set USE_LIBROM="On" -set LIBROM_PATH = ${BUILD_DIR}/libROM +USE_LIBROM="On" +LIBROM_PATH=${BUILD_DIR}/libROM git clone https://github.com/LLNL/libROM +cd libROM +#./scripts/compile.sh -t ./cmake/toolchains/default-toss_4_x86_64_ib-librom-dev.cmake +./scripts/compile.sh +cd ${BUILD_DIR} # call cmake -cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ - -DCMAKE_CXX_COMPILER=mpic++ \ - -DCMAKE_Fortran_COMPILER=mpif77 \ - -DMPIEXEC_NUMPROC_FLAG="-n" \ - -DBLA_VENDOR=${BLAS_VENDOR} \ - -DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_intelmpi_lp64.so \ +cmake -DCMAKE_TOOLCHAIN_FILE=${MGMOL_ROOT}/cmake_toolchains/quartz.default.cmake \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DUSE_LIBROM=${USE_LIBROM} \ -DLIBROM_PATH=${LIBROM_PATH} \ - -DCMAKE_BUILD_TYPE=DEBUG \ .. +# -DCMAKE_CXX_COMPILER=mpic++ \ +# -DCMAKE_Fortran_COMPILER=mpif77 \ +# -DMPIEXEC_NUMPROC_FLAG="-n" \ +# -DBLA_VENDOR=${BLAS_VENDOR} \ +# -DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_intelmpi_lp64.so \ +# -DCMAKE_BUILD_TYPE=DEBUG \ + # call make install -make -j -make install +make -j 16 +### Currently libROM does not have the installation procedure, +### so copying binary file to installation directory will disrupt the relative path to libROM.so, +### causing a run-time error. +#make install # -DBLAS_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ # -DLAPACK_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ diff --git a/scripts/build_ubuntu22_openmpi.sh b/scripts/build_ubuntu22_openmpi.sh index 23f510b6..a0ca470a 100755 --- a/scripts/build_ubuntu22_openmpi.sh +++ b/scripts/build_ubuntu22_openmpi.sh @@ -25,7 +25,7 @@ cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DMPIEXEC_PREFLAGS="--oversubscribe" \ -DMGMOL_WITH_CLANG_FORMAT=ON \ -DCMAKE_PREFIX_PATH=${HOME}/bin \ - -D CMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \ + -DCMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \ .. # call make install diff --git a/scripts/modules.quartz b/scripts/modules.quartz index 220e0a31..82433acb 100644 --- a/scripts/modules.quartz +++ b/scripts/modules.quartz @@ -1,12 +1,19 @@ -module load intel/2022.1.0 +### choose either gcc or intel +#module load intel/2022.1.0 +module load gcc/11.2.1 + +module load cmake module load hdf5-parallel/1.14.0 -#module load hdf5-parallel -#module load boost -module load mkl -module load cmake/3.14.5 +module load boost + +### choose either one +module load mkl-interfaces +#module load mkl + module load python -## manually add boost path -setenv LD_LIBRARY_PATH /usr/tce/packages/boost/boost-1.80.0-mvapich2-2.3.6-gcc-10.3.1/lib:$LD_LIBRARY_PATH + +### manually add boost path +#setenv LD_LIBRARY_PATH /usr/tce/packages/boost/boost-1.80.0-mvapich2-2.3.6-gcc-10.3.1/lib:$LD_LIBRARY_PATH #setenv MKLROOT $LIBRARY_PATH #setenv MKLROOT /usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2dba55fd..3cb7d745 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -193,3 +193,24 @@ endif (${MGMOL_WITH_LIBXC}) install(TARGETS mgmol-opt DESTINATION bin) +# build ROM executable +if(USE_LIBROM) + add_executable(mgmol-rom rom_main.cc) + target_include_directories (mgmol-rom PRIVATE ${Boost_INCLUDE_DIRS} ${LIBROM_INCLUDES}) + + target_link_libraries(mgmol-rom mgmol_src ${link_libs}) + target_link_libraries(mgmol-rom ${SCALAPACK_LIBRARIES}) + target_link_libraries(mgmol-rom ${HDF5_LIBRARIES}) + target_link_libraries(mgmol-rom ${HDF5_HL_LIBRARIES}) + target_link_libraries(mgmol-rom ${BLAS_LIBRARIES}) + target_link_libraries(mgmol-rom ${LAPACK_LIBRARIES}) + target_link_libraries(mgmol-rom ${Boost_LIBRARIES}) + target_link_libraries(mgmol-rom ${LIBROM_LIB}) + if (${OPENMP_CXX_FOUND}) + target_link_libraries(mgmol-rom OpenMP::OpenMP_CXX) + endif() + if(${MGMOL_WITH_LIBXC}) + target_link_libraries(mgmol-rom ${LIBXC_DIR}/lib/libxc.a) + endif (${MGMOL_WITH_LIBXC}) + install(TARGETS mgmol-rom DESTINATION bin) +endif(USE_LIBROM) diff --git a/src/MGmol.cc b/src/MGmol.cc index b43c70e3..0d1703d9 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -106,7 +106,7 @@ extern Timer ions_setupInteractingIons_tm; extern Timer ions_setup_tm; extern Timer updateCenters_tm; -#include "Signal.h" +#include "Signal_mgmol.h" std::set Signal::recv_; template diff --git a/src/md.cc b/src/md.cc index 1adab46b..6b43048e 100644 --- a/src/md.cc +++ b/src/md.cc @@ -31,7 +31,7 @@ #include "ProjectedMatricesMehrstellen.h" #include "ProjectedMatricesSparse.h" #include "Rho.h" -#include "Signal.h" +#include "Signal_mgmol.h" #include "SpreadsAndCenters.h" #include "tools.h" diff --git a/src/rom_main.cc b/src/rom_main.cc new file mode 100644 index 00000000..c7ec278e --- /dev/null +++ b/src/rom_main.cc @@ -0,0 +1,102 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +// +// main.cc +// +// Description: +// Real grid, finite difference, molecular dynamics program +// for with nonorthogonal localized orbitals. +// +// Uses Mehrstellen operators, multigrid accelerations, and +// non-local pseudopotentials. +// +// Includes LDA and PBE exchange and correlation functionals. +// +// Units: +// Potentials, eigenvalues and operators in Rydberg +// Energies in Hartree +// +#include +#include +#include +#include +using namespace std; + +#ifdef _OPENMP +#include +#endif + +#ifdef USE_CNR +#include +#endif + +#include + +#include "Control.h" +#include "DistMatrix.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "MatricesBlacsContext.h" +#include "Mesh.h" +#include "PackedCommunicationBuffer.h" +#include "ReplicatedWorkSpace.h" +#include "SparseDistMatrix.h" +#include "magma_singleton.h" +#include "tools.h" + +#include +#include +#include + +#include +namespace po = boost::program_options; + +#include "librom.h" + +//#include "MemTrack.h" + +int main(int argc, char** argv) +{ + // change handling of memory allocation errors + set_new_handler(noMoreMemory); + + cout.sync_with_stdio(); + + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + cerr << "MPI Initialization failed!!!" << endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + MPI_Comm_rank(MPI_COMM_WORLD, &mype); + assert(mype > -1); + onpe0 = (mype == 0); + + CAROM::Vector librom_vector(10, false); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + cerr << "MPI Finalize failed!!!" << endl; + } + + time_t tt; + time(&tt); + if (onpe0) cout << " Run ended at " << ctime(&tt) << endl; + + // MemTrack::TrackDumpBlocks(); + + // MemTrack::TrackListMemoryUsage(); + + return 0; +} diff --git a/src/tools/Signal.h b/src/tools/Signal_mgmol.h similarity index 94% rename from src/tools/Signal.h rename to src/tools/Signal_mgmol.h index 8e8d0a21..245a1ca2 100644 --- a/src/tools/Signal.h +++ b/src/tools/Signal_mgmol.h @@ -7,7 +7,7 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// Adapted from Jeep: Signal.h,v 1.5 2002/06/28 20:50:33 +// Adapted from Jeep: Signal_mgmol.h,v 1.5 2002/06/28 20:50:33 // The Signal class is a utility to catch UNIX signals. // A set of flags is maintained to remeber which signals were caught. @@ -15,7 +15,7 @@ // access the flag set, reset flags, or interrogate flags. // The Signal class can be used in an application by declaring // -// #include "Signal.h" +// #include "Signal_mgmol.h" // set Signal::recv_; // // A signal can be registered using, e.g. diff --git a/src/tools/Timeout.h b/src/tools/Timeout.h index 18940a78..db60d719 100644 --- a/src/tools/Timeout.h +++ b/src/tools/Timeout.h @@ -17,7 +17,7 @@ #include #include "MPIdata.h" -#include "Signal.h" +#include "Signal_mgmol.h" #if PCS #include