diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..05942636 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,72 @@ +name: CI +on: + workflow_dispatch: {} + pull_request: + types: [opened, labeled, synchronize] + branches: + - 'ROMFPMD' + # push: + # branches: + # - release + +jobs: + docker-image: + uses: ./.github/workflows/docker_image.yml + build: + runs-on: ubuntu-latest + needs: [docker-image] + container: + image: ghcr.io/llnl/mgmol/mgmol_env:latest + options: --user 1001 --privileged + volumes: + - /mnt:/mnt + steps: + - name: Cancel previous runs + uses: styfle/cancel-workflow-action@0.11.0 + with: + access_token: ${{ github.token }} + # - name: Set Swap Space + # uses: pierotofy/set-swap-space@master + # with: + # swap-size-gb: 10 + - name: Check out mgmol + uses: actions/checkout@v1 + with: + submodules: 'true' + - name: cmake + run: | + mkdir build + cd build + cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMPIEXEC_PREFLAGS="--oversubscribe" + - name: make + run: | + cd build && make -j 4 + - name: test + run: | + cd build && ctest --no-compress-output -V -T Test -I 1,20,1 + # code-style: + # runs-on: ubuntu-latest + # needs: [docker-image] + # container: + # image: ghcr.io/llnl/mgmol/mgmol_env:latest + # options: --user 1001 --privileged + # volumes: + # - /mnt:/mnt + # steps: + # - name: Cancel previous runs + # uses: styfle/cancel-workflow-action@0.11.0 + # with: + # access_token: ${{ github.token }} + # - name: Check out mgmol + # uses: actions/checkout@v1 + # with: + # submodules: 'true' + # - name: cmake + # run: | + # mkdir build + # cd build + # cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMGMOL_WITH_CLANG_FORMAT=ON + # - name: make + # run: | + # cd build && make format + diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml new file mode 100644 index 00000000..90a2447b --- /dev/null +++ b/.github/workflows/docker_image.yml @@ -0,0 +1,59 @@ +name: docker-image +on: + workflow_call: + +env: + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: llnl/mgmol/mgmol_env + DOCKERPATH: docker + +jobs: + docker-ci: + runs-on: ubuntu-latest + name: "docker env" + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: Ana06/get-changed-files@v2.2.0 + id: files + - name: DockerPATH configuration + run: echo "DOCKERPATH=$DOCKERPATH" + - name: DockerPATH - check if files in docker path changed + if: contains(steps.files.outputs.all,env.DOCKERPATH) || contains(steps.files.outputs.all,'docker_image.yml') + run: | + echo "CI container needs rebuilding..." + echo "CI_NEEDS_REBUILD=true" >> $GITHUB_ENV + - name: Log into registry ${{ env.REGISTRY }} + if: env.CI_NEEDS_REBUILD + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + if: env.CI_NEEDS_REBUILD + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: type=sha + flavor: latest=true + - name: Build Container motd + if: env.CI_NEEDS_REBUILD + run: | + echo "#!/bin/bash" > ${{env.DOCKERPATH}}/motd.sh + echo "echo --------------------------" >> ${{env.DOCKERPATH}}/motd.sh + echo "echo mgmol_env/CI Development Container" >> ${{env.DOCKERPATH}}/motd.sh + echo "echo \"Revision: `echo ${GITHUB_SHA} | cut -c1-8`\"" >> ${{env.DOCKERPATH}}/motd.sh + echo "echo --------------------------" >> ${{env.DOCKERPATH}}/motd.sh + chmod 755 ${{env.DOCKERPATH}}/motd.sh + cat ${{env.DOCKERPATH}}/motd.sh + - name: Docker Image - Build and push + if: env.CI_NEEDS_REBUILD + uses: docker/build-push-action@v3 + with: + push: true + context: ${{ env.DOCKERPATH }} + tags: ${{ steps.meta.outputs.tags }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..81c6b154 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +src/mgmol_config.h diff --git a/CMakeLists.txt b/CMakeLists.txt index aee0ddb5..75752a3a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,6 +126,22 @@ if (${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT) endif(${SCALAPACK_FOUND}) endif(${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT) +# libROM (optional) +set(USE_LIBROM False CACHE BOOL "Build with libROM") +set(LIBROM_PATH "" CACHE STRING "Path of libROM") +if(USE_LIBROM) + message(STATUS "LIBROM_PATH: ${LIBROM_PATH}") + if(NOT LIBROM_PATH) + message(FATAL_ERROR "Cmake is asked to use libROM, but LIBROM_PATH not specified.") + endif(NOT LIBROM_PATH) + + find_package(libROM REQUIRED) + + if(libROM_FOUND) + set(MGMOL_HAS_LIBROM 1) + endif(libROM_FOUND) +endif(USE_LIBROM) + # ARPACK (optional) set(MGMOL_WITH_ARPACK FALSE CACHE BOOL "Compile with ARPACK package") if(${MGMOL_WITH_ARPACK} OR DEFINED ARPACK_ROOT) @@ -249,6 +265,9 @@ include_directories("${PROJECT_SOURCE_DIR}/src/sparse_linear_algebra") include_directories("${PROJECT_SOURCE_DIR}/src/tools") include_directories("${PROJECT_SOURCE_DIR}/src") +include_directories("${LIBROM_PATH}/lib") +link_libraries(${LIBROM_LIB}) + # add subdirectories for source files, tests add_subdirectory(src) diff --git a/cmake_modules/FindlibROM.cmake b/cmake_modules/FindlibROM.cmake new file mode 100644 index 00000000..9fed8fb6 --- /dev/null +++ b/cmake_modules/FindlibROM.cmake @@ -0,0 +1,11 @@ +if(NOT LIBROM_PATH) + message(FATAL_ERROR "LIBROM_PATH not specified.") +endif(NOT LIBROM_PATH) + +find_library(LIBROM_LIB libROM.so HINTS "${LIBROM_PATH}/build/lib") +find_path(LIBROM_INCLUDES librom.h HINTS "${LIBROM_PATH}/lib") + +mark_as_advanced(LIBROM_LIB LIBROM_INCLUDES) + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(libROM REQUIRED_VARS LIBROM_LIB LIBROM_INCLUDES) \ No newline at end of file diff --git a/cmake_toolchains/quartz.default.cmake b/cmake_toolchains/quartz.default.cmake new file mode 100644 index 00000000..9901dcd6 --- /dev/null +++ b/cmake_toolchains/quartz.default.cmake @@ -0,0 +1,6 @@ +set(CMAKE_C_COMPILER mpicc) +set(CMAKE_CXX_COMPILER mpicxx) +set(CMAKE_Fortran_COMPILER mpif90) + +set(SCALAPACK_ROOT $ENV{MKLROOT}) +set(SCALAPACK_BLACS_LIBRARY $ENV{MKLROOT}/lib/intel64/libmkl_blacs_intelmpi_lp64.so) diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..8b1ffa94 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:22.04 + +ENV ENVDIR=env + +# install sudo +RUN apt-get -yq update && apt-get -yq install sudo + +WORKDIR /$ENVDIR + +# install packages +RUN sudo apt-get install -yq git +RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake +RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpi-dev +RUN sudo apt-get install -yq libboost-all-dev +RUN sudo apt-get install -yq vim +RUN sudo apt-get install -yq git-lfs +RUN sudo apt-get install -yq valgrind hdf5-tools +RUN sudo apt-get install -yq wget +### clang-format seems to be updated to 14.0. Not using it for now. +# RUN sudo apt-get install -yq clang-format + +# install lldb and gdb for debugging +RUN sudo apt-get install -yq lldb gdb + +RUN sudo apt-get clean -q + +ENV LIB_DIR=/$ENVDIR/dependencies +WORKDIR $LIB_DIR + +# cmake toolchain file for librom +RUN echo 'set(CMAKE_C_COMPILER mpicc)\n\ +set(CMAKE_CXX_COMPILER mpicxx)\n\ +set(CMAKE_Fortran_COMPILER mpif90)' > ./librom_env.cmake +ENV TOOLCHAIN_FILE=$LIB_DIR/librom_env.cmake + +# install libROM for scaleupROM +RUN sudo git clone https://github.com/LLNL/libROM.git +WORKDIR ./libROM/build +# libROM without MFEM. +RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=Optimized -DUSE_MFEM=OFF +RUN sudo make -j 16 + +# create and switch to a user +ENV USERNAME=test +RUN echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +RUN useradd --no-log-init -u 1001 --create-home --shell /bin/bash $USERNAME +RUN adduser $USERNAME sudo +USER $USERNAME +WORKDIR /home/$USERNAME + diff --git a/examples/Carbyne/carbyne.rom.cfg b/examples/Carbyne/carbyne.rom.cfg new file mode 100644 index 00000000..cb0cd295 --- /dev/null +++ b/examples/Carbyne/carbyne.rom.cfg @@ -0,0 +1,34 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx= 96 +ny= 96 +nz= 192 +[Domain] +ox= -10. +oy= -10. +oz= -20. +lx= 20. +ly= 20. +lz= 40. +[Potentials] +pseudopotential=pseudo.H_ONCV_PBE_SG15 +pseudopotential=pseudo.C_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +max_steps=5 +atol=1.e-8 +[Orbitals] +initial_type=Fourier +[Restart] +output_level=4 +input_level=4 +input_filename=snapshot0_000 + +[ROM.offline] +restart_filefmt=snapshot0_%03d +restart_min_idx=0 +restart_max_idx=1 +basis_file=carom diff --git a/scripts/build_quartz_libROM.sh b/scripts/build_quartz_libROM.sh new file mode 100644 index 00000000..458dd662 --- /dev/null +++ b/scripts/build_quartz_libROM.sh @@ -0,0 +1,61 @@ +#!/bin/bash +## An example script to build on LLNL Peloton systems. +## For now, this script assumes intel/ mkl libraries are being used. + +## load some modules +source scripts/modules.quartz + +## set some environment variables. Set them explicitly or use loaded module path (preferred) +## Here we use an explicit path for scalapack to be consistent with the path for the blas libraries and avoid +## benign cmake warnings +##setenv SCALAPACK_ROOT /usr/tce/packages/mkl/mkl-2020.0/lib +#setenv SCALAPACK_ROOT ${MKLROOT} +#setenv HDF5_ROOT /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0 +# +## We need to define the cmake blas vendor option here to find the right one. +#set BLAS_VENDOR = Intel10_64lp +# +## manually set the location of BLACS libraries for scalapack +#set BLACS_LIB = ${SCALAPACK_ROOT}/lib/intel64 + +MGMOL_ROOT="$(pwd)" + +INSTALL_DIR=${MGMOL_ROOT}/install_quartz +mkdir -p ${INSTALL_DIR} + +BUILD_DIR=${MGMOL_ROOT}/build_quartz +mkdir -p ${BUILD_DIR} +cd ${BUILD_DIR} + +# clone the libROM GitHub repo in BUILD_DIR +USE_LIBROM="On" +LIBROM_PATH=${BUILD_DIR}/libROM +git clone https://github.com/LLNL/libROM +cd libROM +#./scripts/compile.sh -t ./cmake/toolchains/default-toss_4_x86_64_ib-librom-dev.cmake +./scripts/compile.sh +cd ${BUILD_DIR} + +# call cmake +cmake -DCMAKE_TOOLCHAIN_FILE=${MGMOL_ROOT}/cmake_toolchains/quartz.default.cmake \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DUSE_LIBROM=${USE_LIBROM} \ + -DLIBROM_PATH=${LIBROM_PATH} \ + .. + +# -DCMAKE_CXX_COMPILER=mpic++ \ +# -DCMAKE_Fortran_COMPILER=mpif77 \ +# -DMPIEXEC_NUMPROC_FLAG="-n" \ +# -DBLA_VENDOR=${BLAS_VENDOR} \ +# -DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_intelmpi_lp64.so \ +# -DCMAKE_BUILD_TYPE=DEBUG \ + +# call make install +make -j 16 +### Currently libROM does not have the installation procedure, +### so copying binary file to installation directory will disrupt the relative path to libROM.so, +### causing a run-time error. +#make install + +# -DBLAS_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ +# -DLAPACK_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ diff --git a/scripts/build_ubuntu22_openmpi.sh b/scripts/build_ubuntu22_openmpi.sh index 23f510b6..a0ca470a 100755 --- a/scripts/build_ubuntu22_openmpi.sh +++ b/scripts/build_ubuntu22_openmpi.sh @@ -25,7 +25,7 @@ cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DMPIEXEC_PREFLAGS="--oversubscribe" \ -DMGMOL_WITH_CLANG_FORMAT=ON \ -DCMAKE_PREFIX_PATH=${HOME}/bin \ - -D CMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \ + -DCMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \ .. # call make install diff --git a/scripts/modules.quartz b/scripts/modules.quartz new file mode 100644 index 00000000..82433acb --- /dev/null +++ b/scripts/modules.quartz @@ -0,0 +1,21 @@ +### choose either gcc or intel +#module load intel/2022.1.0 +module load gcc/11.2.1 + +module load cmake +module load hdf5-parallel/1.14.0 +module load boost + +### choose either one +module load mkl-interfaces +#module load mkl + +module load python + +### manually add boost path +#setenv LD_LIBRARY_PATH /usr/tce/packages/boost/boost-1.80.0-mvapich2-2.3.6-gcc-10.3.1/lib:$LD_LIBRARY_PATH + +#setenv MKLROOT $LIBRARY_PATH +#setenv MKLROOT /usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0 +#setenv HDF5ROOT $LD_LIBRARY_PATH +#setenv HDF5ROOT diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b7aec664..e33d03d2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,7 @@ +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/mgmol_config.h.in + ${CMAKE_CURRENT_SOURCE_DIR}/mgmol_config.h @ONLY) + add_subdirectory(DistMatrix) add_subdirectory(linear_algebra) add_subdirectory(local_matrices) @@ -8,7 +11,8 @@ add_subdirectory(radial) add_subdirectory(sparse_linear_algebra) add_subdirectory(tools) -set(link_libs mgmol_distmatrix +set(link_libs + mgmol_distmatrix mgmol_linear_algebra mgmol_local_matrices mgmol_numerical_kernels @@ -155,13 +159,23 @@ set(SOURCES magma_singleton.cc ChebyshevApproximation.cc ChebyshevApproximationInterface.cc + rom.cc ) +if(USE_LIBROM) + list(APPEND SOURCES + rom_workflows.cc) +endif(USE_LIBROM) + add_library(mgmol_src ${SOURCES}) target_include_directories(mgmol_src PRIVATE ${HDF5_INCLUDE_DIRS}) target_include_directories(mgmol_src PRIVATE ${Boost_INCLUDE_DIRS}) -target_include_directories (mgmol_src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_include_directories(mgmol_src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +if(USE_LIBROM) + target_include_directories (mgmol_src PRIVATE ${LIBROM_INCLUDES}) + target_link_libraries(mgmol_src ${LIBROM_LIB}) +endif(USE_LIBROM) target_link_libraries(mgmol_src ${link_libs}) if(${MGMOL_WITH_MAGMA}) @@ -193,3 +207,24 @@ endif (${MGMOL_WITH_LIBXC}) install(TARGETS mgmol-opt DESTINATION bin) +# build ROM executable +if(USE_LIBROM) + add_executable(mgmol-rom rom_main.cc) + target_include_directories (mgmol-rom PRIVATE ${Boost_INCLUDE_DIRS}) + + target_link_libraries(mgmol-rom mgmol_src ${link_libs}) + target_link_libraries(mgmol-rom ${SCALAPACK_LIBRARIES}) + target_link_libraries(mgmol-rom ${HDF5_LIBRARIES}) + target_link_libraries(mgmol-rom ${HDF5_HL_LIBRARIES}) + target_link_libraries(mgmol-rom ${BLAS_LIBRARIES}) + target_link_libraries(mgmol-rom ${LAPACK_LIBRARIES}) + target_link_libraries(mgmol-rom ${Boost_LIBRARIES}) + target_link_libraries(mgmol-rom ${LIBROM_LIB}) + if (${OPENMP_CXX_FOUND}) + target_link_libraries(mgmol-rom OpenMP::OpenMP_CXX) + endif() + if(${MGMOL_WITH_LIBXC}) + target_link_libraries(mgmol-rom ${LIBXC_DIR}/lib/libxc.a) + endif (${MGMOL_WITH_LIBXC}) + install(TARGETS mgmol-rom DESTINATION bin) +endif(USE_LIBROM) diff --git a/src/Control.cc b/src/Control.cc index 7d8dc33f..ce3023e3 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -1917,6 +1917,10 @@ void Control::setOptions(const boost::program_options::variables_map& vm) // synchronize all processors sync(); + +#ifdef MGMOL_HAS_LIBROM + setROMOptions(vm); +#endif } int Control::checkOptions() @@ -2051,3 +2055,66 @@ void Control::printPoissonOptions(std::ostream& os) } os << std::endl; } + +void Control::setROMOptions(const boost::program_options::variables_map& vm) +{ + printWithTimeStamp("Control::setROMOptions()...", std::cout); + + if (onpe0) + { + std::string str = vm["ROM.stage"].as(); + if (str.compare("offline") == 0) + rom_pri_option.rom_stage = ROMStage::OFFLINE; + else if (str.compare("online") == 0) + rom_pri_option.rom_stage = ROMStage::ONLINE; + else if (str.compare("build") == 0) + rom_pri_option.rom_stage = ROMStage::BUILD; + else if (str.compare("none") == 0) + rom_pri_option.rom_stage = ROMStage::UNSUPPORTED; + + rom_pri_option.restart_file_fmt = vm["ROM.offline.restart_filefmt"].as(); + rom_pri_option.restart_file_minidx = vm["ROM.offline.restart_min_idx"].as(); + rom_pri_option.restart_file_maxidx = vm["ROM.offline.restart_max_idx"].as(); + rom_pri_option.basis_file = vm["ROM.offline.basis_file"].as(); + + rom_pri_option.save_librom_snapshot = vm["ROM.offline.save_librom_snapshot"].as(); + } // onpe0 + + // synchronize all processors + syncROMOptions(); +} + +void Control::syncROMOptions() +{ + if (onpe0 && verbose > 0) + (*MPIdata::sout) << "Control::syncROMOptions()" << std::endl; + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + + mmpi.bcast(rom_pri_option.restart_file_fmt, comm_global_); + mmpi.bcast(rom_pri_option.basis_file, comm_global_); + + auto bcast_check = [](int mpirc) { + if (mpirc != MPI_SUCCESS) + { + (*MPIdata::sout) << "MPI Bcast of Control failed!!!" << std::endl; + MPI_Abort(comm_global_, 2); + } + }; + + short rom_stage = (short)static_cast(rom_pri_option.rom_stage); + int mpirc; + mpirc = MPI_Bcast(&rom_stage, 1, MPI_SHORT, 0, comm_global_); + bcast_check(mpirc); + + mpirc = MPI_Bcast(&rom_pri_option.restart_file_minidx, 1, MPI_INT, 0, comm_global_); + bcast_check(mpirc); + + mpirc = MPI_Bcast(&rom_pri_option.restart_file_maxidx, 1, MPI_INT, 0, comm_global_); + bcast_check(mpirc); + + mpirc = MPI_Bcast(&rom_pri_option.save_librom_snapshot, 1, MPI_C_BOOL, 0, comm_global_); + bcast_check(mpirc); + + rom_pri_option.rom_stage = static_cast(rom_stage); +} \ No newline at end of file diff --git a/src/Control.h b/src/Control.h index f1fd8779..1ef7a047 100644 --- a/src/Control.h +++ b/src/Control.h @@ -13,6 +13,10 @@ #include "Species.h" #include "Timeout.h" +/* enumeration and option variables for libROM */ +#include "mgmol_config.h" +#include "rom_Control.h" + #include #include #include @@ -220,6 +224,9 @@ class Control void printRestartLink(); + /* libROM related options */ + ROMPrivateOptions rom_pri_option; + public: static Control* instance() { @@ -707,6 +714,11 @@ class Control } bool AtomsMove() { return (atoms_dyn_ != 0); } + + /* ROM-related options */ + void setROMOptions(const boost::program_options::variables_map& vm); + void syncROMOptions(); + const ROMPrivateOptions getROMOptions() { return rom_pri_option; } }; #endif diff --git a/src/MGmol.cc b/src/MGmol.cc index b6a95777..39f30378 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -1159,7 +1159,19 @@ void MGmol::cleanup() if (ierr < 0) os_ << "WARNING: writing restart data failed!!!" << std::endl; - } + +#ifdef MGMOL_HAS_LIBROM + // Save orbital snapshots + if (ct.getROMOptions().save_librom_snapshot > 0 && ct.AtomsDynamic() == AtomsDynamicType::Quench) + { + ierr = save_orbital_snapshot( + filename, *current_orbitals_); + + if (ierr < 0) + os_ << "WARNING: writing ROM snapshot data failed!!!" << std::endl; + } +#endif + } // if (ct.out_restart_info > 0 && !ct.AtomsMove()) MPI_Barrier(comm_); closing_tm_.stop(); diff --git a/src/MGmol.h b/src/MGmol.h index a44a37f8..1ed04041 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -10,6 +10,8 @@ #ifndef MGMOL_H #define MGMOL_H +#include "mgmol_config.h" + #include "Energy.h" #include "GridFuncVector.h" #include "Hamiltonian.h" @@ -305,6 +307,10 @@ class MGmol : public MGmolInterface } OrbitalsType* loadOrbitalFromRestartFile(const std::string filename); + +#ifdef MGMOL_HAS_LIBROM + int save_orbital_snapshot(std::string snapshot_dir, OrbitalsType& orbitals); +#endif }; // Instantiate static variables here to avoid clang warnings template diff --git a/src/MGmol_prototypes.h b/src/MGmol_prototypes.h index 8c47c388..207201d7 100644 --- a/src/MGmol_prototypes.h +++ b/src/MGmol_prototypes.h @@ -10,9 +10,11 @@ #ifndef MGMOL_PROTOTYPES_H #define MGMOL_PROTOTYPES_H +#include "mgmol_config.h" #include "global.h" #include +namespace po = boost::program_options; class Ions; class KBPsiMatrixSparse; @@ -24,4 +26,8 @@ int read_config(int argc, char** argv, boost::program_options::variables_map& vm, std::string& input_file, std::string& lrs_filename, std::string& constraints_filename, float& total_spin, bool& with_spin, bool& tcheck); +#ifdef MGMOL_HAS_LIBROM +void setupROMConfigOption(po::options_description &rom_cfg); +#endif + #endif diff --git a/src/md.cc b/src/md.cc index c01589e5..089deb82 100644 --- a/src/md.cc +++ b/src/md.cc @@ -640,6 +640,18 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) count++; } +#ifdef MGMOL_HAS_LIBROM + // Save orbital snapshots + if (ct.getROMOptions().save_librom_snapshot > 0) + { + int ierr = save_orbital_snapshot( + ct.md_print_filename + "_mdstep" + std::to_string(mdstep), **orbitals); + + if (ierr < 0) + os_ << "WARNING md(): writing ROM snapshot data failed!!!" << std::endl; + } +#endif + printWithTimeStamp("dumped restart file...", std::cout); } diff --git a/src/mgmol_config.h.in b/src/mgmol_config.h.in new file mode 100644 index 00000000..0b099f6c --- /dev/null +++ b/src/mgmol_config.h.in @@ -0,0 +1,19 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +// Description: A configuration header for mgmol. +// + +#ifndef INCLUDED_MGMOL_CONFIG_H +#define INCLUDED_MGMOL_CONFIG_H + +/* Have google test library. */ +#cmakedefine MGMOL_HAS_LIBROM + +#endif diff --git a/src/read_config.cc b/src/read_config.cc index 534981d8..6ce97648 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -14,6 +14,7 @@ #include #include #include +#include "MGmol_prototypes.h" #include namespace po = boost::program_options; @@ -201,6 +202,10 @@ int read_config(int argc, char** argv, po::variables_map& vm, po::value()->default_value(""), "Output file for dumping cluster information in vtk format"); +#ifdef MGMOL_HAS_LIBROM + setupROMConfigOption(config); +#endif + // Hidden options, will be allowed in config file, but will not be // shown to the user. po::options_description hidden("Hidden options"); @@ -409,3 +414,22 @@ int read_config(int argc, char** argv, po::variables_map& vm, return 0; } + +#ifdef MGMOL_HAS_LIBROM +void setupROMConfigOption(po::options_description &rom_cfg) +{ + rom_cfg.add_options() + ("ROM.stage", po::value()->default_value("none"), + "ROM workflow stage: offline; build; online; none.") + ("ROM.offline.restart_filefmt", po::value()->default_value(""), + "File name format to read for snapshots.") + ("ROM.offline.restart_min_idx", po::value()->default_value(-1), + "Minimum index for snapshot file format.") + ("ROM.offline.restart_max_idx", po::value()->default_value(-1), + "Maximum index for snapshot file format.") + ("ROM.offline.basis_file", po::value()->default_value(""), + "File name for libROM snapshot/POD matrices.") + ("ROM.offline.save_librom_snapshot", po::value()->default_value(false), + "Save libROM snapshot file at FOM simulation."); +} +#endif \ No newline at end of file diff --git a/src/rom.cc b/src/rom.cc new file mode 100644 index 00000000..a554d2cf --- /dev/null +++ b/src/rom.cc @@ -0,0 +1,63 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "mgmol_config.h" +#ifdef MGMOL_HAS_LIBROM + +#include "LocGridOrbitals.h" +#include "MGmol.h" + +#include "librom.h" + +#include +#include +#include +#include + +// Save the wavefunction snapshots +template +int MGmol::save_orbital_snapshot(std::string file_path, OrbitalsType& orbitals) +{ + std::string snapshot_filename = file_path; + struct stat s; + if (stat(file_path.c_str(), &s) == 0) + { + if (s.st_mode & S_IFDIR) + { + snapshot_filename = file_path + "/orbital"; + } + else if (s.st_mode & S_IFREG) + { + snapshot_filename = file_path + "_orbital"; + } + else + { + std::cout << file_path << " exists but is not a directory or a file." << std::endl; + return 1; + } + } + + const int dim = orbitals.getLocNumpt(); + const int totalSamples = orbitals.chromatic_number(); + + CAROM::Options svd_options(dim, totalSamples, 1); + CAROM::BasisGenerator basis_generator(svd_options, false, snapshot_filename); + + for (int i = 0; i < totalSamples; ++i) + basis_generator.takeSample(orbitals.getPsi(i)); + + basis_generator.writeSnapshot(); + + return 0; +} + +template class MGmol; +template class MGmol; + +#endif // MGMOL_HAS_LIBROM diff --git a/src/rom_Control.h b/src/rom_Control.h new file mode 100644 index 00000000..d1fd6854 --- /dev/null +++ b/src/rom_Control.h @@ -0,0 +1,42 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#ifndef ROM_CONTROL_H +#define ROM_CONTROL_H + +#include +#include +#include +#include +#include + +enum class ROMStage +{ + OFFLINE, + ONLINE, + RESTORE, // TODO(kevin): what stage is this? + BUILD, + UNSUPPORTED +}; + +/* Stored as a private member variable of Control class */ +struct ROMPrivateOptions +{ + ROMStage rom_stage = ROMStage::UNSUPPORTED; + + std::string restart_file_fmt = ""; + int restart_file_minidx = -1; + int restart_file_maxidx = -1; + std::string basis_file = ""; + + /* save librom snapshot matrix at FOM simulation. */ + bool save_librom_snapshot = false; +}; + +#endif // ROM_CONTROL_H diff --git a/src/rom_main.cc b/src/rom_main.cc new file mode 100644 index 00000000..dc877a58 --- /dev/null +++ b/src/rom_main.cc @@ -0,0 +1,141 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +// +// main.cc +// +// Description: +// Real grid, finite difference, molecular dynamics program +// for with nonorthogonal localized orbitals. +// +// Uses Mehrstellen operators, multigrid accelerations, and +// non-local pseudopotentials. +// +// Includes LDA and PBE exchange and correlation functionals. +// +// Units: +// Potentials, eigenvalues and operators in Rydberg +// Energies in Hartree +// +#include "rom_workflows.h" + +// A helper function +template +std::ostream& operator<<(std::ostream& os, const std::vector& v) +{ + copy(v.begin(), v.end(), std::ostream_iterator(std::cout, " ")); + return os; +} + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + mgmol_init(comm); + + // read runtime parameters + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + bool tcheck = false; + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // use configure file if it can be found + // std::string config_filename("mgmol.cfg"); + + // read options from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin, tcheck); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + // Enter main scope + { + MGmolInterface* mgmol; + if (ct.isLocMode()) + mgmol = new MGmol(global_comm, *MPIdata::sout); + else + mgmol + = new MGmol(global_comm, *MPIdata::sout); + + unsigned ngpts[3] = { ct.ngpts_[0], ct.ngpts_[1], ct.ngpts_[2] }; + double origin[3] = { ct.ox_, ct.oy_, ct.oz_ }; + const double cell[3] = { ct.lx_, ct.ly_, ct.lz_ }; + Mesh::setup(mmpi.commSpin(), ngpts, origin, cell, ct.lap_type); + + mgmol->setupFromInput(input_filename); + + if (ct.isLocMode() || ct.init_loc == 1) mgmol->setupLRs(lrs_filename); + + mgmol->setupConstraintsFromInput(constraints_filename); + + mgmol_setup(); + + if (!tcheck) + { + mgmol->setup(); + + if (ct.isLocMode()) + readRestartFiles(mgmol); + else + readRestartFiles(mgmol); + } + else + { + *MPIdata::sout << " Input parameters OK\n"; + } + delete mgmol; + + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + time_t tt; + time(&tt); + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; + + // MemTrack::TrackDumpBlocks(); + + // MemTrack::TrackListMemoryUsage(); + + return 0; +} diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc new file mode 100644 index 00000000..4594dd3a --- /dev/null +++ b/src/rom_workflows.cc @@ -0,0 +1,71 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "rom_workflows.h" +#include +#include +#include + +template +std::string string_format( const std::string& format, Args ... args ) +{ + int size_s = std::snprintf( nullptr, 0, format.c_str(), args ... ) + 1; // Extra space for '\0' + if( size_s <= 0 ){ throw std::runtime_error( "Error during formatting." ); } + auto size = static_cast( size_s ); + std::unique_ptr buf( new char[ size ] ); + std::snprintf( buf.get(), size, format.c_str(), args ... ); + return std::string( buf.get(), buf.get() + size - 1 ); // We don't want the '\0' inside +} + +template +void readRestartFiles(MGmolInterface *mgmol_) +{ + Control& ct = *(Control::instance()); + ROMPrivateOptions rom_options = ct.getROMOptions(); + assert(rom_options.restart_file_minidx >= 0); + assert(rom_options.restart_file_maxidx >= 0); + const int minidx = rom_options.restart_file_minidx; + const int maxidx = rom_options.restart_file_maxidx; + const int num_restart = maxidx - minidx + 1; + + MGmol *mgmol = static_cast *>(mgmol_); + OrbitalsType *orbitals = nullptr; + std::string filename; + + /* Read the first snapshot to determin dimension and number of snapshots */ + filename = string_format(rom_options.restart_file_fmt, minidx); + orbitals = mgmol->loadOrbitalFromRestartFile(filename); + const int dim = orbitals->getLocNumpt(); + const int chrom_num = orbitals->chromatic_number(); + const int totalSamples = orbitals->chromatic_number() * num_restart; + delete orbitals; + + /* Initialize libROM classes */ + CAROM::Options svd_options(dim, totalSamples, 1); + CAROM::BasisGenerator basis_generator(svd_options, false, rom_options.basis_file); + + /* Collect the restart files */ + for (int k = minidx; k <= maxidx; k++) + { + filename = string_format(rom_options.restart_file_fmt, k); + orbitals = mgmol->loadOrbitalFromRestartFile(filename); + assert(dim == orbitals->getLocNumpt()); + assert(chrom_num == orbitals->chromatic_number()); + + for (int i = 0; i < chrom_num; ++i) + basis_generator.takeSample(orbitals->getPsi(i)); + + delete orbitals; + } + basis_generator.writeSnapshot(); + basis_generator.endSamples(); +} + +template void readRestartFiles(MGmolInterface *mgmol_); +template void readRestartFiles(MGmolInterface *mgmol_); \ No newline at end of file diff --git a/src/rom_workflows.h b/src/rom_workflows.h new file mode 100644 index 00000000..dfb22b79 --- /dev/null +++ b/src/rom_workflows.h @@ -0,0 +1,41 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#ifndef ROM_WORKFLOWS_H +#define ROM_WORKFLOWS_H + +#include "Control.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "Mesh.h" +#include "mgmol_run.h" +#include "tools.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +namespace po = boost::program_options; + +#include "librom.h" + +template +void readRestartFiles(MGmolInterface *mgmol_); + +#endif // ROM_WORKFLOWS_H