Merge branch 'ROMFPMD' into romfpmd-pr246

.github/workflows/ci.yml

@@ -0,0 +1,72 @@
+name: CI
+on:
+  workflow_dispatch: {}
+  pull_request:
+    types: [opened, labeled, synchronize]
+    branches:
+      - 'ROMFPMD'
+  # push:
+  #   branches:
+  #     - release
+
+jobs:
+  docker-image:
+    uses: ./.github/workflows/docker_image.yml
+  build:
+    runs-on: ubuntu-latest
+    needs: [docker-image]
+    container:
+      image: ghcr.io/llnl/mgmol/mgmol_env:latest
+      options: --user 1001 --privileged
+      volumes:
+        - /mnt:/mnt
+    steps:
+      - name: Cancel previous runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+      # - name: Set Swap Space
+      #   uses: pierotofy/set-swap-space@master
+      #   with:
+      #     swap-size-gb: 10
+      - name: Check out mgmol
+        uses: actions/checkout@v1
+        with:
+            submodules: 'true'
+      - name: cmake
+        run: |
+            mkdir build
+            cd build
+            cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMPIEXEC_PREFLAGS="--oversubscribe"
+      - name: make
+        run: |
+            cd build && make -j 4
+      - name: test
+        run: |
+            cd build && ctest --no-compress-output -V -T Test -I 1,20,1
+  # code-style:
+  #   runs-on: ubuntu-latest
+  #   needs: [docker-image]
+  #   container:
+  #     image: ghcr.io/llnl/mgmol/mgmol_env:latest
+  #     options: --user 1001 --privileged
+  #     volumes:
+  #       - /mnt:/mnt
+  #   steps:
+  #     - name: Cancel previous runs
+  #       uses: styfle/cancel-workflow-action@0.11.0
+  #       with:
+  #         access_token: ${{ github.token }}
+  #     - name: Check out mgmol
+  #       uses: actions/checkout@v1
+  #       with:
+  #           submodules: 'true'
+  #     - name: cmake
+  #       run: |
+  #           mkdir build
+  #           cd build
+  #           cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMGMOL_WITH_CLANG_FORMAT=ON
+  #     - name: make
+  #       run: |
+  #           cd build && make format
+

.github/workflows/docker_image.yml

@@ -0,0 +1,59 @@
+name: docker-image
+on:
+  workflow_call:
+
+env:
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: llnl/mgmol/mgmol_env
+  DOCKERPATH: docker
+
+jobs:
+  docker-ci:
+    runs-on: ubuntu-latest
+    name: "docker env"
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: Ana06/get-changed-files@v2.2.0       
+        id: files
+      - name: DockerPATH configuration
+        run: echo "DOCKERPATH=$DOCKERPATH"
+      - name: DockerPATH - check if files in docker path changed
+        if: contains(steps.files.outputs.all,env.DOCKERPATH) || contains(steps.files.outputs.all,'docker_image.yml')
+        run: |
+          echo "CI container needs rebuilding..."
+          echo "CI_NEEDS_REBUILD=true" >> $GITHUB_ENV
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: env.CI_NEEDS_REBUILD
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        if: env.CI_NEEDS_REBUILD
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: type=sha
+          flavor: latest=true
+      - name: Build Container motd
+        if: env.CI_NEEDS_REBUILD
+        run: |
+          echo "#!/bin/bash" > ${{env.DOCKERPATH}}/motd.sh
+          echo "echo --------------------------" >> ${{env.DOCKERPATH}}/motd.sh
+          echo "echo mgmol_env/CI Development Container"  >> ${{env.DOCKERPATH}}/motd.sh
+          echo "echo \"Revision: `echo ${GITHUB_SHA} | cut -c1-8`\"" >> ${{env.DOCKERPATH}}/motd.sh
+          echo "echo --------------------------" >> ${{env.DOCKERPATH}}/motd.sh
+          chmod 755 ${{env.DOCKERPATH}}/motd.sh
+          cat ${{env.DOCKERPATH}}/motd.sh
+      - name: Docker Image - Build and push
+        if: env.CI_NEEDS_REBUILD
+        uses: docker/build-push-action@v3
+        with:
+          push: true
+          context: ${{ env.DOCKERPATH }}
+          tags: ${{ steps.meta.outputs.tags }}

.gitignore

@@ -0,0 +1 @@
+src/mgmol_config.h

CMakeLists.txt

@@ -126,6 +126,22 @@ if (${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT)
   endif(${SCALAPACK_FOUND})  
 endif(${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT)
 
+# libROM (optional)
+set(USE_LIBROM False CACHE BOOL "Build with libROM")
+set(LIBROM_PATH "" CACHE STRING "Path of libROM")
+if(USE_LIBROM)
+  message(STATUS "LIBROM_PATH: ${LIBROM_PATH}")
+  if(NOT LIBROM_PATH)
+    message(FATAL_ERROR "Cmake is asked to use libROM, but LIBROM_PATH not specified.")
+  endif(NOT LIBROM_PATH)
+
+  find_package(libROM REQUIRED)
+
+  if(libROM_FOUND)
+    set(MGMOL_HAS_LIBROM 1)
+  endif(libROM_FOUND)
+endif(USE_LIBROM)
+
 # ARPACK (optional)
 set(MGMOL_WITH_ARPACK FALSE CACHE BOOL "Compile with ARPACK package")
 if(${MGMOL_WITH_ARPACK} OR DEFINED ARPACK_ROOT)
@@ -249,6 +265,9 @@ include_directories("${PROJECT_SOURCE_DIR}/src/sparse_linear_algebra")
 include_directories("${PROJECT_SOURCE_DIR}/src/tools")
 include_directories("${PROJECT_SOURCE_DIR}/src")
 
+include_directories("${LIBROM_PATH}/lib")
+link_libraries(${LIBROM_LIB})
+
 # add subdirectories for source files, tests
 add_subdirectory(src)
 

cmake_modules/FindlibROM.cmake

@@ -0,0 +1,11 @@
+if(NOT LIBROM_PATH)
+    message(FATAL_ERROR "LIBROM_PATH not specified.")
+endif(NOT LIBROM_PATH)
+
+find_library(LIBROM_LIB libROM.so HINTS "${LIBROM_PATH}/build/lib")
+find_path(LIBROM_INCLUDES librom.h HINTS "${LIBROM_PATH}/lib")
+
+mark_as_advanced(LIBROM_LIB LIBROM_INCLUDES)
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(libROM REQUIRED_VARS LIBROM_LIB LIBROM_INCLUDES)

cmake_toolchains/quartz.default.cmake

@@ -0,0 +1,6 @@
+set(CMAKE_C_COMPILER mpicc)
+set(CMAKE_CXX_COMPILER mpicxx)
+set(CMAKE_Fortran_COMPILER mpif90)
+
+set(SCALAPACK_ROOT $ENV{MKLROOT})
+set(SCALAPACK_BLACS_LIBRARY $ENV{MKLROOT}/lib/intel64/libmkl_blacs_intelmpi_lp64.so)

docker/Dockerfile

@@ -0,0 +1,50 @@
+FROM ubuntu:22.04
+
+ENV ENVDIR=env
+
+# install sudo
+RUN apt-get -yq update && apt-get -yq install sudo
+
+WORKDIR /$ENVDIR
+
+# install packages
+RUN sudo apt-get install -yq git
+RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake
+RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpi-dev
+RUN sudo apt-get install -yq libboost-all-dev
+RUN sudo apt-get install -yq vim
+RUN sudo apt-get install -yq git-lfs
+RUN sudo apt-get install -yq valgrind hdf5-tools
+RUN sudo apt-get install -yq wget
+### clang-format seems to be updated to 14.0. Not using it for now.
+# RUN sudo apt-get install -yq clang-format
+
+# install lldb and gdb for debugging
+RUN sudo apt-get install -yq lldb gdb
+
+RUN sudo apt-get clean -q
+
+ENV LIB_DIR=/$ENVDIR/dependencies
+WORKDIR $LIB_DIR
+
+# cmake toolchain file for librom
+RUN echo 'set(CMAKE_C_COMPILER mpicc)\n\
+set(CMAKE_CXX_COMPILER mpicxx)\n\
+set(CMAKE_Fortran_COMPILER mpif90)' > ./librom_env.cmake
+ENV TOOLCHAIN_FILE=$LIB_DIR/librom_env.cmake
+
+# install libROM for scaleupROM
+RUN sudo git clone https://github.com/LLNL/libROM.git
+WORKDIR ./libROM/build
+# libROM without MFEM.
+RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=Optimized -DUSE_MFEM=OFF
+RUN sudo make -j 16
+
+# create and switch to a user
+ENV USERNAME=test
+RUN echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+RUN useradd --no-log-init -u 1001 --create-home --shell /bin/bash $USERNAME
+RUN adduser $USERNAME sudo
+USER $USERNAME
+WORKDIR /home/$USERNAME
+

examples/Carbyne/carbyne.rom.cfg

@@ -0,0 +1,34 @@
+verbosity=2
+xcFunctional=PBE
+FDtype=4th
+[Mesh]
+nx= 96
+ny= 96
+nz= 192
+[Domain]
+ox= -10.
+oy= -10.
+oz= -20.
+lx= 20.
+ly= 20.
+lz= 40.
+[Potentials]
+pseudopotential=pseudo.H_ONCV_PBE_SG15
+pseudopotential=pseudo.C_ONCV_PBE_SG15
+[Run]
+type=QUENCH
+[Quench]
+max_steps=5
+atol=1.e-8
+[Orbitals]
+initial_type=Fourier
+[Restart]
+output_level=4
+input_level=4
+input_filename=snapshot0_000
+
+[ROM.offline]
+restart_filefmt=snapshot0_%03d
+restart_min_idx=0
+restart_max_idx=1
+basis_file=carom

scripts/build_quartz_libROM.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+## An example script to build on LLNL Peloton systems.
+## For now, this script assumes intel/ mkl libraries are being used.
+
+## load some modules
+source scripts/modules.quartz
+
+## set some environment variables. Set them explicitly or use loaded module path (preferred)
+## Here we use an explicit path for scalapack to be consistent with the path for the blas libraries and avoid
+## benign cmake warnings
+##setenv SCALAPACK_ROOT /usr/tce/packages/mkl/mkl-2020.0/lib
+#setenv SCALAPACK_ROOT ${MKLROOT}
+#setenv HDF5_ROOT /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0
+#
+## We need to define the cmake blas vendor option here to find the right one.
+#set BLAS_VENDOR = Intel10_64lp
+#
+## manually set the location of BLACS libraries for scalapack
+#set BLACS_LIB = ${SCALAPACK_ROOT}/lib/intel64
+
+MGMOL_ROOT="$(pwd)"
+
+INSTALL_DIR=${MGMOL_ROOT}/install_quartz
+mkdir -p ${INSTALL_DIR}
+
+BUILD_DIR=${MGMOL_ROOT}/build_quartz
+mkdir -p ${BUILD_DIR}
+cd ${BUILD_DIR}
+
+# clone the libROM GitHub repo in BUILD_DIR
+USE_LIBROM="On"
+LIBROM_PATH=${BUILD_DIR}/libROM
+git clone https://github.com/LLNL/libROM
+cd libROM
+#./scripts/compile.sh -t ./cmake/toolchains/default-toss_4_x86_64_ib-librom-dev.cmake
+./scripts/compile.sh
+cd ${BUILD_DIR}
+
+# call cmake
+cmake -DCMAKE_TOOLCHAIN_FILE=${MGMOL_ROOT}/cmake_toolchains/quartz.default.cmake \
+      -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
+      -DUSE_LIBROM=${USE_LIBROM} \
+      -DLIBROM_PATH=${LIBROM_PATH} \
+      ..
+
+#      -DCMAKE_CXX_COMPILER=mpic++ \
+#      -DCMAKE_Fortran_COMPILER=mpif77 \
+#      -DMPIEXEC_NUMPROC_FLAG="-n" \
+#      -DBLA_VENDOR=${BLAS_VENDOR} \
+#      -DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_intelmpi_lp64.so \
+#      -DCMAKE_BUILD_TYPE=DEBUG \
+
+# call make install
+make -j 16
+### Currently libROM does not have the installation procedure,
+### so copying binary file to installation directory will disrupt the relative path to libROM.so,
+### causing a run-time error.
+#make install
+
+#      -DBLAS_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \
+#      -DLAPACK_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \  

scripts/build_ubuntu22_openmpi.sh

@@ -25,7 +25,7 @@ cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
       -DMPIEXEC_PREFLAGS="--oversubscribe" \
       -DMGMOL_WITH_CLANG_FORMAT=ON \
       -DCMAKE_PREFIX_PATH=${HOME}/bin \
-      -D CMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \
+      -DCMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \
       ..
 
 # call make install

scripts/modules.quartz

@@ -0,0 +1,21 @@
+### choose either gcc or intel
+#module load intel/2022.1.0
+module load gcc/11.2.1
+
+module load cmake
+module load hdf5-parallel/1.14.0
+module load boost
+
+### choose either one
+module load mkl-interfaces
+#module load mkl
+
+module load python
+
+### manually add boost path
+#setenv LD_LIBRARY_PATH /usr/tce/packages/boost/boost-1.80.0-mvapich2-2.3.6-gcc-10.3.1/lib:$LD_LIBRARY_PATH
+
+#setenv MKLROOT $LIBRARY_PATH
+#setenv MKLROOT /usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0
+#setenv HDF5ROOT $LD_LIBRARY_PATH
+#setenv HDF5ROOT