Rebase dipole

CMakeLists.txt

@@ -131,14 +131,30 @@ set (MGMOL_WITH_SCALAPACK True CACHE BOOL "Build with SCALAPACK")
 if (${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT)
   find_package(SCALAPACK)
   if(${SCALAPACK_FOUND})
-    add_definitions(-DMGMOL_USE_SCALAPACK)
+    add_definitions(-DSCALAPACK)
     message(STATUS "SCALAPACK_INCLUDE_DIRS: ${SCALAPACK_INCLUDE_DIRS}")
     message(STATUS "SCALAPACK_LIBARIES: ${SCALAPACK_LIBRARIES}")
   else(${SCALAPACK_FOUND})
     message(FATAL_ERROR "Required SCALAPACK package not found.")
   endif(${SCALAPACK_FOUND})  
 endif(${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT)
 
+# libROM (optional)
+set(USE_LIBROM False CACHE BOOL "Build with libROM")
+set(LIBROM_PATH "" CACHE STRING "Path of libROM")
+if(USE_LIBROM)
+  message(STATUS "LIBROM_PATH: ${LIBROM_PATH}")
+  if(NOT LIBROM_PATH)
+    message(FATAL_ERROR "Cmake is asked to use libROM, but LIBROM_PATH not specified.")
+  endif(NOT LIBROM_PATH)
+
+  find_package(libROM REQUIRED)
+
+  if(libROM_FOUND)
+    add_definitions(-DMGMOL_HAS_LIBROM)
+  endif(libROM_FOUND)
+endif(USE_LIBROM)
+
 # ARPACK (optional)
 set(MGMOL_WITH_ARPACK FALSE CACHE BOOL "Compile with ARPACK package")
 if(${MGMOL_WITH_ARPACK} OR DEFINED ARPACK_ROOT)
@@ -220,8 +236,7 @@ FortranCInterface_HEADER(
         DGETRF DGETRS DLACPY
 )
 
-if(${SCALAPACK_FOUND})
-  FortranCInterface_HEADER(
+FortranCInterface_HEADER(
     scalapack_mangle.h
     MACRO_NAMESPACE "FC_SCALAPACK_"
     SYMBOLS
@@ -234,8 +249,7 @@ if(${SCALAPACK_FOUND})
         pdtrtri pstrtri pdpocon pspocon pdsygst pssygst pdsyev pssyev
         pdelset pselset pdelget pselget pdlatra pslatra pdlaset pslaset pdgesvd psgesvd
         pdamax psamax
-  )
-endif(${SCALAPACK_FOUND})
+)
 
 FortranCInterface_HEADER(
     arpack_mangle.h
@@ -265,6 +279,9 @@ include_directories("${PROJECT_SOURCE_DIR}/src/sparse_linear_algebra")
 include_directories("${PROJECT_SOURCE_DIR}/src/tools")
 include_directories("${PROJECT_SOURCE_DIR}/src")
 
+include_directories("${LIBROM_PATH}/lib")
+link_libraries(${LIBROM_LIB})
+
 # add subdirectories for source files, tests
 add_subdirectory(src)
 

cmake_modules/FindlibROM.cmake

@@ -0,0 +1,11 @@
+if(NOT LIBROM_PATH)
+    message(FATAL_ERROR "LIBROM_PATH not specified.")
+endif(NOT LIBROM_PATH)
+
+find_library(LIBROM_LIB libROM.so HINTS "${LIBROM_PATH}/build/lib")
+find_path(LIBROM_INCLUDES librom.h HINTS "${LIBROM_PATH}/lib")
+
+mark_as_advanced(LIBROM_LIB LIBROM_INCLUDES)
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(libROM REQUIRED_VARS LIBROM_LIB LIBROM_INCLUDES)

cmake_toolchains/quartz.default.cmake

@@ -0,0 +1,6 @@
+set(CMAKE_C_COMPILER mpicc)
+set(CMAKE_CXX_COMPILER mpicxx)
+set(CMAKE_Fortran_COMPILER mpif90)
+
+set(SCALAPACK_ROOT $ENV{MKLROOT})
+set(SCALAPACK_BLACS_LIBRARY $ENV{MKLROOT}/lib/intel64/libmkl_blacs_intelmpi_lp64.so)

docker/Dockerfile

@@ -0,0 +1,50 @@
+FROM ubuntu:22.04
+
+ENV ENVDIR=env
+
+# install sudo
+RUN apt-get -yq update && apt-get -yq install sudo
+
+WORKDIR /$ENVDIR
+
+# install packages
+RUN sudo apt-get install -yq git
+RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake
+RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpi-dev
+RUN sudo apt-get install -yq libboost-all-dev
+RUN sudo apt-get install -yq vim
+RUN sudo apt-get install -yq git-lfs
+RUN sudo apt-get install -yq valgrind hdf5-tools
+RUN sudo apt-get install -yq wget
+### clang-format seems to be updated to 14.0. Not using it for now.
+# RUN sudo apt-get install -yq clang-format
+
+# install lldb and gdb for debugging
+RUN sudo apt-get install -yq lldb gdb
+
+RUN sudo apt-get clean -q
+
+ENV LIB_DIR=/$ENVDIR/dependencies
+WORKDIR $LIB_DIR
+
+# cmake toolchain file for librom
+RUN echo 'set(CMAKE_C_COMPILER mpicc)\n\
+set(CMAKE_CXX_COMPILER mpicxx)\n\
+set(CMAKE_Fortran_COMPILER mpif90)' > ./librom_env.cmake
+ENV TOOLCHAIN_FILE=$LIB_DIR/librom_env.cmake
+
+# install libROM for scaleupROM
+RUN sudo git clone https://github.com/LLNL/libROM.git
+WORKDIR ./libROM/build
+# libROM without MFEM.
+RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=Optimized -DUSE_MFEM=OFF
+RUN sudo make -j 16
+
+# create and switch to a user
+ENV USERNAME=test
+RUN echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+RUN useradd --no-log-init -u 1001 --create-home --shell /bin/bash $USERNAME
+RUN adduser $USERNAME sudo
+USER $USERNAME
+WORKDIR /home/$USERNAME
+

examples/Carbyne/carbyne.rom.cfg

@@ -0,0 +1,34 @@
+verbosity=2
+xcFunctional=PBE
+FDtype=4th
+[Mesh]
+nx= 96
+ny= 96
+nz= 192
+[Domain]
+ox= -10.
+oy= -10.
+oz= -20.
+lx= 20.
+ly= 20.
+lz= 40.
+[Potentials]
+pseudopotential=pseudo.H_ONCV_PBE_SG15
+pseudopotential=pseudo.C_ONCV_PBE_SG15
+[Run]
+type=QUENCH
+[Quench]
+max_steps=5
+atol=1.e-8
+[Orbitals]
+initial_type=Fourier
+[Restart]
+output_level=4
+input_level=4
+input_filename=snapshot0_000
+
+[ROM.offline]
+restart_filefmt=snapshot0_%03d
+restart_min_idx=0
+restart_max_idx=1
+basis_file=carom

examples/PinnedH2O/coords_test1.in

@@ -0,0 +1,3 @@
+O1  1   0.00    0.00    0.00  0
+H1  2  -0.45    1.42   -1.07  1
+H2  2  -0.45   -1.48   -0.97  1

examples/PinnedH2O/coords_test2.in

@@ -0,0 +1,3 @@
+O1  1   0.00    0.00    0.00  0
+H1  2  -0.45    1.57   -1.07  1
+H2  2  -0.45   -1.48   -0.97  1

examples/PinnedH2O/get_ROM_table.py

@@ -0,0 +1,28 @@
+import subprocess
+import re
+
+pattern = r"For energy fraction: \d+\.\d+, take first (\d+) of \d+ basis vectors"
+
+print("\\begin{tabular}{|c||c|c|c|c|c|c|c|}")
+print("\\hline")
+print("$k$ & $\\varepsilon = 10^{-1}$ & $\\varepsilon = 10^{-2}$ & $\\varepsilon = 10^{-3}$ & $\\varepsilon = 10^{-4}$ & $\\varepsilon = 10^{-5}$ & Snapshots \\\\")
+print("\\hline")
+
+for t in range(10):
+    k = 50*(t+1)
+    snapshots = 4*k
+    grep_command = f"grep 'take first' basis_1_{k}_Pinned_H2O.out"
+    result = subprocess.run(grep_command, shell=True, capture_output=True, text=True)
+    matches = re.findall(pattern, result.stdout)
+    energy_fractions = {
+        "0.9": matches[0],
+        "0.99": matches[1],
+        "0.999": matches[2],
+        "0.9999": matches[3],
+        "0.99999": matches[4],
+    }
+    line = f"{k} & {energy_fractions['0.9']} & {energy_fractions['0.99']} & {energy_fractions['0.999']} & {energy_fractions['0.9999']} & {energy_fractions['0.99999']} & {snapshots} \\\\"
+    print(line)
+
+print("\\hline")
+print("\\end{tabular}")

examples/PinnedH2O/get_result.sh

@@ -0,0 +1,37 @@
+#filename="offline_PinnedH2O" # FOM
+#filename="rom39_PinnedH2O" # ROM compare MD
+#filename="39_force_PinnedH2O" # ROM compare force
+
+#filename="PinnedH2O_test2_ref" # FOM
+filename="PinnedH2O_rom_3DOF_test2_2_2_34" # ROM PinnedH2O 3DOF MD
+
+# Extracting kinetic energy, total energy, temperature from MGmgol output log
+awk '/Kinetic/ {print $3}' $filename.out > ke_$filename.txt
+awk '/Kinetic/ {print $5}' $filename.out >temp_$filename.txt
+awk '/Total/ {print $3}' $filename.out > te_$filename.txt
+
+# Extracting H1, H2, F1, F2 from MGmgol output log
+# if FOM, these files contain the FOM results
+# if compare MD, these files contain the results with projected orbitals
+awk '/O1 / {print $4, $5, $6}' $filename.out > O1_$filename.txt
+awk '/H1 / {print $3, $4, $5}' $filename.out > H1_$filename.txt
+awk '/H2 / {print $3, $4, $5}' $filename.out > H2_$filename.txt
+awk '/O1 / {print $7, $8, $9}' $filename.out > f_O1_$filename.txt
+awk '/H1 / {print $6, $7, $8}' $filename.out > f_H1_$filename.txt
+awk '/H2 / {print $6, $7, $8}' $filename.out > f_H2_$filename.txt
+
+# if compare force, files with "_fom" contain the FOM results
+# files with "_rom" contain the results with projected orbitals
+if [[ "$filename" == *"force_"* ]]; then
+    sed -n '1~2p' H1_$filename.out > H1_rom$filename.txt
+    sed -n '1~2p' H2_$filename.out > H2_rom$filename.txt
+    sed -n '1~2p' f_H1_$filename.out > f_H1_rom$filename.txt
+    sed -n '1~2p' f_H2_$filename.out > f_H2_rom$filename.txt
+
+    sed -n '2~2p' H1_$filename.out > H1_fom$filename.txt
+    sed -n '2~2p' H2_$filename.out > H2_fom$filename.txt
+    sed -n '2~2p' f_H1_$filename.out > f_H1_fom$filename.txt
+    sed -n '2~2p' f_H2_$filename.out > f_H2_fom$filename.txt
+fi
+
+rm -rf snapshot_*

examples/PinnedH2O/job.basis_1_50

@@ -0,0 +1,32 @@
+#!/bin/tcsh
+#SBATCH -N 1
+#SBATCH -t 0:10:00
+#SBATCH -p pdebug
+
+date
+
+setenv OMP_NUM_THREADS 1
+#setenv KMP_DETERMINISTIC_REDUCTION 1
+
+set ncpus = 1
+
+set maindir = /p/lustre2/cheung26/mgmol
+
+setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH
+
+set exe     = ${maindir}/build_quartz/libROM/build/examples/misc/combine_samples
+
+set snapshot_files = ""
+set increment_md_steps = 1
+set num_md_steps = 50
+
+foreach k (`seq $increment_md_steps $increment_md_steps $num_md_steps`)
+    set snapshot_files = "$snapshot_files MD_mdstep${k}_snapshot"
+end
+echo "Snapshot files: $snapshot_files"
+
+set basis_file = "PinnedH2O_orbitals_basis_${increment_md_steps}_${num_md_steps}"
+
+srun -n $ncpus $exe -f $basis_file $snapshot_files > basis_${increment_md_steps}_${num_md_steps}_PinnedH2O.out
+
+date

examples/PinnedH2O/job.offline

@@ -0,0 +1,30 @@
+#!/bin/tcsh
+#SBATCH -N 1
+#SBATCH -t 1:00:00
+#SBATCH -p pdebug
+
+date
+
+setenv OMP_NUM_THREADS 1
+#setenv KMP_DETERMINISTIC_REDUCTION 1
+
+set ncpus = 1
+
+set maindir = /p/lustre2/cheung26/mgmol
+
+setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH
+
+set exe     = mgmol-opt
+
+cp $maindir/install_quartz/bin/$exe .
+
+set cfg_offline = mgmol_offline.cfg
+
+ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 .
+ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 .
+
+source $maindir/scripts/modules.quartz
+
+srun -n $ncpus $exe -c $cfg_offline -i coords.in > offline_PinnedH2O.out
+
+date

examples/PinnedH2O/job.ref

@@ -0,0 +1,36 @@
+#!/bin/tcsh
+#SBATCH -N 1
+#SBATCH -t 1:00:00
+#SBATCH -p pdebug
+
+date
+
+setenv OMP_NUM_THREADS 1
+#setenv KMP_DETERMINISTIC_REDUCTION 1
+
+set ncpus = 1
+set case = 2
+
+set maindir = /p/lustre2/cheung26/mgmol
+
+setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH
+
+set exe     = mgmol-opt
+
+cp $maindir/install_quartz/bin/$exe .
+
+set datadir = $maindir/examples/PinnedH2O
+
+set cfg = mgmol_ref_test${case}.cfg
+cp $datadir/$cfg .
+
+cp $datadir/coords.in .
+
+ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 .
+ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 .
+
+source $maindir/scripts/modules.quartz
+
+srun -n $ncpus $exe -c $cfg -i coords_test${case}.in > PinnedH2O_test${case}_ref.out
+
+date

examples/PinnedH2O/job.rom

@@ -0,0 +1,34 @@
+#!/bin/tcsh
+#SBATCH -N 1
+#SBATCH -t 1:00:00
+#SBATCH -p pdebug
+
+date
+
+setenv OMP_NUM_THREADS 1
+#setenv KMP_DETERMINISTIC_REDUCTION 1
+
+set ncpus = 1
+
+set maindir = /p/lustre2/cheung26/mgmol
+
+setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH
+
+set exe     = mgmol-opt
+
+cp $maindir/install_quartz/bin/$exe .
+
+set increment_md_steps = 1
+set num_md_steps = 50
+set basis_file = PinnedH2O_orbitals_basis_${increment_md_steps}_${num_md_steps}
+
+set cfg_rom = mgmol_rom_${increment_md_steps}_${num_md_steps}.cfg
+
+ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 .
+ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 .
+
+source $maindir/scripts/modules.quartz
+
+srun -n $ncpus $exe -c $cfg_rom -i coords.in > rom_${increment_md_steps}_${num_md_steps}_PinnedH2O.out
+
+date