From 0b145cfef64176ad040ea34a687d2cf0d3ebe6b6 Mon Sep 17 00:00:00 2001 From: Siu Wun Cheung Date: Tue, 12 May 2026 08:52:44 -0700 Subject: [PATCH] Rebase dipole --- CMakeLists.txt | 27 +- cmake_modules/FindlibROM.cmake | 11 + cmake_toolchains/quartz.default.cmake | 6 + docker/Dockerfile | 50 + examples/Carbyne/carbyne.rom.cfg | 34 + examples/PinnedH2O/coords_test1.in | 3 + examples/PinnedH2O/coords_test2.in | 3 + examples/PinnedH2O/get_ROM_table.py | 28 + examples/PinnedH2O/get_result.sh | 37 + examples/PinnedH2O/job.basis_1_50 | 32 + examples/PinnedH2O/job.offline | 30 + examples/PinnedH2O/job.ref | 36 + examples/PinnedH2O/job.rom | 34 + examples/PinnedH2O/job.rom_3DOF | 35 + examples/PinnedH2O/mgmol_offline.cfg | 38 + examples/PinnedH2O/mgmol_ref_test1.cfg | 35 + examples/PinnedH2O/mgmol_ref_test2.cfg | 31 + examples/PinnedH2O/mgmol_rom_1_50.cfg | 42 + examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg | 48 + examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg | 44 + examples/PinnedH2O/plot_PinnedH2O_force.m | 84 ++ examples/PinnedH2O/plot_PinnedH2O_md.m | 59 + examples/PinnedH2O/postprocess_PinnedH2O.py | 27 + scripts/build_quartz_libROM.sh | 62 + scripts/modules.quartz | 21 + src/AOMMprojector.cc | 16 +- src/CMakeLists.txt | 86 +- src/ChebyshevApproximation.cc | 17 +- src/ChebyshevApproximation.h | 10 - src/Control.cc | 128 +++ src/Control.h | 12 + src/DFTsolver.cc | 7 +- src/DMStrategyFactory.cc | 4 - src/DavidsonSolver.cc | 13 +- src/DensityMatrix.cc | 9 +- src/DistMatrix/BlacsContext.cc | 4 +- src/DistMatrix/DistMatrix.cc | 142 +-- src/DistMatrix/DistVector.h | 2 +- src/DistMatrix/MGmol_scalapack.h | 10 +- src/DistMatrix/blacs.h | 4 +- src/EigenDMStrategy.cc | 10 +- src/Electrostatic.h | 1 + src/ExtendedGridOrbitals.cc | 196 ++-- src/ExtendedGridOrbitals.h | 41 +- src/Forces.cc | 2 - src/GramMatrix.cc | 15 +- src/GrassmanCG.h | 5 - src/Hamiltonian.cc | 2 - src/HamiltonianMVPSolver.cc | 8 +- src/HamiltonianMVP_DMStrategy.cc | 12 +- src/Hartree_CG.h | 3 +- src/Ions.cc | 2 + src/KBPsiMatrixSparse.cc | 7 - src/LapFactory.h | 20 +- src/LocGridOrbitals.cc | 71 +- src/LocGridOrbitals.h | 25 +- src/LocalizationRegions.cc | 7 +- src/LocalizationRegions.h | 7 +- src/MGmol.cc | 141 +-- src/MGmol.h | 45 +- src/MGmolInterface.h | 3 + src/MGmol_prototypes.h | 6 + src/MVPSolver.cc | 10 +- src/MVP_DMStrategy.cc | 5 +- src/OrbitalsExtrapolationOrder2.cc | 20 +- src/OrbitalsExtrapolationOrder3.cc | 26 +- src/PBdiel.cc | 27 +- src/PBdiel.h | 6 +- src/PBdiel_CG.cc | 12 +- src/PBdiel_CG.h | 12 +- src/PCGSolver.cc | 17 +- src/PCGSolver.h | 14 +- src/PCGSolver_Diel.cc | 27 +- src/PCGSolver_Diel.h | 11 +- src/Poisson.h | 8 + src/PolakRibiereSolver.cc | 14 +- src/Potentials.cc | 92 ++ src/Potentials.h | 9 + src/Power.cc | 7 - src/Power.h | 1 + src/PowerGen.cc | 9 +- src/ProjectedMatrices.cc | 34 +- src/ProjectedMatrices.h | 10 +- src/ProjectedMatrices2N.cc | 7 +- src/ProjectedMatricesMehrstellen.cc | 7 +- src/ProjectedMatricesMehrstellen.h | 2 +- src/ReplicatedMatrix.cc | 3 +- src/ReplicatedWorkSpace.cc | 2 - src/ReplicatedWorkSpace.h | 7 +- src/Rho.cc | 8 +- src/Rho.h | 2 + src/SP2.cc | 7 +- src/jade.cc | 17 +- src/linear_algebra/gemm_impl.cc | 5 + src/linear_algebra/mputils.cc | 2 + src/local_matrices/LocalVector.h | 1 - src/md.cc | 153 ++- src/mgmol_config.h | 19 + src/mgmol_config.h.in | 19 + src/mgmol_run.cc | 22 +- src/mlwf.cc | 9 +- src/pb/DielFunc.h | 2 +- src/pb/GridFunc.cc | 104 +- src/pb/GridFunc.h | 219 ++-- src/pb/GridFuncInterface.h | 51 + src/pb/GridFuncVector.h | 22 +- src/quench.cc | 25 +- src/read_config.cc | 43 +- src/rom.cc | 127 +++ src/rom_Control.h | 63 + src/rom_main.cc | 180 +++ src/rom_workflows.cc | 1015 +++++++++++++++++ src/rom_workflows.h | 64 ++ src/setup.cc | 4 - src/tools.cc | 7 +- src/tools/CMakeLists.txt | 1 + src/tools/PinnedH2O.cc | 273 +++++ src/tools/PinnedH2O.h | 62 + src/tools/SaveData.h | 6 +- tests/CMakeLists.txt | 134 +-- tests/PinnedH2O_3DOF/README.md | 1 + tests/PinnedH2O_3DOF/assemble_results.sh | 31 + tests/PinnedH2O_3DOF/aux/PinnedH2O_3DOF.png | Bin 0 -> 23439 bytes .../aux/coords_1.00_1.00_0.0.in | 3 + .../aux/coords_1.02_0.98_2.0.in | 3 + tests/PinnedH2O_3DOF/aux/get_ROM_table.py | 31 + .../PinnedH2O_3DOF/aux/plot_PinnedH2O_3DOF.py | 69 ++ tests/PinnedH2O_3DOF/aux/rotation_test.cc | 192 ++++ tests/PinnedH2O_3DOF/aux/rotation_test.py | 99 ++ .../compare_energy_and_forces.py | 233 ++++ tests/PinnedH2O_3DOF/coords_rotate1.in | 3 + tests/PinnedH2O_3DOF/coords_rotate2.in | 3 + tests/PinnedH2O_3DOF/generate_coord.py | 31 + tests/PinnedH2O_3DOF/job.basis | 47 + tests/PinnedH2O_3DOF/job.offline | 59 + tests/PinnedH2O_3DOF/job.online | 67 ++ tests/PinnedH2O_3DOF/job.online_rotate | 36 + tests/PinnedH2O_3DOF/mgmol_offline.cfg | 30 + tests/PinnedH2O_3DOF/mgmol_online.cfg | 36 + tests/PinnedH2O_3DOF/mgmol_online_rotate.cfg | 36 + tests/PinnedH2O_3DOF/mgmol_ref_rotate.cfg | 27 + tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc | 258 +++++ tests/PinnedH2O_3DOF/transfer_nlrom.sh | 20 + tests/ROM/test_rom_poisson/carbyne.in | 14 + tests/ROM/test_rom_poisson/carbyne.ion.cfg | 63 + .../ROM/test_rom_poisson/carbyne.poisson.cfg | 63 + tests/RestartEnergyAndForces/job.test | 28 + tests/WFEnergyAndForces/job.test | 27 + tests/testDensityMatrix.cc | 10 +- tests/testGramMatrix.cc | 10 +- tests/testIons.cc | 73 +- util/compareTimers.py | 59 +- 152 files changed, 5528 insertions(+), 1149 deletions(-) create mode 100644 cmake_modules/FindlibROM.cmake create mode 100644 cmake_toolchains/quartz.default.cmake create mode 100644 docker/Dockerfile create mode 100644 examples/Carbyne/carbyne.rom.cfg create mode 100644 examples/PinnedH2O/coords_test1.in create mode 100644 examples/PinnedH2O/coords_test2.in create mode 100644 examples/PinnedH2O/get_ROM_table.py create mode 100644 examples/PinnedH2O/get_result.sh create mode 100644 examples/PinnedH2O/job.basis_1_50 create mode 100644 examples/PinnedH2O/job.offline create mode 100644 examples/PinnedH2O/job.ref create mode 100644 examples/PinnedH2O/job.rom create mode 100644 examples/PinnedH2O/job.rom_3DOF create mode 100644 examples/PinnedH2O/mgmol_offline.cfg create mode 100644 examples/PinnedH2O/mgmol_ref_test1.cfg create mode 100644 examples/PinnedH2O/mgmol_ref_test2.cfg create mode 100644 examples/PinnedH2O/mgmol_rom_1_50.cfg create mode 100644 examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg create mode 100644 examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg create mode 100644 examples/PinnedH2O/plot_PinnedH2O_force.m create mode 100644 examples/PinnedH2O/plot_PinnedH2O_md.m create mode 100644 examples/PinnedH2O/postprocess_PinnedH2O.py create mode 100644 scripts/build_quartz_libROM.sh create mode 100644 scripts/modules.quartz create mode 100644 src/mgmol_config.h create mode 100644 src/mgmol_config.h.in create mode 100644 src/pb/GridFuncInterface.h create mode 100644 src/rom.cc create mode 100644 src/rom_Control.h create mode 100644 src/rom_main.cc create mode 100644 src/rom_workflows.cc create mode 100644 src/rom_workflows.h create mode 100644 src/tools/PinnedH2O.cc create mode 100644 src/tools/PinnedH2O.h create mode 100644 tests/PinnedH2O_3DOF/README.md create mode 100644 tests/PinnedH2O_3DOF/assemble_results.sh create mode 100644 tests/PinnedH2O_3DOF/aux/PinnedH2O_3DOF.png create mode 100644 tests/PinnedH2O_3DOF/aux/coords_1.00_1.00_0.0.in create mode 100644 tests/PinnedH2O_3DOF/aux/coords_1.02_0.98_2.0.in create mode 100644 tests/PinnedH2O_3DOF/aux/get_ROM_table.py create mode 100644 tests/PinnedH2O_3DOF/aux/plot_PinnedH2O_3DOF.py create mode 100644 tests/PinnedH2O_3DOF/aux/rotation_test.cc create mode 100644 tests/PinnedH2O_3DOF/aux/rotation_test.py create mode 100644 tests/PinnedH2O_3DOF/compare_energy_and_forces.py create mode 100644 tests/PinnedH2O_3DOF/coords_rotate1.in create mode 100644 tests/PinnedH2O_3DOF/coords_rotate2.in create mode 100644 tests/PinnedH2O_3DOF/generate_coord.py create mode 100644 tests/PinnedH2O_3DOF/job.basis create mode 100644 tests/PinnedH2O_3DOF/job.offline create mode 100644 tests/PinnedH2O_3DOF/job.online create mode 100644 tests/PinnedH2O_3DOF/job.online_rotate create mode 100644 tests/PinnedH2O_3DOF/mgmol_offline.cfg create mode 100644 tests/PinnedH2O_3DOF/mgmol_online.cfg create mode 100644 tests/PinnedH2O_3DOF/mgmol_online_rotate.cfg create mode 100644 tests/PinnedH2O_3DOF/mgmol_ref_rotate.cfg create mode 100644 tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc create mode 100644 tests/PinnedH2O_3DOF/transfer_nlrom.sh create mode 100644 tests/ROM/test_rom_poisson/carbyne.in create mode 100644 tests/ROM/test_rom_poisson/carbyne.ion.cfg create mode 100644 tests/ROM/test_rom_poisson/carbyne.poisson.cfg create mode 100644 tests/RestartEnergyAndForces/job.test create mode 100644 tests/WFEnergyAndForces/job.test diff --git a/CMakeLists.txt b/CMakeLists.txt index f87cc11e..23d058f0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -131,7 +131,7 @@ set (MGMOL_WITH_SCALAPACK True CACHE BOOL "Build with SCALAPACK") if (${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT) find_package(SCALAPACK) if(${SCALAPACK_FOUND}) - add_definitions(-DMGMOL_USE_SCALAPACK) + add_definitions(-DSCALAPACK) message(STATUS "SCALAPACK_INCLUDE_DIRS: ${SCALAPACK_INCLUDE_DIRS}") message(STATUS "SCALAPACK_LIBARIES: ${SCALAPACK_LIBRARIES}") else(${SCALAPACK_FOUND}) @@ -139,6 +139,22 @@ if (${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT) endif(${SCALAPACK_FOUND}) endif(${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT) +# libROM (optional) +set(USE_LIBROM False CACHE BOOL "Build with libROM") +set(LIBROM_PATH "" CACHE STRING "Path of libROM") +if(USE_LIBROM) + message(STATUS "LIBROM_PATH: ${LIBROM_PATH}") + if(NOT LIBROM_PATH) + message(FATAL_ERROR "Cmake is asked to use libROM, but LIBROM_PATH not specified.") + endif(NOT LIBROM_PATH) + + find_package(libROM REQUIRED) + + if(libROM_FOUND) + add_definitions(-DMGMOL_HAS_LIBROM) + endif(libROM_FOUND) +endif(USE_LIBROM) + # ARPACK (optional) set(MGMOL_WITH_ARPACK FALSE CACHE BOOL "Compile with ARPACK package") if(${MGMOL_WITH_ARPACK} OR DEFINED ARPACK_ROOT) @@ -220,8 +236,7 @@ FortranCInterface_HEADER( DGETRF DGETRS DLACPY ) -if(${SCALAPACK_FOUND}) - FortranCInterface_HEADER( +FortranCInterface_HEADER( scalapack_mangle.h MACRO_NAMESPACE "FC_SCALAPACK_" SYMBOLS @@ -234,8 +249,7 @@ if(${SCALAPACK_FOUND}) pdtrtri pstrtri pdpocon pspocon pdsygst pssygst pdsyev pssyev pdelset pselset pdelget pselget pdlatra pslatra pdlaset pslaset pdgesvd psgesvd pdamax psamax - ) -endif(${SCALAPACK_FOUND}) +) FortranCInterface_HEADER( arpack_mangle.h @@ -265,6 +279,9 @@ include_directories("${PROJECT_SOURCE_DIR}/src/sparse_linear_algebra") include_directories("${PROJECT_SOURCE_DIR}/src/tools") include_directories("${PROJECT_SOURCE_DIR}/src") +include_directories("${LIBROM_PATH}/lib") +link_libraries(${LIBROM_LIB}) + # add subdirectories for source files, tests add_subdirectory(src) diff --git a/cmake_modules/FindlibROM.cmake b/cmake_modules/FindlibROM.cmake new file mode 100644 index 00000000..9fed8fb6 --- /dev/null +++ b/cmake_modules/FindlibROM.cmake @@ -0,0 +1,11 @@ +if(NOT LIBROM_PATH) + message(FATAL_ERROR "LIBROM_PATH not specified.") +endif(NOT LIBROM_PATH) + +find_library(LIBROM_LIB libROM.so HINTS "${LIBROM_PATH}/build/lib") +find_path(LIBROM_INCLUDES librom.h HINTS "${LIBROM_PATH}/lib") + +mark_as_advanced(LIBROM_LIB LIBROM_INCLUDES) + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(libROM REQUIRED_VARS LIBROM_LIB LIBROM_INCLUDES) \ No newline at end of file diff --git a/cmake_toolchains/quartz.default.cmake b/cmake_toolchains/quartz.default.cmake new file mode 100644 index 00000000..9901dcd6 --- /dev/null +++ b/cmake_toolchains/quartz.default.cmake @@ -0,0 +1,6 @@ +set(CMAKE_C_COMPILER mpicc) +set(CMAKE_CXX_COMPILER mpicxx) +set(CMAKE_Fortran_COMPILER mpif90) + +set(SCALAPACK_ROOT $ENV{MKLROOT}) +set(SCALAPACK_BLACS_LIBRARY $ENV{MKLROOT}/lib/intel64/libmkl_blacs_intelmpi_lp64.so) diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..8b1ffa94 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:22.04 + +ENV ENVDIR=env + +# install sudo +RUN apt-get -yq update && apt-get -yq install sudo + +WORKDIR /$ENVDIR + +# install packages +RUN sudo apt-get install -yq git +RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake +RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpi-dev +RUN sudo apt-get install -yq libboost-all-dev +RUN sudo apt-get install -yq vim +RUN sudo apt-get install -yq git-lfs +RUN sudo apt-get install -yq valgrind hdf5-tools +RUN sudo apt-get install -yq wget +### clang-format seems to be updated to 14.0. Not using it for now. +# RUN sudo apt-get install -yq clang-format + +# install lldb and gdb for debugging +RUN sudo apt-get install -yq lldb gdb + +RUN sudo apt-get clean -q + +ENV LIB_DIR=/$ENVDIR/dependencies +WORKDIR $LIB_DIR + +# cmake toolchain file for librom +RUN echo 'set(CMAKE_C_COMPILER mpicc)\n\ +set(CMAKE_CXX_COMPILER mpicxx)\n\ +set(CMAKE_Fortran_COMPILER mpif90)' > ./librom_env.cmake +ENV TOOLCHAIN_FILE=$LIB_DIR/librom_env.cmake + +# install libROM for scaleupROM +RUN sudo git clone https://github.com/LLNL/libROM.git +WORKDIR ./libROM/build +# libROM without MFEM. +RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=Optimized -DUSE_MFEM=OFF +RUN sudo make -j 16 + +# create and switch to a user +ENV USERNAME=test +RUN echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +RUN useradd --no-log-init -u 1001 --create-home --shell /bin/bash $USERNAME +RUN adduser $USERNAME sudo +USER $USERNAME +WORKDIR /home/$USERNAME + diff --git a/examples/Carbyne/carbyne.rom.cfg b/examples/Carbyne/carbyne.rom.cfg new file mode 100644 index 00000000..cb0cd295 --- /dev/null +++ b/examples/Carbyne/carbyne.rom.cfg @@ -0,0 +1,34 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx= 96 +ny= 96 +nz= 192 +[Domain] +ox= -10. +oy= -10. +oz= -20. +lx= 20. +ly= 20. +lz= 40. +[Potentials] +pseudopotential=pseudo.H_ONCV_PBE_SG15 +pseudopotential=pseudo.C_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +max_steps=5 +atol=1.e-8 +[Orbitals] +initial_type=Fourier +[Restart] +output_level=4 +input_level=4 +input_filename=snapshot0_000 + +[ROM.offline] +restart_filefmt=snapshot0_%03d +restart_min_idx=0 +restart_max_idx=1 +basis_file=carom diff --git a/examples/PinnedH2O/coords_test1.in b/examples/PinnedH2O/coords_test1.in new file mode 100644 index 00000000..307e170c --- /dev/null +++ b/examples/PinnedH2O/coords_test1.in @@ -0,0 +1,3 @@ +O1 1 0.00 0.00 0.00 0 +H1 2 -0.45 1.42 -1.07 1 +H2 2 -0.45 -1.48 -0.97 1 diff --git a/examples/PinnedH2O/coords_test2.in b/examples/PinnedH2O/coords_test2.in new file mode 100644 index 00000000..e1e31f8b --- /dev/null +++ b/examples/PinnedH2O/coords_test2.in @@ -0,0 +1,3 @@ +O1 1 0.00 0.00 0.00 0 +H1 2 -0.45 1.57 -1.07 1 +H2 2 -0.45 -1.48 -0.97 1 diff --git a/examples/PinnedH2O/get_ROM_table.py b/examples/PinnedH2O/get_ROM_table.py new file mode 100644 index 00000000..fa4289f3 --- /dev/null +++ b/examples/PinnedH2O/get_ROM_table.py @@ -0,0 +1,28 @@ +import subprocess +import re + +pattern = r"For energy fraction: \d+\.\d+, take first (\d+) of \d+ basis vectors" + +print("\\begin{tabular}{|c||c|c|c|c|c|c|c|}") +print("\\hline") +print("$k$ & $\\varepsilon = 10^{-1}$ & $\\varepsilon = 10^{-2}$ & $\\varepsilon = 10^{-3}$ & $\\varepsilon = 10^{-4}$ & $\\varepsilon = 10^{-5}$ & Snapshots \\\\") +print("\\hline") + +for t in range(10): + k = 50*(t+1) + snapshots = 4*k + grep_command = f"grep 'take first' basis_1_{k}_Pinned_H2O.out" + result = subprocess.run(grep_command, shell=True, capture_output=True, text=True) + matches = re.findall(pattern, result.stdout) + energy_fractions = { + "0.9": matches[0], + "0.99": matches[1], + "0.999": matches[2], + "0.9999": matches[3], + "0.99999": matches[4], + } + line = f"{k} & {energy_fractions['0.9']} & {energy_fractions['0.99']} & {energy_fractions['0.999']} & {energy_fractions['0.9999']} & {energy_fractions['0.99999']} & {snapshots} \\\\" + print(line) + +print("\\hline") +print("\\end{tabular}") diff --git a/examples/PinnedH2O/get_result.sh b/examples/PinnedH2O/get_result.sh new file mode 100644 index 00000000..b737960d --- /dev/null +++ b/examples/PinnedH2O/get_result.sh @@ -0,0 +1,37 @@ +#filename="offline_PinnedH2O" # FOM +#filename="rom39_PinnedH2O" # ROM compare MD +#filename="39_force_PinnedH2O" # ROM compare force + +#filename="PinnedH2O_test2_ref" # FOM +filename="PinnedH2O_rom_3DOF_test2_2_2_34" # ROM PinnedH2O 3DOF MD + +# Extracting kinetic energy, total energy, temperature from MGmgol output log +awk '/Kinetic/ {print $3}' $filename.out > ke_$filename.txt +awk '/Kinetic/ {print $5}' $filename.out >temp_$filename.txt +awk '/Total/ {print $3}' $filename.out > te_$filename.txt + +# Extracting H1, H2, F1, F2 from MGmgol output log +# if FOM, these files contain the FOM results +# if compare MD, these files contain the results with projected orbitals +awk '/O1 / {print $4, $5, $6}' $filename.out > O1_$filename.txt +awk '/H1 / {print $3, $4, $5}' $filename.out > H1_$filename.txt +awk '/H2 / {print $3, $4, $5}' $filename.out > H2_$filename.txt +awk '/O1 / {print $7, $8, $9}' $filename.out > f_O1_$filename.txt +awk '/H1 / {print $6, $7, $8}' $filename.out > f_H1_$filename.txt +awk '/H2 / {print $6, $7, $8}' $filename.out > f_H2_$filename.txt + +# if compare force, files with "_fom" contain the FOM results +# files with "_rom" contain the results with projected orbitals +if [[ "$filename" == *"force_"* ]]; then + sed -n '1~2p' H1_$filename.out > H1_rom$filename.txt + sed -n '1~2p' H2_$filename.out > H2_rom$filename.txt + sed -n '1~2p' f_H1_$filename.out > f_H1_rom$filename.txt + sed -n '1~2p' f_H2_$filename.out > f_H2_rom$filename.txt + + sed -n '2~2p' H1_$filename.out > H1_fom$filename.txt + sed -n '2~2p' H2_$filename.out > H2_fom$filename.txt + sed -n '2~2p' f_H1_$filename.out > f_H1_fom$filename.txt + sed -n '2~2p' f_H2_$filename.out > f_H2_fom$filename.txt +fi + +rm -rf snapshot_* diff --git a/examples/PinnedH2O/job.basis_1_50 b/examples/PinnedH2O/job.basis_1_50 new file mode 100644 index 00000000..67b0d61f --- /dev/null +++ b/examples/PinnedH2O/job.basis_1_50 @@ -0,0 +1,32 @@ +#!/bin/tcsh +#SBATCH -N 1 +#SBATCH -t 0:10:00 +#SBATCH -p pdebug + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 1 + +set maindir = /p/lustre2/cheung26/mgmol + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = ${maindir}/build_quartz/libROM/build/examples/misc/combine_samples + +set snapshot_files = "" +set increment_md_steps = 1 +set num_md_steps = 50 + +foreach k (`seq $increment_md_steps $increment_md_steps $num_md_steps`) + set snapshot_files = "$snapshot_files MD_mdstep${k}_snapshot" +end +echo "Snapshot files: $snapshot_files" + +set basis_file = "PinnedH2O_orbitals_basis_${increment_md_steps}_${num_md_steps}" + +srun -n $ncpus $exe -f $basis_file $snapshot_files > basis_${increment_md_steps}_${num_md_steps}_PinnedH2O.out + +date diff --git a/examples/PinnedH2O/job.offline b/examples/PinnedH2O/job.offline new file mode 100644 index 00000000..b6b55e64 --- /dev/null +++ b/examples/PinnedH2O/job.offline @@ -0,0 +1,30 @@ +#!/bin/tcsh +#SBATCH -N 1 +#SBATCH -t 1:00:00 +#SBATCH -p pdebug + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 1 + +set maindir = /p/lustre2/cheung26/mgmol + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = mgmol-opt + +cp $maindir/install_quartz/bin/$exe . + +set cfg_offline = mgmol_offline.cfg + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +srun -n $ncpus $exe -c $cfg_offline -i coords.in > offline_PinnedH2O.out + +date diff --git a/examples/PinnedH2O/job.ref b/examples/PinnedH2O/job.ref new file mode 100644 index 00000000..f9826e8e --- /dev/null +++ b/examples/PinnedH2O/job.ref @@ -0,0 +1,36 @@ +#!/bin/tcsh +#SBATCH -N 1 +#SBATCH -t 1:00:00 +#SBATCH -p pdebug + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 1 +set case = 2 + +set maindir = /p/lustre2/cheung26/mgmol + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = mgmol-opt + +cp $maindir/install_quartz/bin/$exe . + +set datadir = $maindir/examples/PinnedH2O + +set cfg = mgmol_ref_test${case}.cfg +cp $datadir/$cfg . + +cp $datadir/coords.in . + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +srun -n $ncpus $exe -c $cfg -i coords_test${case}.in > PinnedH2O_test${case}_ref.out + +date diff --git a/examples/PinnedH2O/job.rom b/examples/PinnedH2O/job.rom new file mode 100644 index 00000000..75a97bf4 --- /dev/null +++ b/examples/PinnedH2O/job.rom @@ -0,0 +1,34 @@ +#!/bin/tcsh +#SBATCH -N 1 +#SBATCH -t 1:00:00 +#SBATCH -p pdebug + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 1 + +set maindir = /p/lustre2/cheung26/mgmol + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = mgmol-opt + +cp $maindir/install_quartz/bin/$exe . + +set increment_md_steps = 1 +set num_md_steps = 50 +set basis_file = PinnedH2O_orbitals_basis_${increment_md_steps}_${num_md_steps} + +set cfg_rom = mgmol_rom_${increment_md_steps}_${num_md_steps}.cfg + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +srun -n $ncpus $exe -c $cfg_rom -i coords.in > rom_${increment_md_steps}_${num_md_steps}_PinnedH2O.out + +date diff --git a/examples/PinnedH2O/job.rom_3DOF b/examples/PinnedH2O/job.rom_3DOF new file mode 100644 index 00000000..b7dc5d6c --- /dev/null +++ b/examples/PinnedH2O/job.rom_3DOF @@ -0,0 +1,35 @@ +#!/bin/tcsh +#SBATCH -N 1 +#SBATCH -t 1:00:00 +#SBATCH -p pdebug + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 1 +set case = 2 + +set maindir = /p/lustre2/cheung26/mgmol + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = mgmol-opt + +cp $maindir/install_quartz/bin/$exe . + +set bondlength_num_increments = 2 +set bondangle_num_increments = 2 +set num_basis = 34 + +set cfg_rom = mgmol_rom_3DOF_test${case}.cfg + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +srun -n $ncpus $exe -c $cfg_rom -i coords_test${case}.in > PinnedH2O_rom_3DOF_test${case}_${bondlength_num_increments}_${bondangle_num_increments}_${num_basis}.out + +date diff --git a/examples/PinnedH2O/mgmol_offline.cfg b/examples/PinnedH2O/mgmol_offline.cfg new file mode 100644 index 00000000..f535cff2 --- /dev/null +++ b/examples/PinnedH2O/mgmol_offline.cfg @@ -0,0 +1,38 @@ +verbosity=1 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=500 +dt=40. +thermostat=ON +[Thermostat] +type=Berendsen +temperature=1000. +relax_time=800. +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +[Restart] +output_level=4 +[ROM.offline] +save_librom_snapshot=true +librom_snapshot_freq=1 diff --git a/examples/PinnedH2O/mgmol_ref_test1.cfg b/examples/PinnedH2O/mgmol_ref_test1.cfg new file mode 100644 index 00000000..fd456f92 --- /dev/null +++ b/examples/PinnedH2O/mgmol_ref_test1.cfg @@ -0,0 +1,35 @@ +verbosity=1 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=500 +dt=40. +thermostat=ON +[Thermostat] +type=Berendsen +temperature=300. +relax_time=800. +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +[Restart] +output_level=4 diff --git a/examples/PinnedH2O/mgmol_ref_test2.cfg b/examples/PinnedH2O/mgmol_ref_test2.cfg new file mode 100644 index 00000000..67aa443c --- /dev/null +++ b/examples/PinnedH2O/mgmol_ref_test2.cfg @@ -0,0 +1,31 @@ +verbosity=1 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=500 +dt=40. +thermostat=OFF +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +[Restart] +output_level=4 diff --git a/examples/PinnedH2O/mgmol_rom_1_50.cfg b/examples/PinnedH2O/mgmol_rom_1_50.cfg new file mode 100644 index 00000000..13cc3921 --- /dev/null +++ b/examples/PinnedH2O/mgmol_rom_1_50.cfg @@ -0,0 +1,42 @@ +verbosity=1 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=500 +dt=40. +thermostat=ON +[Thermostat] +type=Berendsen +temperature=300. +relax_time=800. +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +[Restart] +output_level=4 +[ROM] +stage=test_orbital +[ROM.offline] +basis_file=PinnedH2O_orbitals_basis_1_50 +[ROM.basis] +compare_md=false +number_of_orbital_basis=39 diff --git a/examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg b/examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg new file mode 100644 index 00000000..b7a78a0d --- /dev/null +++ b/examples/PinnedH2O/mgmol_rom_3DOF_test1.cfg @@ -0,0 +1,48 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=500 +dt=40. +thermostat=ON +[Thermostat] +type=Berendsen +temperature=300. +relax_time=800. +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +nempty=30 +[DensityMatrix] +solver=MVP +nb_inner_it=20 +mixing=0.8 +tol=1.e-8 +[Restart] +output_level=4 +[ROM] +stage=online_pinned_H2O_3dof +[ROM.offline] +basis_file=/usr/workspace/nlrom/MGmol/PinnedH2O_3DOF/data_8/PinnedH2O_3DOF_orbitals_basis_2_2 +[ROM.basis] +compare_md=false +number_of_orbital_basis=34 diff --git a/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg b/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg new file mode 100644 index 00000000..bb2912d9 --- /dev/null +++ b/examples/PinnedH2O/mgmol_rom_3DOF_test2.cfg @@ -0,0 +1,44 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=MD +[MD] +num_steps=500 +dt=40. +thermostat=OFF +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +nempty=30 +[DensityMatrix] +solver=MVP +nb_inner_it=20 +mixing=0.8 +tol=1.e-8 +[Restart] +output_level=4 +[ROM] +stage=online_pinned_H2O_3dof +[ROM.offline] +basis_file=/usr/workspace/nlrom/MGmol/PinnedH2O_3DOF/data_1/PinnedH2O_3DOF_orbitals_basis_2_2 +[ROM.basis] +compare_md=false +number_of_orbital_basis=34 diff --git a/examples/PinnedH2O/plot_PinnedH2O_force.m b/examples/PinnedH2O/plot_PinnedH2O_force.m new file mode 100644 index 00000000..a41457a5 --- /dev/null +++ b/examples/PinnedH2O/plot_PinnedH2O_force.m @@ -0,0 +1,84 @@ +clc; clear all; close all; + +%% +plot_fom = 0; +plot_rom = 0; +rdim = 77; + +%% +load F_fom.mat +fprintf(1, 'Force statistics using FOM orbitals\n'); +fprintf(1, 'Mean of force on H1: %6.4e, %6.4e, %6.4e\n', mean(F1_fom)); +fprintf(1, 'Variance of force on H1: %6.4e, %6.4e, %6.4e\n', var(F1_fom)); +fprintf(1, 'Mean of force on H2: %6.4e, %6.4e, %6.4e\n', mean(F2_fom)); +fprintf(1, 'Variance of force on H2: %6.4e, %6.4e, %6.4e\n', var(F2_fom)); + +if plot_fom + plotForce(F1_fom, 'F_H1_fom'); + plotForce(F2_fom, 'F_H2_fom'); + plotForceHistograms(F1_fom, 'H1_fom'); + plotForceHistograms(F2_fom, 'H2_fom'); +end + +%% +load(['F_rom' int2str(rdim) '.mat']) +fprintf(1, 'Force statistics using projected orbitals\n'); +fprintf(1, 'Mean of force on H1: %6.4e, %6.4e, %6.4e\n', mean(F1_rom)); +fprintf(1, 'Variance of force on H1: %6.4e, %6.4e, %6.4e\n', var(F1_rom)); +%H1_correlation = sum(F1_fom(:) .* F1_rom(:)) / (norm(F1_fom(:)) * norm(F1_rom(:))) +fprintf(1, 'Mean of force on H2: %6.4e, %6.4e, %6.4e\n', mean(F2_rom)); +fprintf(1, 'Variance of force on H2: %6.4e, %6.4e, %6.4e\n', var(F2_rom)); +%H2_correlation = sum(F2_fom(:) .* F2_rom(:)) / (norm(F2_fom(:)) * norm(F2_rom(:))) + +if plot_rom + plotForce(F1_rom, ['F_H1_rom' int2str(rdim)]); + plotForce(F2_rom, ['F_H2_rom' int2str(rdim)]); + plotForceHistograms(F1_rom, ['H1_rom' int2str(rdim)]); + plotForceHistograms(F2_rom, ['H2_rom' int2str(rdim)]); + plotForceHistogram(abs(F1_fom - F1_rom), ['H1_rom' int2str(rdim)], 'Fdiff'); + plotForceHistogram(abs(F2_fom - F2_rom), ['H2_rom' int2str(rdim)], 'Fdiff'); +end + +%% +function plotForce(F, suffix) + figure; + imagesc(F'); + axis tight; + axis equal; + colorbar; + saveas(gcf, suffix, 'jpg'); +end + +function plotForceHistogram(F, suffix, var) + figure; + if strcmp(var,'Fx') + X = F(:,1); + var_name = 'x-directional Force'; + elseif strcmp(var,'Fy') + X = F(:,2); + var_name = 'y-directional Force'; + elseif strcmp(var,'Fz') + X = F(:,3); + var_name = 'z-directional Force'; + elseif strcmp(var,'Fmag') + X = sqrt(sum(F.^2, 2)); + var_name = 'Force Magitude'; + elseif strcmp(var,'Fdiff') + X = sqrt(sum(F.^2, 2)); + var_name = 'Magitude of Difference in Force'; + else + error('Invalid type'); + end + histogram(X, 20); + xlabel(var_name); + ylabel('Frequency'); + title(['Histogram of ' var_name]); + saveas(gcf, [var '_' suffix], 'jpg'); +end + +function plotForceHistograms(F, suffix) + plotForceHistogram(F, suffix, 'Fx'); + plotForceHistogram(F, suffix, 'Fy'); + plotForceHistogram(F, suffix, 'Fz'); + plotForceHistogram(F, suffix, 'Fmag'); +end diff --git a/examples/PinnedH2O/plot_PinnedH2O_md.m b/examples/PinnedH2O/plot_PinnedH2O_md.m new file mode 100644 index 00000000..47c309b2 --- /dev/null +++ b/examples/PinnedH2O/plot_PinnedH2O_md.m @@ -0,0 +1,59 @@ +clc; clear all; close all; + +%% +plot_fom = 0; +rdims = [77, 39]; + +%% +load md_fom.mat +if plot_fom + plotAngleHistogram(H1_fom, H2_fom, 'fom'); +end + +%% + +all_H1_rom = zeros(length(rdims), size(H1_fom, 1), 3); +all_H2_rom = zeros(length(rdims), size(H2_fom, 1), 3); +k = 0; + +for rdim = rdims + k = k + 1; + load(['md_rom' int2str(rdim) '.mat']) + plotAngleHistogram(H1_rom, H2_rom, ['rom' int2str(rdim)]); + all_H1_rom(k, :, :) = H1_rom; + all_H2_rom(k, :, :) = H2_rom; +end + +plotAtomTrajectory(H1_fom(:,1), all_H1_rom(:,:,1), rdims, 'x', 1) +plotAtomTrajectory(H1_fom(:,2), all_H1_rom(:,:,2), rdims, 'y', 1) +plotAtomTrajectory(H1_fom(:,3), all_H1_rom(:,:,3), rdims, 'z', 1) + +plotAtomTrajectory(H2_fom(:,1), all_H2_rom(:,:,1), rdims, 'x', 2) +plotAtomTrajectory(H2_fom(:,2), all_H2_rom(:,:,2), rdims, 'y', 2) +plotAtomTrajectory(H2_fom(:,3), all_H2_rom(:,:,3), rdims, 'z', 2) + +%% +function plotAtomTrajectory(X_fom, all_X_rom, rdims, var, idx) + figure; + hold on; + plot(X_fom, 'Linewidth', 2, 'DisplayName', 'FOM'); + k = 0; + for rdim = rdims + k = k + 1; + X_rom = all_X_rom(k, :); + plot(X_rom, 'Linewidth', 2, 'DisplayName', ['ROM dim = ' int2str(rdim)]); + end + title([var '-coordinate of H' int2str(idx)]) + legend; + saveas(gcf, [var '_H' int2str(idx)], 'jpg'); +end + +function plotAngleHistogram(X1, X2, suffix) + figure; + A = acosd(sum(X1.*X2,2) ./ sqrt(sum(X1.^2,2)) ./ sqrt(sum(X2.^2,2))); + histogram(A, 20); + xlabel('Angle'); + ylabel('Frequency'); + title('Histogram of angle'); + saveas(gcf, ['angle_' suffix], 'jpg'); +end \ No newline at end of file diff --git a/examples/PinnedH2O/postprocess_PinnedH2O.py b/examples/PinnedH2O/postprocess_PinnedH2O.py new file mode 100644 index 00000000..142a34ac --- /dev/null +++ b/examples/PinnedH2O/postprocess_PinnedH2O.py @@ -0,0 +1,27 @@ +import subprocess +import re + +print("\\begin{tabular}{|c||c|c|c|c|c|c|c|}") +print("\\hline") +print("$k$ & $\\varepsilon = 10^{-1}$ & $\\varepsilon = 10^{-2}$ & $\\varepsilon = 10^{-3}$ & $\\varepsilon = 10^{-4}$ & $\\varepsilon = 10^{-5}$ & Snapshots \\\\") +print("\\hline") + +pattern = r"For energy fraction: \d+\.\d+, take first (\d+) of \d+ basis vectors" +for t in range(10): + k = 50*(t+1) + snapshots = 4*k + grep_command = f"grep 'take first' basis_1_{k}_Pinned_H2O.out" + result = subprocess.run(grep_command, shell=True, capture_output=True, text=True) + matches = re.findall(pattern, result.stdout) + energy_fractions = { + "0.9": matches[0], + "0.99": matches[1], + "0.999": matches[2], + "0.9999": matches[3], + "0.99999": matches[4], + } + line = f"{k} & {energy_fractions['0.9']} & {energy_fractions['0.99']} & {energy_fractions['0.999']} & {energy_fractions['0.9999']} & {energy_fractions['0.99999']} & {snapshots} \\\\" + print(line) + +print("\\hline") +print("\\end{tabular}") diff --git a/scripts/build_quartz_libROM.sh b/scripts/build_quartz_libROM.sh new file mode 100644 index 00000000..2b835a73 --- /dev/null +++ b/scripts/build_quartz_libROM.sh @@ -0,0 +1,62 @@ +#!/bin/bash +## An example script to build on LLNL Peloton systems. +## For now, this script assumes intel/ mkl libraries are being used. + +## load some modules +source scripts/modules.quartz + +## set some environment variables. Set them explicitly or use loaded module path (preferred) +## Here we use an explicit path for scalapack to be consistent with the path for the blas libraries and avoid +## benign cmake warnings +##setenv SCALAPACK_ROOT /usr/tce/packages/mkl/mkl-2020.0/lib +#setenv SCALAPACK_ROOT ${MKLROOT} +#setenv HDF5_ROOT /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0 +# +## We need to define the cmake blas vendor option here to find the right one. +#set BLAS_VENDOR = Intel10_64lp +# +## manually set the location of BLACS libraries for scalapack +#set BLACS_LIB = ${SCALAPACK_ROOT}/lib/intel64 + +MGMOL_ROOT="$(pwd)" + +INSTALL_DIR=${MGMOL_ROOT}/install_quartz +mkdir -p ${INSTALL_DIR} + +BUILD_DIR=${MGMOL_ROOT}/build_quartz +mkdir -p ${BUILD_DIR} +cd ${BUILD_DIR} + +# clone the libROM GitHub repo in BUILD_DIR +USE_LIBROM="On" +LIBROM_PATH=${BUILD_DIR}/libROM +git clone https://github.com/LLNL/libROM +cd libROM +git checkout 321d18f4d5adfa29f0a3de9be2699fee9732f2bf +#./scripts/compile.sh -t ./cmake/toolchains/default-toss_4_x86_64_ib-librom-dev.cmake +./scripts/compile.sh +cd ${BUILD_DIR} + +# call cmake +cmake -DCMAKE_TOOLCHAIN_FILE=${MGMOL_ROOT}/cmake_toolchains/quartz.default.cmake \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DUSE_LIBROM=${USE_LIBROM} \ + -DLIBROM_PATH=${LIBROM_PATH} \ + .. + +# -DCMAKE_CXX_COMPILER=mpic++ \ +# -DCMAKE_Fortran_COMPILER=mpif77 \ +# -DMPIEXEC_NUMPROC_FLAG="-n" \ +# -DBLA_VENDOR=${BLAS_VENDOR} \ +# -DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_intelmpi_lp64.so \ +# -DCMAKE_BUILD_TYPE=DEBUG \ + +# call make install +make -j 16 +### Currently libROM does not have the installation procedure, +### so copying binary file to installation directory will disrupt the relative path to libROM.so, +### causing a run-time error. +make install + +# -DBLAS_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ +# -DLAPACK_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ diff --git a/scripts/modules.quartz b/scripts/modules.quartz new file mode 100644 index 00000000..82433acb --- /dev/null +++ b/scripts/modules.quartz @@ -0,0 +1,21 @@ +### choose either gcc or intel +#module load intel/2022.1.0 +module load gcc/11.2.1 + +module load cmake +module load hdf5-parallel/1.14.0 +module load boost + +### choose either one +module load mkl-interfaces +#module load mkl + +module load python + +### manually add boost path +#setenv LD_LIBRARY_PATH /usr/tce/packages/boost/boost-1.80.0-mvapich2-2.3.6-gcc-10.3.1/lib:$LD_LIBRARY_PATH + +#setenv MKLROOT $LIBRARY_PATH +#setenv MKLROOT /usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0 +#setenv HDF5ROOT $LD_LIBRARY_PATH +#setenv HDF5ROOT diff --git a/src/AOMMprojector.cc b/src/AOMMprojector.cc index ac401490..a35acf13 100644 --- a/src/AOMMprojector.cc +++ b/src/AOMMprojector.cc @@ -17,9 +17,11 @@ AOMMprojector::AOMMprojector(LocGridOrbitals& phi, const std::shared_ptr& lrs) { - Control& ct = *(Control::instance()); - Mesh* mymesh = Mesh::instance(); + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + bool with_spin = (mmpi.nspin() > 1); const short subdivx = mymesh->subdivx(); // radius of kernel functions @@ -41,18 +43,10 @@ AOMMprojector::AOMMprojector(LocGridOrbitals& phi, kernel_proj_matrices_ = new ProjectedMatricesSparse(ct.numst, ct.occ_width, lrs); else - { -#ifdef MGMOL_USE_SCALAPACK - MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - bool with_spin = (mmpi.nspin() > 1); kernel_proj_matrices_ = new ProjectedMatrices>( ct.numst, with_spin, ct.occ_width); -#else - std::cerr << "AOMMprojector requires ScaLapack" << std::endl; - abort(); -#endif - } + // kernel functions use their own projected matrices and masks kernel_phi_ = new LocGridOrbitals( "AOMM", phi, kernel_proj_matrices_, kernelMasks_, nullptr); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2a85c686..d052d585 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,8 @@ +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/mgmol_config.h.in + ${CMAKE_CURRENT_SOURCE_DIR}/mgmol_config.h @ONLY) + +add_subdirectory(DistMatrix) add_subdirectory(linear_algebra) add_subdirectory(local_matrices) add_subdirectory(numerical_kernels) @@ -6,12 +10,9 @@ add_subdirectory(pb) add_subdirectory(radial) add_subdirectory(sparse_linear_algebra) add_subdirectory(tools) -if(${MGMOL_WITH_SCALAPACK}) - add_subdirectory(DistMatrix) -endif(${MGMOL_WITH_SCALAPACK}) -if(${MGMOL_WITH_SCALAPACK}) -set(link_libs mgmol_distmatrix +set(link_libs + mgmol_distmatrix mgmol_linear_algebra mgmol_local_matrices mgmol_numerical_kernels @@ -20,17 +21,6 @@ set(link_libs mgmol_distmatrix mgmol_sparse_linear_algebra mgmol_tools ) -else(${MGMOL_WITH_SCALAPACK}) -set(link_libs - mgmol_linear_algebra - mgmol_local_matrices - mgmol_numerical_kernels - mgmol_pb - mgmol_radial - mgmol_sparse_linear_algebra - mgmol_tools - ) -endif(${MGMOL_WITH_SCALAPACK}) set(SOURCES mgmol_run.cc @@ -45,6 +35,10 @@ set(SOURCES DielectricControl.cc ReplicatedMatrix.cc ReplicatedVector.cc + SquareSubMatrix2DistMatrix.cc + LocalMatrices2DistMatrix.cc + DistMatrix2SquareLocalMatrices.cc + GrassmanCGFactory.cc DMStrategyFactory.cc manage_memory.cc SubCell.cc @@ -83,6 +77,9 @@ set(SOURCES MGmol.cc MGmol_NEB.cc ABPG.cc + GrassmanLineMinimization.cc + GrassmanCG.cc + GrassmanCGSparse.cc LBFGS.cc IonicStepper.cc Energy.cc @@ -121,6 +118,8 @@ set(SOURCES MultipoleExpansion.cc SpreadsAndCenters.cc Preconditioning.cc + OrbitalsTransform.cc + NOLMOTransform.cc LocalizationRegions.cc Hartree.cc ShiftedHartree.cc @@ -139,6 +138,7 @@ set(SOURCES lbfgsrlx.cc OrthoAndersonMix.cc AndersonMix.cc + MLWFTransform.cc Ion.cc GridMask.cc GridMaskMult.cc @@ -148,6 +148,7 @@ set(SOURCES md.cc get_vnlpsi.cc quench.cc + mlwf.cc readInput.cc Forces.cc computeHij.cc @@ -166,30 +167,23 @@ set(SOURCES magma_singleton.cc ChebyshevApproximation.cc ChebyshevApproximationInterface.cc - mlwf.cc + rom.cc ) -if(${MGMOL_WITH_SCALAPACK}) - set(SCALAPACK_SOURCES - SquareSubMatrix2DistMatrix.cc - LocalMatrices2DistMatrix.cc - DistMatrix2SquareLocalMatrices.cc - GrassmanCGFactory.cc - GrassmanLineMinimization.cc - GrassmanCG.cc - GrassmanCGSparse.cc - MLWFTransform.cc - NOLMOTransform.cc - OrbitalsTransform.cc - ) - add_library(mgmol_src ${SOURCES} ${SCALAPACK_SOURCES}) -else() - add_library(mgmol_src ${SOURCES}) -endif(${MGMOL_WITH_SCALAPACK}) +if(USE_LIBROM) + list(APPEND SOURCES + rom_workflows.cc) +endif(USE_LIBROM) + +add_library(mgmol_src ${SOURCES}) target_include_directories(mgmol_src PRIVATE ${HDF5_INCLUDE_DIRS}) target_include_directories(mgmol_src PRIVATE ${Boost_INCLUDE_DIRS}) -target_include_directories (mgmol_src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_include_directories(mgmol_src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +if(USE_LIBROM) + target_include_directories (mgmol_src PRIVATE ${LIBROM_INCLUDES}) + target_link_libraries(mgmol_src ${LIBROM_LIB}) +endif(USE_LIBROM) target_link_libraries(mgmol_src ${link_libs}) if(${MGMOL_WITH_MAGMA}) @@ -222,3 +216,25 @@ if(${MGMOL_WITH_LIBXC}) endif (${MGMOL_WITH_LIBXC}) install(TARGETS mgmol-opt DESTINATION bin) + +# build ROM executable +if(USE_LIBROM) + add_executable(mgmol-rom rom_main.cc) + target_include_directories (mgmol-rom PRIVATE ${Boost_INCLUDE_DIRS}) + + target_link_libraries(mgmol-rom mgmol_src ${link_libs}) + target_link_libraries(mgmol-rom ${SCALAPACK_LIBRARIES}) + target_link_libraries(mgmol-rom ${HDF5_LIBRARIES}) + target_link_libraries(mgmol-rom ${HDF5_HL_LIBRARIES}) + target_link_libraries(mgmol-rom ${BLAS_LIBRARIES}) + target_link_libraries(mgmol-rom ${LAPACK_LIBRARIES}) + target_link_libraries(mgmol-rom ${Boost_LIBRARIES}) + target_link_libraries(mgmol-rom ${LIBROM_LIB}) + if (${OPENMP_CXX_FOUND}) + target_link_libraries(mgmol-rom OpenMP::OpenMP_CXX) + endif() + if(${MGMOL_WITH_LIBXC}) + target_link_libraries(mgmol-rom ${LIBXC_DIR}/lib/libxc.a) + endif (${MGMOL_WITH_LIBXC}) + install(TARGETS mgmol-rom DESTINATION bin) +endif(USE_LIBROM) diff --git a/src/ChebyshevApproximation.cc b/src/ChebyshevApproximation.cc index c6c362e2..9ef65322 100644 --- a/src/ChebyshevApproximation.cc +++ b/src/ChebyshevApproximation.cc @@ -8,15 +8,22 @@ // Please also read this link https://github.com/llnl/mgmol/LICENSE #include "ChebyshevApproximation.h" +#include "DistMatrix.h" #include "MPIdata.h" #include "ReplicatedMatrix.h" -#ifdef MGMOL_USE_SCALAPACK -#include "DistMatrix.h" -#endif - #include +template +Timer ChebyshevApproximation::compute_tm_( + "ChebyshevApproximation::compute"); +template +Timer ChebyshevApproximation::compute2_tm_( + "ChebyshevApproximation::compute2"); +template +Timer ChebyshevApproximation::build_nodes_tm_( + "ChebyshevApproximation::build_nodes"); + template ChebyshevApproximation::ChebyshevApproximation(const double a, const double b, const int order, ChebyshevApproximationFunction* func) @@ -210,7 +217,5 @@ MatrixType ChebyshevApproximation::computeChebyshevApproximation( return mat; } -#ifdef MGMOL_USE_SCALAPACK template class ChebyshevApproximation>; -#endif template class ChebyshevApproximation; diff --git a/src/ChebyshevApproximation.h b/src/ChebyshevApproximation.h index bef1d669..4c3e6d26 100644 --- a/src/ChebyshevApproximation.h +++ b/src/ChebyshevApproximation.h @@ -59,14 +59,4 @@ class ChebyshevApproximation : public ChebyshevApproximationInterface } }; -template -Timer ChebyshevApproximation::compute_tm_( - "ChebyshevApproximation::compute"); -template -Timer ChebyshevApproximation::compute2_tm_( - "ChebyshevApproximation::compute2"); -template -Timer ChebyshevApproximation::build_nodes_tm_( - "ChebyshevApproximation::build_nodes"); - #endif diff --git a/src/Control.cc b/src/Control.cc index fcba37f2..ac99ac7f 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -1834,6 +1834,10 @@ void Control::setOptions(const boost::program_options::variables_map& vm) // synchronize all processors sync(); + +#ifdef MGMOL_HAS_LIBROM + setROMOptions(vm); +#endif } int Control::checkOptions() @@ -2004,3 +2008,127 @@ void Control::printPoissonOptions(std::ostream& os) } os << std::endl; } + +void Control::setROMOptions(const boost::program_options::variables_map& vm) +{ + printWithTimeStamp("Control::setROMOptions()...", std::cout); + + if (onpe0) + { + std::string str = vm["ROM.stage"].as(); + if (str.compare("offline") == 0) + rom_pri_option.rom_stage = ROMStage::OFFLINE; + else if (str.compare("online") == 0) + rom_pri_option.rom_stage = ROMStage::ONLINE; + else if (str.compare("build") == 0) + rom_pri_option.rom_stage = ROMStage::BUILD; + else if (str.compare("online_pinned_H2O_3dof") == 0) + rom_pri_option.rom_stage = ROMStage::ONLINE_PINNED_H2O_3DOF; + else if (str.compare("test_orbital") == 0) + rom_pri_option.rom_stage = ROMStage::TEST_ORBITAL; + else if (str.compare("online_poisson") == 0) + rom_pri_option.rom_stage = ROMStage::ONLINE_POISSON; + else if (str.compare("test_poisson") == 0) + rom_pri_option.rom_stage = ROMStage::TEST_POISSON; + else if (str.compare("test_rho") == 0) + rom_pri_option.rom_stage = ROMStage::TEST_RHO; + else if (str.compare("test_ion") == 0) + rom_pri_option.rom_stage = ROMStage::TEST_ION; + else if (str.compare("none") == 0) + rom_pri_option.rom_stage = ROMStage::UNSUPPORTED; + + rom_pri_option.restart_file_fmt + = vm["ROM.offline.restart_filefmt"].as(); + rom_pri_option.restart_file_minidx + = vm["ROM.offline.restart_min_idx"].as(); + rom_pri_option.restart_file_maxidx + = vm["ROM.offline.restart_max_idx"].as(); + rom_pri_option.basis_file + = vm["ROM.offline.basis_file"].as(); + + str = vm["ROM.offline.variable"].as(); + if (str.compare("orbitals") == 0) + rom_pri_option.variable = ROMVariable::ORBITALS; + else if (str.compare("potential") == 0) + rom_pri_option.variable = ROMVariable::POTENTIAL; + else + rom_pri_option.variable = ROMVariable::NONE; + + rom_pri_option.save_librom_snapshot + = vm["ROM.offline.save_librom_snapshot"].as(); + rom_pri_option.librom_snapshot_freq + = vm["ROM.offline.librom_snapshot_freq"].as(); + + rom_pri_option.compare_md = vm["ROM.basis.compare_md"].as(); + rom_pri_option.num_orbbasis + = vm["ROM.basis.number_of_orbital_basis"].as(); + rom_pri_option.num_potbasis + = vm["ROM.basis.number_of_potential_basis"].as(); + rom_pri_option.pot_rom_file + = vm["ROM.potential_rom_file"].as(); + } // onpe0 + + // synchronize all processors + syncROMOptions(); +} + +void Control::syncROMOptions() +{ + if (onpe0 && verbose > 0) + (*MPIdata::sout) << "Control::syncROMOptions()" << std::endl; + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + + mmpi.bcast(rom_pri_option.restart_file_fmt, comm_global_); + mmpi.bcast(rom_pri_option.basis_file, comm_global_); + mmpi.bcast(rom_pri_option.pot_rom_file, comm_global_); + + auto bcast_check = [](int mpirc) + { + if (mpirc != MPI_SUCCESS) + { + (*MPIdata::sout) << "MPI Bcast of Control failed!!!" << std::endl; + MPI_Abort(comm_global_, 2); + } + }; + + short rom_stage = (short)static_cast(rom_pri_option.rom_stage); + int mpirc; + mpirc = MPI_Bcast(&rom_stage, 1, MPI_SHORT, 0, comm_global_); + bcast_check(mpirc); + + mpirc = MPI_Bcast( + &rom_pri_option.restart_file_minidx, 1, MPI_INT, 0, comm_global_); + bcast_check(mpirc); + + mpirc = MPI_Bcast( + &rom_pri_option.restart_file_maxidx, 1, MPI_INT, 0, comm_global_); + bcast_check(mpirc); + + mpirc = MPI_Bcast( + &rom_pri_option.save_librom_snapshot, 1, MPI_C_BOOL, 0, comm_global_); + bcast_check(mpirc); + + mpirc = MPI_Bcast( + &rom_pri_option.librom_snapshot_freq, 1, MPI_INT, 0, comm_global_); + bcast_check(mpirc); + + short rom_var = (short)static_cast(rom_pri_option.variable); + mpirc = MPI_Bcast(&rom_var, 1, MPI_SHORT, 0, comm_global_); + bcast_check(mpirc); + + rom_pri_option.rom_stage = static_cast(rom_stage); + rom_pri_option.variable = static_cast(rom_var); + + mpirc + = MPI_Bcast(&rom_pri_option.compare_md, 1, MPI_C_BOOL, 0, comm_global_); + bcast_check(mpirc); + + mpirc + = MPI_Bcast(&rom_pri_option.num_orbbasis, 1, MPI_INT, 0, comm_global_); + bcast_check(mpirc); + + mpirc + = MPI_Bcast(&rom_pri_option.num_potbasis, 1, MPI_INT, 0, comm_global_); + bcast_check(mpirc); +} diff --git a/src/Control.h b/src/Control.h index 32049a72..3bed7464 100644 --- a/src/Control.h +++ b/src/Control.h @@ -13,6 +13,10 @@ #include "Species.h" #include "Timeout.h" +/* enumeration and option variables for libROM */ +#include "mgmol_config.h" +#include "rom_Control.h" + #include #include #include @@ -217,6 +221,9 @@ class Control void printRestartLink(); + /* libROM related options */ + ROMPrivateOptions rom_pri_option; + public: static Control* instance() { @@ -735,6 +742,11 @@ class Control } bool AtomsMove() { return (atoms_dyn_ != 0); } + + /* ROM-related options */ + void setROMOptions(const boost::program_options::variables_map& vm); + void syncROMOptions(); + const ROMPrivateOptions getROMOptions() { return rom_pri_option; } }; #endif diff --git a/src/DFTsolver.cc b/src/DFTsolver.cc index debf3cb8..55e6df26 100644 --- a/src/DFTsolver.cc +++ b/src/DFTsolver.cc @@ -13,16 +13,13 @@ #include "DMStrategy.h" #include "Electrostatic.h" #include "Energy.h" +#include "GrassmanCGFactory.h" #include "Ions.h" #include "MGmol.h" #include "Potentials.h" #include "ProjectedMatricesInterface.h" #include "Rho.h" -#ifdef MGMOL_USE_SCALAPACK -#include "GrassmanCGFactory.h" -#endif - template DFTsolver::DFTsolver(Hamiltonian* hamiltonian, ProjectedMatricesInterface* proj_matrices, Energy* energy, @@ -51,7 +48,6 @@ DFTsolver::DFTsolver(Hamiltonian* hamiltonian, break; } -#ifdef MGMOL_USE_SCALAPACK case OuterSolverType::NLCG: { orbitals_stepper_ = GrassmanCGFactory::create( @@ -60,7 +56,6 @@ DFTsolver::DFTsolver(Hamiltonian* hamiltonian, break; } -#endif default: std::cerr << "DFTsolver: Undefined iterative electronic structure " diff --git a/src/DMStrategyFactory.cc b/src/DMStrategyFactory.cc index 8df599b3..115c8cbe 100644 --- a/src/DMStrategyFactory.cc +++ b/src/DMStrategyFactory.cc @@ -1,7 +1,6 @@ #include "DMStrategyFactory.h" #include "ReplicatedMatrix.h" -#ifdef MGMOL_USE_SCALAPACK template <> DMStrategy>* DMStrategyFactory, @@ -36,7 +35,6 @@ DMStrategyFactory, return dm_strategy; } } -#endif template <> DMStrategy>* @@ -58,7 +56,6 @@ DMStrategyFactory, return nullptr; } -#ifdef MGMOL_USE_SCALAPACK template <> DMStrategy>* DMStrategyFactory, @@ -81,7 +78,6 @@ DMStrategyFactory, return dm_strategy; } -#endif template <> DMStrategy>* diff --git a/src/DavidsonSolver.cc b/src/DavidsonSolver.cc index 4804af64..2b097be9 100644 --- a/src/DavidsonSolver.cc +++ b/src/DavidsonSolver.cc @@ -527,8 +527,6 @@ int DavidsonSolver::solve( } else { - if (mmpi.PE0() && ct.verbose > 2) - os_ << "Update h11..." << std::endl; hamiltonian_->applyDeltaPot(orbitals, hphi); orbitals.addDotWithNcol2Matrix(hphi, h11); } @@ -544,8 +542,6 @@ int DavidsonSolver::solve( } // update h22, h12 and h21 - if (mmpi.PE0() && ct.verbose > 2) - os_ << "Update h22, h12 and h21..." << std::endl; orbitals.addDotWithNcol2Matrix(hphi, h12); work_orbitals.addDotWithNcol2Matrix(hphi, h22); @@ -722,10 +718,9 @@ int DavidsonSolver::solve( // replace orbitals with eigenvectors corresponding to largest // eigenvalues of DM - if (mmpi.PE0() && ct.verbose > 2) - os_ << "Update trial eigenvectors..." << std::endl; - orbitals.multiply_by_matrix(dm12, 0., orbitals); - work_orbitals.multiply_by_matrix(dm22, 1., orbitals); + orbitals.multiply_by_matrix(dm12); + work_orbitals.multiply_by_matrix(dm22); + orbitals.axpy((ORBDTYPE)1., work_orbitals); orbitals.incrementIterativeIndex(); orbitals.incrementIterativeIndex(); work_orbitals.incrementIterativeIndex(2); @@ -864,8 +859,6 @@ void DavidsonSolver::printTimers(std::ostream& os) target_tm_.print(os); } -#ifdef MGMOL_USE_SCALAPACK template class DavidsonSolver, dist_matrix::DistMatrix>; -#endif template class DavidsonSolver, ReplicatedMatrix>; diff --git a/src/DensityMatrix.cc b/src/DensityMatrix.cc index ef47c26b..a82ec288 100644 --- a/src/DensityMatrix.cc +++ b/src/DensityMatrix.cc @@ -9,17 +9,12 @@ #include "DensityMatrix.h" +#include "DistMatrix.h" #include "MGmol_MPI.h" #include "ReplicatedMatrix.h" #include "ReplicatedWorkSpace.h" #include "hdf_tools.h" -#ifdef MGMOL_USE_SCALAPACK -#include "DistMatrix.h" -#else -typedef double DISTMATDTYPE; -#endif - #include #include #include @@ -482,7 +477,5 @@ int DensityMatrix::read(HDFrestart& h5f_file, std::string& name) return ierr; } -#ifdef MGMOL_USE_SCALAPACK template class DensityMatrix>; -#endif template class DensityMatrix; diff --git a/src/DistMatrix/BlacsContext.cc b/src/DistMatrix/BlacsContext.cc index 6d056218..7610a3cb 100644 --- a/src/DistMatrix/BlacsContext.cc +++ b/src/DistMatrix/BlacsContext.cc @@ -13,11 +13,11 @@ #include #include -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK #include "blacs.h" #endif -#ifndef MGMOL_USE_SCALAPACK +#ifndef SCALAPACK void Cblacs_pinfo(int* mypnum, int* nprocs) { diff --git a/src/DistMatrix/DistMatrix.cc b/src/DistMatrix/DistMatrix.cc index c35d6e74..89f19400 100644 --- a/src/DistMatrix/DistMatrix.cc +++ b/src/DistMatrix/DistMatrix.cc @@ -23,14 +23,14 @@ #include -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK #include "blacs.h" #endif namespace dist_matrix { -#ifndef MGMOL_USE_SCALAPACK +#ifndef SCALAPACK int NUMROC(int* a, int* b, int* c, int* d, int* e) { return *a; } int INDXL2G(Pint indxloc, Pint nb, Pint iproc, Pint isrcproc, Pint nprocs) { @@ -135,7 +135,7 @@ void DistMatrix::resize(const int m, const int n, const int mb, const int nb) m_ = m; n_ = n; -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK mb_ = std::min(mb, m); nb_ = std::min(nb, n); #else @@ -280,7 +280,7 @@ double DistMatrix::traceProduct(const DistMatrix& x) const tsum = LinearAlgebraUtils::MPdot( size_, &val_[0], &x.val_[0]); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK MGmol_MPI& mmpi = *(MGmol_MPI::instance()); mmpi.allreduce(&tsum, &sum, 1, MPI_SUM); #else @@ -325,7 +325,7 @@ void DistMatrix::axpy(const double alpha, const DistMatrix& x) template <> void DistMatrix::identity(void) { -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK char uplo = 'n'; double alpha = 0.0; double beta = 1.0; @@ -342,7 +342,7 @@ void DistMatrix::identity(void) template <> void DistMatrix::identity(void) { -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK char uplo = 'n'; float alpha = 0.0; float beta = 1.0; @@ -371,7 +371,7 @@ DistMatrix& DistMatrix::operator=(const DistMatrix& src) exit(1); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK if (src.mb_ == mb_ && src.nb_ == nb_ && src.bc_ == bc_) #endif { @@ -390,7 +390,7 @@ DistMatrix& DistMatrix::operator=(const DistMatrix& src) memcpy(&val_[0], &src.val_[0], size_ * sizeof(double)); } } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK else { // redistribute using function pdgemr2d @@ -423,7 +423,7 @@ DistMatrix& DistMatrix::operator=(const DistMatrix& src) exit(1); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK if (src.mb_ == mb_ && src.nb_ == nb_ && src.bc_ == bc_) #endif { @@ -442,7 +442,7 @@ DistMatrix& DistMatrix::operator=(const DistMatrix& src) memcpy(&val_[0], &src.val_[0], size_ * sizeof(float)); } } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK else { // redistribute using function pdgemr2d @@ -478,7 +478,7 @@ DistMatrix& DistMatrix::assign( exit(1); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK if (src.bc_ == bc_ && src.m_ == m_ && src.n_ == n_ && src.mb_ == mb_ && src.nb_ == nb_) #endif @@ -493,7 +493,7 @@ DistMatrix& DistMatrix::assign( } } } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK else { // redistribute @@ -528,7 +528,7 @@ DistMatrix& DistMatrix::assign( exit(1); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK if (src.bc_ == bc_ && src.m_ == m_ && src.n_ == n_ && src.mb_ == mb_ && src.nb_ == nb_) #endif @@ -543,7 +543,7 @@ DistMatrix& DistMatrix::assign( } } } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK else { // redistribute @@ -589,7 +589,7 @@ void DistMatrix::gemv(const char transa, const double alpha, assert(a.n() == m_); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pdgemv(&transa, &m, &n, &alpha, &a.val_[0], &ione, &ione, a.desc(), &b.val_[0], &ione, &ione, b.desc(), &ione, &beta, &val_[0], &ione, @@ -612,7 +612,7 @@ void DistMatrix::matvec( char transa = 'N'; double alpha = 1.; double beta = 0.; -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pdgemv(&transa, &m_, &n_, &alpha, &val_[0], &ione, &ione, desc(), &v.val_[0], &ione, &ione, v.desc(), &ione, &beta, &y.val_[0], &ione, @@ -648,7 +648,7 @@ void DistMatrix::gemv(const char transa, const float alpha, assert(a.n() == m_); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; psgemv(&transa, &m, &n, &alpha, &a.val_[0], &ione, &ione, a.desc(), &b.val_[0], &ione, &ione, b.desc(), &ione, &beta, &val_[0], &ione, @@ -672,7 +672,7 @@ void DistMatrix::symv(const char uplo, const double alpha, { assert(a.n() == m_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pdsymv(&uplo, &m_, &alpha, &a.val_[0], &ione, &ione, a.desc(), &b.val_[0], &ione, &ione, b.desc(), &ione, &beta, &val_[0], &ione, @@ -696,7 +696,7 @@ void DistMatrix::symv(const char uplo, const float alpha, { assert(a.n() == m_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pssymv(&uplo, &m_, &alpha, &a.val_[0], &ione, &ione, a.desc(), &b.val_[0], &ione, &ione, b.desc(), &ione, &beta, &val_[0], &ione, @@ -747,7 +747,7 @@ void DistMatrix::gemm(const char transa, const char transb, assert(k == b.n()); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pdgemm(&transa, &transb, &m, &n, &k, &alpha, &a.val_[0], &ione, &ione, a.desc(), &b.val_[0], &ione, &ione, b.desc(), &beta, &val_[0], @@ -793,7 +793,7 @@ void DistMatrix::gemm(const char transa, const char transb, assert(k == b.n()); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; psgemm(&transa, &transb, &m, &n, &k, &alpha, &a.val_[0], &ione, &ione, a.desc(), &b.val_[0], &ione, &ione, b.desc(), &beta, &val_[0], @@ -834,7 +834,7 @@ void DistMatrix::symm(const char side, const char uplo, assert(a.m() == b.n()); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pdsymm(&side, &uplo, &m_, &n_, &alpha, &a.val_[0], &ione, &ione, a.desc(), &b.val_[0], &ione, &ione, b.desc(), &beta, &val_[0], @@ -870,7 +870,7 @@ void DistMatrix::symm(const char side, const char uplo, assert(a.m() == b.n()); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pssymm(&side, &uplo, &m_, &n_, &alpha, &a.val_[0], &ione, &ione, a.desc(), &b.val_[0], &ione, &ione, b.desc(), &beta, &val_[0], @@ -907,7 +907,7 @@ void DistMatrix::trmm(const char side, const char uplo, { assert(a.n_ == n_); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pdtrmm(&side, &uplo, &trans, &diag, &m_, &n_, &alpha, &a.val_[0], &ione, &ione, a.desc_, &val_[0], &ione, &ione, desc_); @@ -933,7 +933,7 @@ void DistMatrix::trmm(const char side, const char uplo, const char trans, { assert(a.n_ == n_); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pstrmm(&side, &uplo, &trans, &diag, &m_, &n_, &alpha, &a.val_[0], &ione, &ione, a.desc_, &val_[0], &ione, &ione, desc_); @@ -972,7 +972,7 @@ void DistMatrix::trsm(const char side, const char uplo, { assert(a.n_ == n_); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pdtrsm(&side, &uplo, &trans, &diag, &m_, &n_, &alpha, &a.val_[0], &ione, &ione, a.desc_, &val_[0], &ione, &ione, desc_); @@ -998,7 +998,7 @@ void DistMatrix::trsm(const char side, const char uplo, const char trans, { assert(a.n_ == n_); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pstrsm(&side, &uplo, &trans, &diag, &m_, &n_, &alpha, &a.val_[0], &ione, &ione, a.desc_, &val_[0], &ione, &ione, desc_); @@ -1023,7 +1023,7 @@ void DistMatrix::trtrs(const char uplo, const char trans, { assert(m_ == n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pdtrtrs(&uplo, &trans, &diag, &m_, &b.n_, &val_[0], &ione, &ione, desc_, &b.val_[0], &ione, &ione, b.desc_, &info); @@ -1048,7 +1048,7 @@ void DistMatrix::trtrs(const char uplo, const char trans, { assert(m_ == n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pstrtrs(&uplo, &trans, &diag, &m_, &b.n_, &val_[0], &ione, &ione, desc_, &b.val_[0], &ione, &ione, b.desc_, &info); @@ -1077,7 +1077,7 @@ int DistMatrix::potrf(char uplo) { assert(m_ == n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pdpotrf(&uplo, &m_, val_.data(), &ione, &ione, desc_, &info); #else @@ -1105,7 +1105,7 @@ int DistMatrix::potrf(char uplo) { assert(m_ == n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pspotrf(&uplo, &m_, &val_[0], &ione, &ione, desc_, &info); #else @@ -1132,7 +1132,7 @@ void DistMatrix::getrf(std::vector& ipiv) int info; if (active_) { -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; ipiv.resize(mb_ + mloc_); pdgetrf(&m_, &n_, &val_[0], &ione, &ione, desc_, &ipiv[0], &info); @@ -1154,7 +1154,7 @@ void DistMatrix::getrf(std::vector& ipiv) int info; if (active_) { -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; ipiv.resize(mb_ + mloc_); psgetrf(&m_, &n_, &val_[0], &ione, &ione, desc_, &ipiv[0], &info); @@ -1183,7 +1183,7 @@ void DistMatrix::potrs(char uplo, DistMatrix& b) assert(m_ == n_); assert(b.m_ == m_); assert(b.n_ <= n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pdpotrs(&uplo, &m_, &b.n_, &val_[0], &ione, &ione, desc_, &b.val_[0], &ione, &ione, b.desc_, &info); @@ -1207,7 +1207,7 @@ void DistMatrix::potrs(char uplo, DistMatrix& b) assert(m_ == n_); assert(b.m_ == m_); assert(b.n_ <= n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pspotrs(&uplo, &m_, &b.n_, &val_[0], &ione, &ione, desc_, &b.val_[0], &ione, &ione, b.desc_, &info); @@ -1235,7 +1235,7 @@ void DistMatrix::getrs( assert(m_ == n_); assert(b.m_ == m_); assert(b.n_ <= n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; assert((int)ipiv.size() == (mb_ + mloc_)); pdgetrs(&trans, &m_, &b.n_, &val_[0], &ione, &ione, desc_, &ipiv[0], @@ -1263,7 +1263,7 @@ void DistMatrix::getrs( assert(m_ == n_); assert(b.m_ == m_); assert(b.n_ <= n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; assert((int)ipiv.size() == (mb_ + mloc_)); psgetrs(&trans, &m_, &b.n_, &val_[0], &ione, &ione, desc_, &ipiv[0], @@ -1295,7 +1295,7 @@ int DistMatrix::potri(char uplo) { assert(m_ == n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pdpotri(&uplo, &m_, &val_[0], &ione, &ione, desc_, &info); #else @@ -1322,7 +1322,7 @@ int DistMatrix::potri(char uplo) { assert(m_ == n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pspotri(&uplo, &m_, &val_[0], &ione, &ione, desc_, &info); #else @@ -1350,7 +1350,7 @@ int DistMatrix::trtri(char uplo, char diag) { assert(m_ == n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pdtrtri(&uplo, &diag, &m_, &val_[0], &ione, &ione, desc_, &info); #else @@ -1383,7 +1383,7 @@ double DistMatrix::norm(char ty) double norm_val = 1.; if (active_) { -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; if (ty == 'I' || ty == 'i') lwork = mloc_; if (ty == '1') lwork = nloc_; @@ -1409,7 +1409,7 @@ double DistMatrix::norm(char ty) float norm_val = 1.; if (active_) { -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; if (ty == 'I' || ty == 'i') lwork = mloc_; if (ty == '1') lwork = nloc_; @@ -1444,7 +1444,7 @@ double DistMatrix::pocon(char uplo, double anorm) { assert(m_ == n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; int lwork = 2 * mloc_ + 3 * nloc_ + nb_ * std::min(npcol_, nprow_); lwork = std::max(lwork, 1); @@ -1490,7 +1490,7 @@ double DistMatrix::pocon(char uplo, float anorm) { assert(m_ == n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; int lwork = 2 * mloc_ + 3 * nloc_ + nb_ * std::min(npcol_, nprow_); lwork = std::max(lwork, 1); @@ -1552,7 +1552,7 @@ void DistMatrix::syrk( k = a.m(); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pdsyrk(&uplo, &trans, &n, &k, &alpha, &a.val_[0], &ione, &ione, a.desc(), &beta, &val_[0], &ione, &ione, desc_); @@ -1584,7 +1584,7 @@ void DistMatrix::syrk( k = a.m(); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pssyrk(&uplo, &trans, &n, &k, &alpha, &a.val_[0], &ione, &ione, a.desc(), &beta, &val_[0], &ione, &ione, desc_); @@ -1603,7 +1603,7 @@ template <> void DistMatrix::getsub( const DistMatrix& a, int m, int n, int ia, int ja) { -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int iap = ia + 1; int jap = ja + 1; assert(n <= n_); @@ -1627,7 +1627,7 @@ template <> void DistMatrix::getsub( const DistMatrix& a, int m, int n, int ia, int ja) { -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int iap = ia + 1; int jap = ja + 1; assert(n <= n_); @@ -1651,7 +1651,7 @@ template <> void DistMatrix::getsub( const DistMatrix& a, int m, int n, int ia, int ja) { -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int iap = ia + 1; int jap = ja + 1; assert(n <= n_); @@ -1685,7 +1685,7 @@ void DistMatrix::transpose( assert(a.m() == n_); assert(a.n() == m_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pdtran(&m_, &n_, &alpha, &a.val_[0], &ione, &ione, a.desc(), &beta, &val_[0], &ione, &ione, desc_); @@ -1714,7 +1714,7 @@ void DistMatrix::transpose( assert(a.m() == n_); assert(a.n() == m_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; pstran(&m_, &n_, &alpha, &a.val_[0], &ione, &ione, a.desc(), &beta, &val_[0], &ione, &ione, desc_); @@ -1859,7 +1859,7 @@ double DistMatrix::sumProdElements(const DistMatrix& a) const } double sum = 0.; -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK MPI_Allreduce(&tsum, &sum, 1, MPI_DOUBLE, MPI_SUM, comm_global_); #else sum = tsum; @@ -1896,7 +1896,7 @@ DistMatrix::DistMatrix(const std::string& name, const BlacsContext& bc, template <> void DistMatrix::initFromReplicated(double* const src, const int lda) { -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK // all the nodes send data const int rev = 1; const int ii = 1; @@ -1921,7 +1921,7 @@ void DistMatrix::allgather(double* const a, const int lda) const assert(m_ == n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK // all the nodes receive the data const int ii = -1; const int jj = -1; @@ -1970,7 +1970,7 @@ void DistMatrix::allgather(float* const a, const int lda) const assert(m_ == n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK const int ii = -1; const int jj = -1; const int rev = 0; @@ -2101,7 +2101,7 @@ double DistMatrix::trace(void) const if (active_) { -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; trace = pdlatra(&n_, &val_[0], &ione, &ione, desc_); #else @@ -2123,7 +2123,7 @@ double DistMatrix::trace(void) const if (active_) { -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; trace = (double)pslatra(&n_, &val_[0], &ione, &ione, desc_); #else @@ -2154,7 +2154,7 @@ void DistMatrix::sygst( { assert(m_ == n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; double scale; pdsygst(&itype, &uplo, &m_, &val_[0], &ione, &ione, desc_, &b.val_[0], @@ -2178,7 +2178,7 @@ void DistMatrix::sygst(int itype, char uplo, const DistMatrix& b) { assert(m_ == n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ione = 1; float scale; pssygst(&itype, &uplo, &m_, &val_[0], &ione, &ione, desc_, &b.val_[0], @@ -2202,7 +2202,7 @@ void DistMatrix::syev( { assert(m_ == n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK assert(mb_ > 1); int ione = 1, izero = 0; int nn = std::max(mb_, m_); @@ -2226,7 +2226,7 @@ void DistMatrix::syev( delete[] work; } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK MPI_Bcast(&w[0], m_, MPI_DOUBLE, 0, comm_global_); #endif } @@ -2240,7 +2240,7 @@ void DistMatrix::syev( { assert(m_ == n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK assert(mb_ > 1); int ione = 1, izero = 0; int nn = std::max(mb_, m_); @@ -2264,7 +2264,7 @@ void DistMatrix::syev( delete[] work; } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK MPI_Bcast(&w[0], m_, MPI_FLOAT, 0, comm_global_); #endif } @@ -2278,7 +2278,7 @@ void DistMatrix::gesvd(char jobu, char jobvt, std::vector& s, { assert(m_ == n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK assert(mb_ > 1); int ione = 1; int lwork = -1; @@ -2309,7 +2309,7 @@ void DistMatrix::gesvd(char jobu, char jobvt, std::vector& s, delete[] work; } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int m = std::min(m_, n_); MPI_Bcast(&s[0], m, MPI_DOUBLE, 0, comm_global_); #endif @@ -2324,7 +2324,7 @@ void DistMatrix::gesvd(char jobu, char jobvt, std::vector& s, { assert(m_ == n_); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK assert(mb_ > 1); int ione = 1; int lwork = -1; @@ -2355,7 +2355,7 @@ void DistMatrix::gesvd(char jobu, char jobvt, std::vector& s, delete[] work; } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int m = std::min(m_, n_); MPI_Bcast(&s[0], m, MPI_FLOAT, 0, comm_global_); #endif @@ -2387,7 +2387,7 @@ int DistMatrix::iamax(const int j, double& val) { int indx = -1; int incx = 1; -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ix = 1; int jx = y(j) + 1; // C to fortran int proc_col = pc(j); @@ -2414,7 +2414,7 @@ int DistMatrix::iamax(const int j, float& val) { int indx = -1; int incx = 1; -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int ix = 1; int jx = y(j) + 1; // C to fortran int proc_col = pc(j); @@ -2440,7 +2440,7 @@ template <> void DistMatrix::swapColumns(const int j1, const int j2) { if (j1 == j2) return; -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int jx = j1 + 1; // C to Fortran int jy = j2 + 1; // C to Fortran int ione = 1; @@ -2458,7 +2458,7 @@ template <> void DistMatrix::swapColumns(const int j1, const int j2) { if (j1 == j2) return; -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK int jx = j1 + 1; // C to Fortran int jy = j2 + 1; // C to Fortran int ione = 1; diff --git a/src/DistMatrix/DistVector.h b/src/DistMatrix/DistVector.h index ba3b481a..d800f89c 100644 --- a/src/DistMatrix/DistVector.h +++ b/src/DistMatrix/DistVector.h @@ -96,7 +96,7 @@ class DistVector : public DistMatrix DistMatrix::val_.size(), DistMatrix::val_.data(), v.val_.data()); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK MGmol_MPI& mmpi = *(MGmol_MPI::instance()); mmpi.allreduce(&tsum, &sum, 1, MPI_SUM); #else diff --git a/src/DistMatrix/MGmol_scalapack.h b/src/DistMatrix/MGmol_scalapack.h index 9e685c0d..ae9467a4 100644 --- a/src/DistMatrix/MGmol_scalapack.h +++ b/src/DistMatrix/MGmol_scalapack.h @@ -6,8 +6,8 @@ // All rights reserved. // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -#ifndef MGMOL_MGMOL_USE_SCALAPACK_H -#define MGMOL_MGMOL_USE_SCALAPACK_H +#ifndef MGMOL_SCALAPACK_H +#define MGMOL_SCALAPACK_H #include "scalapack_mangle.h" @@ -18,7 +18,7 @@ typedef const float* const Pfloat; extern "C" { -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK // PBLAS void pdswap( Pint, double*, Pint, Pint, Pint, Pint, double*, Pint, Pint, Pint, Pint); @@ -60,7 +60,7 @@ extern "C" void pdamax(Pint, double*, int*, double*, Pint, Pint, Pint, Pint); void psamax(Pint, float*, int*, float*, Pint, Pint, Pint, Pint); - // MGMOL_USE_SCALAPACK + // SCALAPACK void pdelset(double*, Pint, Pint, int*, Pdouble); void pselset(float*, Pint, Pint, int*, Pfloat); float pselget(Pchar, Pchar, float*, Pfloat, Pint, Pint, Pint); @@ -120,7 +120,7 @@ extern "C" int*); void psgesvd(Pchar, Pchar, int*, int*, float*, int*, int*, int*, float*, float*, int*, int*, int*, float*, int*, int*, int*, float*, int*, int*); - // MGMOL_USE_SCALAPACK TOOLS + // SCALAPACK TOOLS int NUMROC(Pint, Pint, Pint, Pint, Pint); int INDXL2G(Pint, Pint, Pint, Pint, Pint); int INDXG2L(Pint, Pint, Pint, Pint, Pint); diff --git a/src/DistMatrix/blacs.h b/src/DistMatrix/blacs.h index f52f9333..6d4d2aa2 100644 --- a/src/DistMatrix/blacs.h +++ b/src/DistMatrix/blacs.h @@ -33,7 +33,7 @@ extern "C" int blacs_pnum(int*, int*, int*); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK extern "C" { #endif @@ -53,7 +53,7 @@ extern "C" void Cblacs_exit(int); int Cblacs_pnum(int, int, int); int Csys2blacs_handle(MPI_Comm); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK } #endif diff --git a/src/EigenDMStrategy.cc b/src/EigenDMStrategy.cc index bd596e73..cf803e0f 100644 --- a/src/EigenDMStrategy.cc +++ b/src/EigenDMStrategy.cc @@ -29,7 +29,6 @@ void EigenDMStrategy::initialize(OrbitalsType& orbitals) template int EigenDMStrategy::update(OrbitalsType& orbitals) { -#ifdef MGMOL_USE_SCALAPACK Control& ct = *(Control::instance()); dist_matrix::DistMatrix zz("Z", ct.numst, ct.numst); @@ -42,16 +41,9 @@ int EigenDMStrategy::update(OrbitalsType& orbitals) // if( onpe0 && ct.verbose>2 ) // (*MPIdata::sout)<<"get_dm_diag: rotate orbitals "<::update(OrbitalsType& " - "orbitals) not implemented" - << std::endl; - abort(); -#endif return 0; } diff --git a/src/Electrostatic.h b/src/Electrostatic.h index 06a9e17c..619da18a 100644 --- a/src/Electrostatic.h +++ b/src/Electrostatic.h @@ -47,6 +47,7 @@ class Electrostatic ~Electrostatic(); static Timer solve_tm() { return solve_tm_; } + const bool isDielectric() { return diel_flag_; } pb::GridFunc* getRhoc() { return grhoc_; } Poisson* getPoissonSolver() { return poisson_solver_; } diff --git a/src/ExtendedGridOrbitals.cc b/src/ExtendedGridOrbitals.cc index dc3cae03..ddd18207 100644 --- a/src/ExtendedGridOrbitals.cc +++ b/src/ExtendedGridOrbitals.cc @@ -14,9 +14,7 @@ #include "DotProductManagerFactory.h" #include "GridFunc.h" #include "Laph4M.h" -#ifdef MGMOL_USE_SCALAPACK #include "LocalMatrices2DistMatrix.h" -#endif #include "LocalizationRegions.h" #include "MPIdata.h" #include "Mesh.h" @@ -32,6 +30,10 @@ #include #include +#ifdef MGMOL_HAS_LIBROM +#include "librom.h" +#endif + #define ORBITAL_OCCUPATION 2. std::string getDatasetName(const std::string& name, const int color); @@ -40,6 +42,18 @@ DotProductManager>* ExtendedGridOrbitals::dotProductManager_ = nullptr; +template +int ExtendedGridOrbitals::lda_ = 0; +template +int ExtendedGridOrbitals::numpt_ = 0; +template +int ExtendedGridOrbitals::data_wghosts_index_ = -1; +template +int ExtendedGridOrbitals::numst_ = -1; +template +std::vector> + ExtendedGridOrbitals::overlapping_gids_; + template Timer ExtendedGridOrbitals::matB_tm_( "ExtendedGridOrbitals" + std::to_string(8 * sizeof(ScalarType)) + "::matB"); @@ -74,14 +88,6 @@ Timer ExtendedGridOrbitals::normalize_tm_( template Timer ExtendedGridOrbitals::axpy_tm_( "ExtendedGridOrbitals" + std::to_string(8 * sizeof(ScalarType)) + "::axpy"); -template -Timer ExtendedGridOrbitals::gemm_nn_tm_( - "ExtendedGridOrbitals" + std::to_string(8 * sizeof(ScalarType)) - + "::gemm_nn"); -template -Timer ExtendedGridOrbitals::gemm_tn_tm_( - "ExtendedGridOrbitals" + std::to_string(8 * sizeof(ScalarType)) - + "::gemm_tn"); template ExtendedGridOrbitals::ExtendedGridOrbitals(std::string name, @@ -135,6 +141,9 @@ ExtendedGridOrbitals::ExtendedGridOrbitals(const std::string& name, block_vector_(A.block_vector_, copy_data), grid_(A.grid_) { + // if(onpe0)cout<<"call ExtendedGridOrbitals(const + // ExtendedGridOrbitals &A, const bool copy_data)"<::initFourier() resetIterativeIndex(); } +template +void ExtendedGridOrbitals::multiply_by_matrix( + const dist_matrix::DistMatrix& dmatrix, + ScalarType* const product, const int ldp) +{ +#if 0 + (*MPIdata::sout)<<"self multiply_by_matrix"<& wspace( + ReplicatedWorkSpace::instance()); + DISTMATDTYPE* work_matrix = wspace.square_matrix(); + + // build a local complete matrix from a distributed matrix + dmatrix.allgather(work_matrix, numst_); + + multiply_by_matrix(work_matrix, product, ldp); +} + template void ExtendedGridOrbitals::multiply_by_matrix( const DISTMATDTYPE* const matrix, ScalarType* product, const int ldp) const @@ -413,6 +441,7 @@ void ExtendedGridOrbitals::multiply_by_matrix( memory_space_type>::allocate_host_view(product_size); MemorySpace::Memory::copy_view_to_host( product, product_size, product_host_view); + memset(product_host_view, 0, ldp * numst_ * sizeof(ScalarType)); unsigned int const phi_size = numpt_ * numst_; ScalarType* phi_host_view = MemorySpace::Memory::multiply_by_matrix( getPsi(0), phi_size, phi_host_view); // TODO this can be done on the GPU - gemm_nn_tm_.start(); LinearAlgebraUtils::MPgemmNN(numpt_, numst_, numst_, 1., phi_host_view, lda_, matrix, numst_, 0., product_host_view, ldp); - gemm_nn_tm_.stop(); MemorySpace::Memory::free_host_view( phi_host_view); @@ -458,14 +485,14 @@ void ExtendedGridOrbitals::multiplyByMatrix( { assert(matrix.nmat() == 1); - gemm_nn_tm_.start(); + prod_matrix_tm_.start(); const MATDTYPE* const mat = matrix.getSubMatrix(); LinearAlgebraUtils::MPgemmNN(numpt_, numst_, numst_, 1., getPsi(0), lda_, mat, numst_, 0., product, ldp); - gemm_nn_tm_.stop(); + prod_matrix_tm_.stop(); } // Here the result is stored in one of the matrices used in the multiplication, @@ -515,39 +542,30 @@ void ExtendedGridOrbitals::multiply_by_matrix( multiply_by_matrix(matrix, product.psi(0), product.lda_); } -#ifdef MGMOL_USE_SCALAPACK template <> template <> void ExtendedGridOrbitals::multiply_by_matrix( - const dist_matrix::DistMatrix& matrix, const double alpha, - ExtendedGridOrbitals& product) + const dist_matrix::DistMatrix& matrix) { - multiply_by_DistMatrix(matrix, alpha, product); + multiply_by_DistMatrix(matrix); } -#endif template <> template <> void ExtendedGridOrbitals::multiply_by_matrix( - const ReplicatedMatrix& matrix, const double alpha, - ExtendedGridOrbitals& product) + const ReplicatedMatrix& matrix) { - multiply_by_ReplicatedMatrix(matrix, alpha, product); + multiply_by_ReplicatedMatrix(matrix); } -#ifdef MGMOL_USE_SCALAPACK template void ExtendedGridOrbitals::multiply_by_DistMatrix( - const dist_matrix::DistMatrix& matrix, const double alpha, - ExtendedGridOrbitals& product) + const dist_matrix::DistMatrix& matrix) { prod_matrix_tm_.start(); - ScalarType* product_ptr - = (this == &product) - ? MemorySpace::Memory::allocate( - numpt_ * numst_) - : product.getPsi(0); + ScalarType* product = new ScalarType[numpt_ * numst_]; + memset(product, 0, numpt_ * numst_ * sizeof(ScalarType)); ReplicatedWorkSpace& wspace( ReplicatedWorkSpace::instance()); @@ -564,33 +582,27 @@ void ExtendedGridOrbitals::multiply_by_DistMatrix( getPsi(0), phi_size, phi_host_view); // TODO this can be done on the GPU - gemm_nn_tm_.start(); LinearAlgebraUtils::MPgemmNN(numpt_, numst_, numst_, 1., - phi_host_view, lda_, work_matrix, numst_, alpha, product_ptr, numpt_); - gemm_nn_tm_.stop(); + phi_host_view, lda_, work_matrix, numst_, 0., product, numpt_); for (int color = 0; color < numst_; color++) - memcpy(phi_host_view + color * lda_, product_ptr + color * numpt_, - slnumpt); + memcpy(phi_host_view + color * lda_, product + color * numpt_, slnumpt); MemorySpace::Memory::copy_view_to_dev( phi_host_view, phi_size, getPsi(0)); MemorySpace::Memory::free_host_view( phi_host_view); - if (this == &product) - MemorySpace::Memory::free(product_ptr); + delete[] product; prod_matrix_tm_.stop(); } -#endif template void ExtendedGridOrbitals::multiply_by_ReplicatedMatrix( - const ReplicatedMatrix& matrix, const double alpha, - ExtendedGridOrbitals& product) + const ReplicatedMatrix& matrix) { - gemm_nn_tm_.start(); + prod_matrix_tm_.start(); #ifdef HAVE_MAGMA magma_trans_t magma_transa = magma_trans_const('n'); @@ -598,43 +610,31 @@ void ExtendedGridOrbitals::multiply_by_ReplicatedMatrix( auto& magma_singleton = MagmaSingleton::get_magma_singleton(); - ScalarType* product_ptr - = (this == &product) - ? MemorySpace::Memory::allocate( - numst_ * lda_) - : product.getPsi(0); + ScalarType* tmp + = MemorySpace::Memory::allocate( + numst_ * lda_); magmablas_dgemm(magma_transa, magma_transb, numpt_, numst_, numst_, 1., - block_vector_.vect(0), lda_, matrix.data(), matrix.ld(), alpha, - product_ptr, lda_, magma_singleton.queue_); + block_vector_.vect(0), lda_, matrix.data(), matrix.ld(), 0., tmp, lda_, + magma_singleton.queue_); - if (this == &product) - { - MemorySpace::Memory::copy( - product_ptr, numst_ * lda_, block_vector_.vect(0)); + MemorySpace::Memory::copy( + tmp, numst_ * lda_, block_vector_.vect(0)); - MemorySpace::Memory::free(product); - } + MemorySpace::Memory::free(tmp); #else - ScalarType* product_ptr - = (this == &product) - ? MemorySpace::Memory::allocate( - numst_ * lda_) - : product.getPsi(0); + ScalarType* tmp + = MemorySpace::Memory::allocate( + numst_ * lda_); LinearAlgebraUtils::MPgemmNN(numpt_, numst_, numst_, 1., - block_vector_.vect(0), lda_, matrix.data(), matrix.ld(), alpha, - product_ptr, lda_); + block_vector_.vect(0), lda_, matrix.data(), matrix.ld(), 0., tmp, lda_); - if (this == &product) - { - memcpy(block_vector_.vect(0), product_ptr, - numst_ * lda_ * sizeof(ScalarType)); + memcpy(block_vector_.vect(0), tmp, numst_ * lda_ * sizeof(ScalarType)); - MemorySpace::Memory::free(product_ptr); - } + MemorySpace::Memory::free(tmp); #endif - gemm_nn_tm_.stop(); + prod_matrix_tm_.stop(); } template @@ -1077,19 +1077,12 @@ void ExtendedGridOrbitals::computeLocalProduct( const ScalarType* const array, const int ld, LocalMatrices& ss, const bool transpose) { - gemm_tn_tm_.start(); - assert(numpt_ > 0); assert(numpt_ <= ld); assert(array != nullptr); assert(numst_ != 0); assert(grid_.vel() > 0.); - Control& ct = *(Control::instance()); - if (onpe0 && ct.verbose > 2) - std::cout << "ExtendedGridOrbitals::computeLocalProduct()..." - << std::endl; - const ScalarType* const a = transpose ? array : block_vector_.vect(0); const ScalarType* const b = transpose ? block_vector_.vect(0) : array; @@ -1098,8 +1091,6 @@ void ExtendedGridOrbitals::computeLocalProduct( LinearAlgebraUtils::MPgemmTN(numst_, numst_, numpt_, grid_.vel(), a, lda, b, ldb, 0., ss.getRawPtr(0), ss.m()); - - gemm_tn_tm_.stop(); } template @@ -1123,7 +1114,6 @@ void ExtendedGridOrbitals::computeDiagonalElementsDotProduct( mmpi.allreduce(&tmp[0], &ss[0], numst_, MPI_SUM); } -#ifdef MGMOL_USE_SCALAPACK template void ExtendedGridOrbitals::computeGram( dist_matrix::DistMatrix& gram_mat) @@ -1155,7 +1145,6 @@ void ExtendedGridOrbitals::computeGram( sl2dm->accumulate(ss, gram_mat); } -#endif // compute the lower-triangular part of the overlap matrix template @@ -1271,7 +1260,6 @@ void ExtendedGridOrbitals::orthonormalizeLoewdin( } if (!multbymat) { -#ifdef MGMOL_USE_SCALAPACK ProjectedMatrices>* projmatrices = dynamic_cast< ProjectedMatrices>*>( @@ -1284,7 +1272,6 @@ void ExtendedGridOrbitals::orthonormalizeLoewdin( projmatrices->setGram2Id(getIterativeIndex()); } -#endif } if (matrixTransform == nullptr) delete localP; @@ -1519,10 +1506,8 @@ void ExtendedGridOrbitals::projectOut( // TODO this can be done on the GPU // Compute numpt_ rows (for subdomain 0) - gemm_nn_tm_.start(); LinearAlgebraUtils::MPgemmNN(numpt_, numst_, numst_, 1., phi_host_view, lda_, localMat, numst_, 0., tproduct, numpt_); - gemm_nn_tm_.stop(); MemorySpace::Memory::free_host_view( phi_host_view); @@ -1616,7 +1601,6 @@ void ExtendedGridOrbitals::initRand() resetIterativeIndex(); } -#ifdef MGMOL_USE_SCALAPACK template <> template <> void ExtendedGridOrbitals::addDotWithNcol2Matrix( @@ -1635,16 +1619,12 @@ void ExtendedGridOrbitals::addDotWithNcol2DistMatrix( assert(numst_ > 0); - Control& ct = *(Control::instance()); - if (onpe0 && ct.verbose > 2) - std::cout << "ExtendedGridOrbitals::addDotWithNcol2DistMatrix()..." - << std::endl; - const double vel = grid_.vel(); // replicated matrix const int size_work = numst_ * numst_; std::vector work(size_work); + memset(work.data(), 0, size_work * sizeof(double)); unsigned int const block_vector_size = numpt_ * numst_; ScalarType* block_vector_host_view = MemorySpace::Memory::addDotWithNcol2DistMatrix( Apsi.getPsi(0), phi_size, phi_host_view); // TODO this can be done on the GPU - gemm_tn_tm_.start(); LinearAlgebraUtils::MPgemmTN(numst_, numst_, numpt_, vel, - block_vector_host_view + 0 * numpt_, lda_, phi_host_view, lda_, 0., + block_vector_host_view + 0 * numpt_, lda_, phi_host_view, lda_, 1., work.data(), numst_); - gemm_tn_tm_.stop(); MemorySpace::Memory::free_host_view( phi_host_view); @@ -1680,7 +1658,6 @@ void ExtendedGridOrbitals::addDotWithNcol2DistMatrix( addDot_tm_.stop(); } -#endif template <> template <> @@ -1696,16 +1673,9 @@ void ExtendedGridOrbitals::addDotWithNcol2ReplicatedMatrix( { addDot_tm_.start(); - Control& ct = *(Control::instance()); - if (onpe0 && ct.verbose > 2) - std::cout - << "ExtendedGridOrbitals::addDotWithNcol2ReplicatedMatrix()..." - << std::endl; - ReplicatedMatrix tmp("tmp", numst_, numst_); const double vel = grid_.vel(); - gemm_tn_tm_.start(); #ifdef HAVE_MAGMA magma_trans_t magma_transa = magma_trans_const('t'); magma_trans_t magma_transb = magma_trans_const('n'); @@ -1720,7 +1690,6 @@ void ExtendedGridOrbitals::addDotWithNcol2ReplicatedMatrix( block_vector_.vect(0), lda_, Apsi.getPsi(0), lda_, 0., tmp.data(), tmp.ld()); #endif - gemm_tn_tm_.stop(); tmp.consolidate(); @@ -1750,8 +1719,6 @@ void ExtendedGridOrbitals::printTimers(std::ostream& os) dot_product_tm_.print(os); addDot_tm_.print(os); prod_matrix_tm_.print(os); - gemm_nn_tm_.print(os); - gemm_tn_tm_.print(os); assign_tm_.print(os); normalize_tm_.print(os); axpy_tm_.print(os); @@ -1847,6 +1814,29 @@ void ExtendedGridOrbitals::initWF( #endif } +#ifdef MGMOL_HAS_LIBROM +template +void ExtendedGridOrbitals::set(std::string file_path, int rdim) +{ + const int dim = getLocNumpt(); + + CAROM::BasisReader reader(file_path); + CAROM::Matrix* orbital_basis = reader.getSpatialBasis(rdim); + + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + pb::GridFunc gf_psi( + mymesh->grid(), ct.bcWF[0], ct.bcWF[1], ct.bcWF[2]); + CAROM::Vector psi; + for (int i = 0; i < rdim; ++i) + { + orbital_basis->getColumn(i, psi); + gf_psi.assign(psi.getData()); + setPsi(gf_psi, i); + } +} +#endif + template void ExtendedGridOrbitals::axpy( const ORBDTYPE alpha, const ExtendedGridOrbitals&); diff --git a/src/ExtendedGridOrbitals.h b/src/ExtendedGridOrbitals.h index 97d37a1f..9d161f2f 100644 --- a/src/ExtendedGridOrbitals.h +++ b/src/ExtendedGridOrbitals.h @@ -11,9 +11,7 @@ #define MGMOL_EXTENDEDGRIDORBITALS_H #include "BlockVector.h" -#ifdef MGMOL_USE_SCALAPACK #include "DistMatrix.h" -#endif #include "DotProductManager.h" #include "GridFunc.h" #include "HDFrestart.h" @@ -36,9 +34,6 @@ class ProjectedMatricesInterface; class LocalizationRegions; class ClusterOrbitals; -#ifndef MGMOL_USE_SCALAPACK -typedef double DISTMATDTYPE; -#endif template class ExtendedGridOrbitals : public Orbitals @@ -58,8 +53,6 @@ class ExtendedGridOrbitals : public Orbitals static Timer assign_tm_; static Timer normalize_tm_; static Timer axpy_tm_; - static Timer gemm_nn_tm_; - static Timer gemm_tn_tm_; static int lda_; // leading dimension for storage static int numpt_; @@ -89,15 +82,14 @@ class ExtendedGridOrbitals : public Orbitals // void projectOut(ScalarType* const, const int); - void multiply_by_ReplicatedMatrix(const ReplicatedMatrix& matrix, - const double alpha, ExtendedGridOrbitals& product); -#ifdef MGMOL_USE_SCALAPACK + void multiply_by_ReplicatedMatrix(const ReplicatedMatrix& matrix); void multiply_by_DistMatrix( - const dist_matrix::DistMatrix& matrix, const double alpha, - ExtendedGridOrbitals& product); -#endif + const dist_matrix::DistMatrix& matrix); + void multiply_by_matrix( const DISTMATDTYPE* const, ScalarType*, const int) const; + void multiply_by_matrix(const dist_matrix::DistMatrix& matrix, + ScalarType* const product, const int ldp); void scal(const int i, const double alpha) { block_vector_.scal(i, alpha); } virtual void assign(const int i, const ScalarType* const v, const int n = 1) { @@ -124,10 +116,8 @@ class ExtendedGridOrbitals : public Orbitals /*! * Specialized functions */ -#ifdef MGMOL_USE_SCALAPACK void addDotWithNcol2DistMatrix( ExtendedGridOrbitals&, dist_matrix::DistMatrix&) const; -#endif void addDotWithNcol2ReplicatedMatrix( ExtendedGridOrbitals&, ReplicatedMatrix&) const; @@ -312,11 +302,9 @@ class ExtendedGridOrbitals : public Orbitals void computeGram(const int verbosity = 0); void computeGramAndInvS(const int verbosity = 0); -#ifdef MGMOL_USE_SCALAPACK void computeGram(dist_matrix::DistMatrix& gram_mat); void computeGram(const ExtendedGridOrbitals& orbitals, dist_matrix::DistMatrix& gram_mat); -#endif ScalarType maxAbsValue() const { return block_vector_.maxAbsValue(); } @@ -386,8 +374,7 @@ class ExtendedGridOrbitals : public Orbitals void multiply_by_matrix( const DISTMATDTYPE* const matrix, ExtendedGridOrbitals& product) const; template - void multiply_by_matrix(const MatrixType&, const double alpha, - ExtendedGridOrbitals& product); + void multiply_by_matrix(const MatrixType&); void multiplyByMatrix2states(const int st1, const int st2, const double* mat, ExtendedGridOrbitals& product); @@ -417,18 +404,10 @@ class ExtendedGridOrbitals : public Orbitals const pb::Grid& mygrid = mymesh->grid(); return mygrid.maxDomainSize(); } -}; -template -int ExtendedGridOrbitals::lda_ = 0; -template -int ExtendedGridOrbitals::numpt_ = 0; -template -int ExtendedGridOrbitals::data_wghosts_index_ = -1; -template -int ExtendedGridOrbitals::numst_ = -1; -template -std::vector> - ExtendedGridOrbitals::overlapping_gids_; +#ifdef MGMOL_HAS_LIBROM + void set(std::string file_path, int rdim); +#endif +}; #endif diff --git a/src/Forces.cc b/src/Forces.cc index 42261056..96fcf56d 100644 --- a/src/Forces.cc +++ b/src/Forces.cc @@ -461,7 +461,6 @@ Forces::getReplicatedDM() proj_matrices_); if (projmatrices) return projmatrices->getReplicatedDM(); } -#ifdef MGMOL_USE_SCALAPACK { ProjectedMatrices>* projmatrices = dynamic_cast< @@ -470,7 +469,6 @@ Forces::getReplicatedDM() assert(projmatrices); return projmatrices->getReplicatedDM(); } -#endif } // Get the nl energy as the trace of loc_kbpsi*mat_X for several loc_kbpsi diff --git a/src/GramMatrix.cc b/src/GramMatrix.cc index 76fa1419..1443d450 100644 --- a/src/GramMatrix.cc +++ b/src/GramMatrix.cc @@ -8,18 +8,13 @@ // Please also read this link https://github.com/llnl/mgmol/LICENSE #include "GramMatrix.h" -#include "Power.h" -#include "ReplicatedMatrix.h" -#include "ReplicatedVector.h" - -#ifdef MGMOL_USE_SCALAPACK #include "DistMatrix.h" #include "DistMatrix2SquareLocalMatrices.h" #include "DistMatrixTools.h" #include "DistVector.h" -#else -typedef double DISTMATDTYPE; -#endif +#include "Power.h" +#include "ReplicatedMatrix.h" +#include "ReplicatedVector.h" #include #include @@ -114,7 +109,6 @@ void GramMatrix::solveLST(MatrixType& z) const ls_->trtrs('l', 't', 'n', z); } -#ifdef MGMOL_USE_SCALAPACK template <> double GramMatrix>::computeCond() { @@ -141,7 +135,6 @@ double GramMatrix>::computeCond() return cond; } -#endif template <> double GramMatrix::computeCond() @@ -332,10 +325,8 @@ void GramMatrix::applyInv(VectorType& mat) ls_->potrs('l', mat); } -#ifdef MGMOL_USE_SCALAPACK template class GramMatrix>; template void GramMatrix>::applyInv( dist_matrix::DistVector&); -#endif template class GramMatrix; template void GramMatrix::applyInv(ReplicatedVector&); diff --git a/src/GrassmanCG.h b/src/GrassmanCG.h index 4e7770ba..01c5b394 100644 --- a/src/GrassmanCG.h +++ b/src/GrassmanCG.h @@ -10,10 +10,7 @@ #ifndef MGMOL_GRASSMANCG_H #define MGMOL_GRASSMANCG_H -#ifdef MGMOL_USE_SCALAPACK #include "DistMatrix.h" -#endif - #include "GrassmanLineMinimization.h" #include "Hamiltonian.h" #include "Potentials.h" @@ -39,12 +36,10 @@ class GrassmanCG : public GrassmanLineMinimization void conjugate() override; double computeStepSize(T& orbitals) override; -#ifdef MGMOL_USE_SCALAPACK void computeOrbitalsProdWithH( T& orbitals1, T& orbitals2, dist_matrix::DistMatrix& mat); void computeOrbitalsProdWithH( T& orbitals, dist_matrix::DistMatrix& mat); -#endif void parallelTransportUpdate(const double lambda, T& orbitals) override; }; #endif diff --git a/src/Hamiltonian.cc b/src/Hamiltonian.cc index 0eb2031d..28aff992 100644 --- a/src/Hamiltonian.cc +++ b/src/Hamiltonian.cc @@ -171,7 +171,6 @@ void Hamiltonian::applyDeltaPot(const T& phi, T& hphi) // add to hij the elements // corresponding to the local part of the Hamiltonian -#ifdef MGMOL_USE_SCALAPACK template <> template <> void Hamiltonian>::addHlocal2matrix( @@ -207,7 +206,6 @@ void Hamiltonian>::addHlocal2matrix( // hij.print(std::cout, 0, 0, 5, 5); } -#endif template <> template <> diff --git a/src/HamiltonianMVPSolver.cc b/src/HamiltonianMVPSolver.cc index f94d682e..c697b5eb 100644 --- a/src/HamiltonianMVPSolver.cc +++ b/src/HamiltonianMVPSolver.cc @@ -350,15 +350,15 @@ void HamiltonianMVPSolver, ProjectedMatrices>, LocGridOrbitals>; + +template class HamiltonianMVPSolver, + ProjectedMatricesSparse, LocGridOrbitals>; + template class HamiltonianMVPSolver, ProjectedMatrices>, ExtendedGridOrbitals>; -#endif -template class HamiltonianMVPSolver, - ProjectedMatricesSparse, LocGridOrbitals>; template class HamiltonianMVPSolver, ExtendedGridOrbitals>; diff --git a/src/HamiltonianMVP_DMStrategy.cc b/src/HamiltonianMVP_DMStrategy.cc index 456e532f..1d03a9b3 100644 --- a/src/HamiltonianMVP_DMStrategy.cc +++ b/src/HamiltonianMVP_DMStrategy.cc @@ -9,6 +9,7 @@ #include "HamiltonianMVP_DMStrategy.h" #include "Control.h" +#include "DistMatrix.h" #include "HamiltonianMVPSolver.h" #include "Ions.h" #include "LocGridOrbitals.h" @@ -17,10 +18,6 @@ #include "ProjectedMatricesSparse.h" #include "ReplicatedMatrix.h" -#ifdef MGMOL_USE_SCALAPACK -#include "DistMatrix.h" -#endif - template HamiltonianMVP_DMStrategy::HamiltonianMVP_DMStrategy(MPI_Comm comm, std::ostream& os, @@ -101,15 +98,14 @@ void HamiltonianMVP_DMStrategyreset(); } -#ifdef MGMOL_USE_SCALAPACK template class HamiltonianMVP_DMStrategy, ProjectedMatrices>, LocGridOrbitals>; +template class HamiltonianMVP_DMStrategy, + ProjectedMatricesSparse, LocGridOrbitals>; + template class HamiltonianMVP_DMStrategy, ProjectedMatrices>, ExtendedGridOrbitals>; -#endif -template class HamiltonianMVP_DMStrategy, - ProjectedMatricesSparse, LocGridOrbitals>; template class HamiltonianMVP_DMStrategy, ExtendedGridOrbitals>; diff --git a/src/Hartree_CG.h b/src/Hartree_CG.h index d51c422b..6d048970 100644 --- a/src/Hartree_CG.h +++ b/src/Hartree_CG.h @@ -27,10 +27,9 @@ class Hartree_CG : public Poisson Hartree_CG(const pb::Grid& grid, const short bc[3]) : Poisson(grid, bc) { OperatorType oper(Poisson::grid_); - Control& ct = *(Control::instance()); poisson_solver_ = std::make_shared>( - oper, ct.lap_type, bc[0], bc[1], bc[2]); + oper, bc[0], bc[1], bc[2]); }; // Destructor diff --git a/src/Ions.cc b/src/Ions.cc index de62212b..f995fdac 100644 --- a/src/Ions.cc +++ b/src/Ions.cc @@ -200,6 +200,8 @@ void Ions::setup() ions_setup_tm.start(); + Ion::resetIndexCount(); + updateListIons(); // #ifndef NDEBUG diff --git a/src/KBPsiMatrixSparse.cc b/src/KBPsiMatrixSparse.cc index d1cf791a..b12604da 100644 --- a/src/KBPsiMatrixSparse.cc +++ b/src/KBPsiMatrixSparse.cc @@ -16,10 +16,7 @@ #include "Mesh.h" #include "ProjectedMatrices.h" #include "ReplicatedMatrix.h" - -#ifdef MGMOL_USE_SCALAPACK #include "SquareSubMatrix2DistMatrix.h" -#endif #include @@ -418,7 +415,6 @@ void KBPsiMatrixSparse::computeHvnlMatrix( computeHvnlMatrix(this, ions, proj_matrices); } -#ifdef MGMOL_USE_SCALAPACK template <> void KBPsiMatrixSparse::computeHvnlMatrix( const KBPsiMatrixInterface* const kbpsi2, const Ions& ions, @@ -429,7 +425,6 @@ void KBPsiMatrixSparse::computeHvnlMatrix( SquareSubMatrix2DistMatrix* ss2dm = SquareSubMatrix2DistMatrix::instance(); ss2dm->accumulate(submat, hij, 0.); } -#endif template <> void KBPsiMatrixSparse::computeHvnlMatrix( @@ -701,9 +696,7 @@ template void KBPsiMatrixSparse::computeKBpsi(const Ions& ions, template void KBPsiMatrixSparse::computeAll( const Ions&, ExtendedGridOrbitals&); -#ifdef MGMOL_USE_SCALAPACK template double KBPsiMatrixSparse::getEvnl(const Ions& ions, ProjectedMatrices>* proj_matrices); -#endif template double KBPsiMatrixSparse::getEvnl( const Ions& ions, ProjectedMatrices* proj_matrices); diff --git a/src/LapFactory.h b/src/LapFactory.h index 22afd663..50228157 100644 --- a/src/LapFactory.h +++ b/src/LapFactory.h @@ -6,8 +6,10 @@ // All rights reserved. // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -#ifndef MGMOL_LAPFACTORY_H -#define MGMOL_LAPFACTORY_H + +// $Id$ +#ifndef LAPFACTORY_H +#define LAPFACTORY_H #include "Laph2.h" #include "Laph4.h" @@ -15,7 +17,7 @@ #include "Laph4MP.h" #include "Laph6.h" #include "Laph8.h" -#include "MGmol_MPI.h" +#include "MPIdata.h" template class LapFactory @@ -23,7 +25,7 @@ class LapFactory public: static pb::Lap* createLap(const pb::Grid& grid, const int type) { - pb::Lap* lap = nullptr; + pb::Lap* lap; switch (type) { case 0: @@ -45,12 +47,10 @@ class LapFactory lap = new pb::Laph4MP(grid); break; default: - MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - if (mmpi.instancePE0()) - std::cerr - << "LapFactory::createLap() --- option invalid:" << type - << std::endl; - mmpi.abort(); + (*MPIdata::serr) + << "LapFactory::createLap() --- option invalid:" << type + << std::endl; + exit(2); } return lap; } diff --git a/src/LocGridOrbitals.cc b/src/LocGridOrbitals.cc index 7dc5c00d..8b22ada8 100644 --- a/src/LocGridOrbitals.cc +++ b/src/LocGridOrbitals.cc @@ -13,6 +13,7 @@ #include "ColoredRegions.h" #include "Control.h" +#include "DistMatrix.h" #include "DotProductManagerFactory.h" #include "FunctionsPacking.h" #include "GridFunc.h" @@ -20,6 +21,7 @@ #include "HDFrestart.h" #include "Laph4M.h" #include "LocGridOrbitals.h" +#include "LocalMatrices2DistMatrix.h" #include "LocalizationRegions.h" #include "MPIdata.h" #include "Masks4Orbitals.h" @@ -34,11 +36,6 @@ #include "lapack_c.h" #include "memory_space.h" -#ifdef MGMOL_USE_SCALAPACK -#include "DistMatrix.h" -#include "LocalMatrices2DistMatrix.h" -#endif - #include #include #include @@ -52,6 +49,18 @@ DotProductManager>* LocGridOrbitals::dotProductManager_ = nullptr; +template +short LocGridOrbitals::subdivx_ = 0; +template +int LocGridOrbitals::lda_ = 0; +template +int LocGridOrbitals::numpt_ = 0; +template +int LocGridOrbitals::loc_numpt_ = 0; + +template +int LocGridOrbitals::data_wghosts_index_ = -1; + template Timer LocGridOrbitals::get_dm_tm_( "LocGridOrbitals" + std::to_string(8 * sizeof(ScalarType)) + "::get_dm"); @@ -781,7 +790,6 @@ int LocGridOrbitals::packStates( return pack_->chromatic_number(); } -#ifdef MGMOL_USE_SCALAPACK template void LocGridOrbitals::multiply_by_matrix( const dist_matrix::DistMatrix& dmatrix, @@ -796,7 +804,6 @@ void LocGridOrbitals::multiply_by_matrix( multiply_by_matrix(0, chromatic_number_, work_matrix, product, ldp); } -#endif template void LocGridOrbitals::multiply_by_matrix(const int first_color, @@ -977,17 +984,14 @@ void LocGridOrbitals::multiply_by_matrix( 0, chromatic_number_, matrix, product.psi(0), product.lda_); } -#ifdef MGMOL_USE_SCALAPACK template void LocGridOrbitals::multiply_by_matrix( - const dist_matrix::DistMatrix& matrix, const double alpha, - LocGridOrbitals& product) + const dist_matrix::DistMatrix& matrix) { prod_matrix_tm_.start(); - ScalarType* product_ptr - = (this == &product) ? new ScalarType[loc_numpt_ * chromatic_number_] - : product.getPsi(0); + ScalarType* product = new ScalarType[loc_numpt_ * chromatic_number_]; + memset(product, 0, loc_numpt_ * chromatic_number_ * sizeof(ScalarType)); ReplicatedWorkSpace& wspace( ReplicatedWorkSpace::instance()); @@ -1010,19 +1014,17 @@ void LocGridOrbitals::multiply_by_matrix( // Compute loc_numpt_ rows (for subdomain iloc) LinearAlgebraUtils::MPgemmNN(loc_numpt_, chromatic_number_, chromatic_number_, 1., phi, lda_, matrix_local, - chromatic_number_, alpha, product_ptr, loc_numpt_); + chromatic_number_, 0., product, loc_numpt_); for (int color = 0; color < chromatic_number_; color++) - memcpy( - phi + color * lda_, product_ptr + color * loc_numpt_, slnumpt); + memcpy(phi + color * lda_, product + color * loc_numpt_, slnumpt); } delete[] matrix_local; - if (this == &product) delete[] product_ptr; + delete[] product; prod_matrix_tm_.stop(); } -#endif template int LocGridOrbitals::read_hdf5(HDFrestart& h5f_file) @@ -1736,7 +1738,6 @@ void LocGridOrbitals::computeDiagonalElementsDotProductLocal( } } -#ifdef MGMOL_USE_SCALAPACK template void LocGridOrbitals::computeGram( dist_matrix::DistMatrix& gram_mat) @@ -1760,7 +1761,6 @@ void LocGridOrbitals::computeGram( sl2dm->accumulate(ss, gram_mat); } -#endif // compute the lower-triangular part of the overlap matrix template @@ -1862,27 +1862,14 @@ void LocGridOrbitals::orthonormalizeLoewdin( localP = new SquareLocalMatrices( subdivx_, chromatic_number_); - // try with ReplicatedMatrix first - ProjectedMatrices* projmatrices - = dynamic_cast*>(proj_matrices_); - if (projmatrices) - { - projmatrices->computeLoewdinTransform( - *localP, getIterativeIndex(), update_matrices); - } -#ifdef MGMOL_USE_SCALAPACK - else - { - ProjectedMatrices>* projmatrices - = dynamic_cast< - ProjectedMatrices>*>( - proj_matrices_); - assert(projmatrices != nullptr); - assert(localP); - projmatrices->computeLoewdinTransform( - *localP, getIterativeIndex(), update_matrices); - } -#endif + ProjectedMatrices>* projmatrices + = dynamic_cast< + ProjectedMatrices>*>( + proj_matrices_); + assert(projmatrices != nullptr); + assert(localP); + projmatrices->computeLoewdinTransform( + *localP, getIterativeIndex(), update_matrices); multiplyByMatrix(*localP); @@ -2466,7 +2453,6 @@ void LocGridOrbitals::initRand() } // Compute nstates column of Psi^T*A*Psi starting at column 0 -#ifdef MGMOL_USE_SCALAPACK template void LocGridOrbitals::addDotWithNcol2Matrix( LocGridOrbitals& Apsi, @@ -2512,7 +2498,6 @@ void LocGridOrbitals::addDotWithNcol2Matrix( addDot_tm_.stop(); } -#endif template void LocGridOrbitals::computeGlobalIndexes( diff --git a/src/LocGridOrbitals.h b/src/LocGridOrbitals.h index 117b085b..e7f97f00 100644 --- a/src/LocGridOrbitals.h +++ b/src/LocGridOrbitals.h @@ -38,10 +38,6 @@ class MasksSet; class Masks4Orbitals; class ReplicatedMatrix; -#ifndef MGMOL_USE_SCALAPACK -typedef double DISTMATDTYPE; -#endif - template class LocGridOrbitals : public Orbitals { @@ -108,10 +104,8 @@ class LocGridOrbitals : public Orbitals const DISTMATDTYPE* const matrix, LocGridOrbitals& product) const; void multiply_by_matrix(const int, const int, const DISTMATDTYPE* const, ScalarType*, const int) const; -#ifdef MGMOL_USE_SCALAPACK void multiply_by_matrix(const dist_matrix::DistMatrix& matrix, ScalarType* const product, const int ldp); -#endif void scal(const int i, const double alpha) { block_vector_.scal(i, alpha); } virtual void assign(const int i, const ScalarType* const v, const int n = 1) { @@ -334,11 +328,9 @@ class LocGridOrbitals : public Orbitals void computeGram(const int verbosity = 0); void computeGramAndInvS(const int verbosity = 0); -#ifdef MGMOL_USE_SCALAPACK void computeGram(dist_matrix::DistMatrix& gram_mat); void computeGram(const LocGridOrbitals& orbitals, dist_matrix::DistMatrix& gram_mat); -#endif ScalarType maxAbsValue() const { return block_vector_.maxAbsValue(); } @@ -364,10 +356,8 @@ class LocGridOrbitals : public Orbitals void getLocalOverlap(const LocGridOrbitals& orbitals, SquareLocalMatrices&); -#ifdef MGMOL_USE_SCALAPACK void addDotWithNcol2Matrix( LocGridOrbitals&, dist_matrix::DistMatrix&) const; -#endif void addDotWithNcol2Matrix(LocGridOrbitals&, ReplicatedMatrix&) const { std::cerr << "LocGridOrbitals::addDotWithNcol2Matrix not implemented " @@ -417,10 +407,7 @@ class LocGridOrbitals : public Orbitals LocGridOrbitals& product) const; void multiply_by_matrix( const DISTMATDTYPE* const matrix, LocGridOrbitals& product) const; -#ifdef MGMOL_USE_SCALAPACK - void multiply_by_matrix(const dist_matrix::DistMatrix&, - const double alpha, LocGridOrbitals& product); -#endif + void multiply_by_matrix(const dist_matrix::DistMatrix&); void multiplyByMatrix2states(const int st1, const int st2, const double* mat, LocGridOrbitals& product); @@ -449,14 +436,4 @@ class LocGridOrbitals : public Orbitals double getMaxR() const { return 2. * lrs_->max_radii(); } }; -template -short LocGridOrbitals::subdivx_ = 0; -template -int LocGridOrbitals::lda_ = 0; -template -int LocGridOrbitals::numpt_ = 0; -template -int LocGridOrbitals::loc_numpt_ = 0; -template -int LocGridOrbitals::data_wghosts_index_ = -1; #endif diff --git a/src/LocalizationRegions.cc b/src/LocalizationRegions.cc index 1f9436f7..47aaf209 100644 --- a/src/LocalizationRegions.cc +++ b/src/LocalizationRegions.cc @@ -12,16 +12,13 @@ #include "LocGridOrbitals.h" #include "LocalizationRegions.h" #include "Mesh.h" +#include "OrbitalsTransform.h" #include "SpreadsAndCenters.h" #include "SquareLocalMatrices.h" #include "SymmetricPair.h" #include "hdf_tools.h" #include "tools.h" -#ifdef MGMOL_USE_SCALAPACK -#include "OrbitalsTransform.h" -#endif - #include #include #include @@ -392,7 +389,6 @@ float LocalizationRegions::updateRadiiConstVol(const SpreadsAndCenters& sc) return updateRadii(sc, ratio); } -#ifdef MGMOL_USE_SCALAPACK float LocalizationRegions::updateRadii( const OrbitalsTransform* ot, const float ratio) { @@ -405,7 +401,6 @@ float LocalizationRegions::updateRadii( return getMeanRadius(); } -#endif float LocalizationRegions::moveTo(const vector& target_centers) { diff --git a/src/LocalizationRegions.h b/src/LocalizationRegions.h index 702430ce..f835be49 100644 --- a/src/LocalizationRegions.h +++ b/src/LocalizationRegions.h @@ -22,14 +22,11 @@ #include "mgmol_mpi_tools.h" #include "tools.h" -#ifdef MGMOL_USE_SCALAPACK -#include "OrbitalsTransform.h" -#endif - #include #include #include +class OrbitalsTransform; class SymmetricPair; typedef struct LRData @@ -401,9 +398,7 @@ class LocalizationRegions template float move(const SpreadsAndCenters& sc, const bool flag = false); -#ifdef MGMOL_USE_SCALAPACK float updateRadii(const OrbitalsTransform* ot, const float ratio); -#endif template float updateRadii(const SpreadsAndCenters& sc, const float ratio); template diff --git a/src/MGmol.cc b/src/MGmol.cc index 8e9df794..cec599a5 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -22,6 +22,8 @@ #include "DFTsolver.h" #include "DMStrategyFactory.h" #include "DavidsonSolver.h" +#include "DistMatrix.h" +#include "DistMatrix2SquareLocalMatrices.h" #include "Electrostatic.h" #include "Energy.h" #include "EnergySpreadPenalty.h" @@ -29,6 +31,7 @@ #include "FDoper.h" #include "FIRE.h" #include "Forces.h" +#include "GrassmanLineMinimization.h" #include "GridFunc.h" #include "HDFrestart.h" #include "Hamiltonian.h" @@ -42,6 +45,7 @@ #include "MGOrbitalsPreconditioning.h" #include "MGkernels.h" #include "MGmol.h" +#include "MLWFTransform.h" #include "MPIdata.h" #include "MVPSolver.h" #include "MasksSet.h" @@ -59,24 +63,16 @@ #include "ReplicatedVector.h" #include "Rho.h" #include "SP2.h" +#include "SparseDistMatrix.h" #include "SpreadPenalty.h" #include "SpreadPenaltyVolume.h" #include "SpreadsAndCenters.h" +#include "SubMatrices.h" #include "SubspaceProjector.h" #include "XCfunctionalFactory.h" #include "XConGrid.h" #include "manage_memory.h" -#ifdef MGMOL_USE_SCALAPACK -#include "DistMatrix.h" -#include "DistMatrix2SquareLocalMatrices.h" -#include "DistVector.h" -#include "GrassmanLineMinimization.h" -#include "MLWFTransform.h" -#include "SparseDistMatrix.h" -#include "SubMatrices.h" -#endif - namespace mgmol { std::ostream* out = nullptr; @@ -104,20 +100,19 @@ extern Timer loopdot_tm; extern Timer loopaxpy_tm; extern Timer loopscal_tm; extern Timer loopcp_tm; +extern Timer get_NOLMO_tm; +extern Timer get_MLWF_tm; extern Timer md_iterations_tm; extern Timer md_tau_tm; extern Timer md_moveVnuc_tm; extern Timer md_updateMasks_tm; extern Timer md_extrapolateOrbitals_tm; extern Timer md_updateRhoAndPot_tm; +extern Timer md_updateDMandEnergy_tm; extern Timer quench_tm; extern Timer ions_setupInteractingIons_tm; extern Timer ions_setup_tm; extern Timer updateCenters_tm; -#ifdef MGMOL_USE_SCALAPACK -extern Timer get_NOLMO_tm; -extern Timer get_MLWF_tm; -#endif #include "mgmol_Signal.h" std::set Signal::recv_; @@ -255,15 +250,9 @@ int MGmol::initial() new ProjectedMatricesMehrstellen( ct.numst, with_spin, ct.occ_width)); else - { -#ifdef MGMOL_USE_SCALAPACK proj_matrices_.reset(new ProjectedMatricesMehrstellen< dist_matrix::DistMatrix>( ct.numst, with_spin, ct.occ_width)); -#else - std::cerr << "Not implemented" << std::endl; -#endif - } } else if (ct.short_sighted) proj_matrices_.reset(new ProjectedMatricesSparse( @@ -272,15 +261,9 @@ int MGmol::initial() proj_matrices_.reset(new ProjectedMatrices( ct.numst, with_spin, ct.occ_width)); else - { -#ifdef MGMOL_USE_SCALAPACK proj_matrices_.reset( new ProjectedMatrices>( ct.numst, with_spin, ct.occ_width)); -#else - std::cerr << "Not implemented" << std::endl; -#endif - } forces_.reset(new Forces( hamiltonian_.get(), rho_.get(), proj_matrices_.get())); @@ -482,14 +465,10 @@ int MGmol::initial() hamiltonian_.get(), this, proj_matrices_.get(), current_orbitals_)); else - { -#ifdef MGMOL_USE_SCALAPACK dm_strategy_.reset(DMStrategyFactory>::create(comm_, os_, *ions_, rho_.get(), energy_.get(), electrostat_.get(), hamiltonian_.get(), this, proj_matrices_.get(), current_orbitals_)); -#endif - } // theta = invB * Hij proj_matrices_->updateThetaAndHB(); @@ -593,17 +572,12 @@ void MGmol::printMM() std::ofstream tfile("s.mm", std::ios::out); proj_matrices_->printGramMM(tfile); std::ofstream tfileh("h.mm", std::ios::out); -#ifdef MGMOL_USE_SCALAPACK ProjectedMatrices>* projmatrices = dynamic_cast< ProjectedMatrices>*>( proj_matrices_.get()); assert(projmatrices != nullptr); projmatrices->printHamiltonianMM(tfileh); -#else - std::cerr << "MGmol::printMM() not implemented" - << std::endl; -#endif } } @@ -660,7 +634,6 @@ void MGmol::write_header() << (omp_get_max_threads() > 1 ? "s " : " "); os_ << "active" << std::endl << std::endl; #endif -#ifdef MGMOL_USE_SCALAPACK if (!ct.rmatrices) { os_ << " ScaLapack block size: " @@ -673,7 +646,6 @@ void MGmol::write_header() MatricesBlacsContext& mbc(MatricesBlacsContext::instance()); mbc.print(os_); } -#endif ct.printPoissonOptions(os_); } // onpe0 @@ -774,17 +746,20 @@ void MGmol::printEigAndOcc() && ct.occupationWidthIsZero()) && onpe0) { + bool printflag = false; // try with ReplicatedMatrix first - std::shared_ptr> projmatrices - = std::dynamic_pointer_cast>( - proj_matrices_); - if (projmatrices) { - projmatrices->printEigenvalues(os_); - projmatrices->printOccupations(os_); + std::shared_ptr> projmatrices + = std::dynamic_pointer_cast< + ProjectedMatrices>(proj_matrices_); + if (projmatrices) + { + projmatrices->printEigenvalues(os_); + projmatrices->printOccupations(os_); + printflag = true; + } } -#ifdef MGMOL_USE_SCALAPACK - else + if (!printflag) { std::shared_ptr< ProjectedMatrices>> @@ -796,7 +771,6 @@ void MGmol::printEigAndOcc() projmatrices->printEigenvalues(os_); projmatrices->printOccupations(os_); } -#endif } } @@ -869,8 +843,7 @@ void MGmol::printTimers() "====== " << std::endl; } - pb::GridFunc::printTimers(os_); - pb::GridFunc::printTimers(os_); + pb::GridFuncInterface::printTimers(os_); pb::GridFuncVector::printTimers(os_); pb::GridFuncVector::printTimers(os_); pb::printMGkernelTimers(os_); @@ -897,6 +870,14 @@ void MGmol::printTimers() loopscal_tm.print(os_); loopdot_tm.print(os_); + dist_matrix::SubMatrices::printTimers(os_); + + DistMatrix2SquareLocalMatrices::printTimers(os_); + + dist_matrix::SparseDistMatrix::printTimers(os_); + + dist_matrix::DistMatrix::printTimers(os_); + ReplicatedMatrix2SquareLocalMatrices::printTimers(os_); LocalMatrices2ReplicatedMatrix::printTimers(os_); @@ -913,15 +894,8 @@ void MGmol::printTimers() get_res_tm_.print(os_); comp_res_tm_.print(os_); vnlpsi_tm.print(os_); -#ifdef MGMOL_USE_SCALAPACK - dist_matrix::SubMatrices::printTimers(os_); - DistMatrix2SquareLocalMatrices::printTimers(os_); - dist_matrix::SparseDistMatrix::printTimers(os_); - dist_matrix::DistMatrix::printTimers(os_); - get_MLWF_tm.print(os_); get_NOLMO_tm.print(os_); -#endif Energy::eval_te_tm().print(os_); Electrostatic::solve_tm().print(os_); PoissonInterface::printTimers(os_); @@ -930,10 +904,8 @@ void MGmol::printTimers() ShortSightedInverse::printTimers(os_); if (std::is_same>::value) { -#ifdef MGMOL_USE_SCALAPACK MVPSolver, dist_matrix::DistMatrix>::printTimers(os_); -#endif MVPSolver, ReplicatedMatrix>::printTimers(os_); } @@ -953,10 +925,8 @@ void MGmol::printTimers() forces_->printTimers(os_); if (ct.OuterSolver() == OuterSolverType::ABPG) ABPG::printTimers(os_); -#ifdef MGMOL_USE_SCALAPACK else if (ct.OuterSolver() == OuterSolverType::NLCG) GrassmanLineMinimization::printTimers(os_); -#endif adaptLR_tm_.print(os_); updateCenters_tm.print(os_); md_iterations_tm.print(os_); @@ -965,6 +935,7 @@ void MGmol::printTimers() init_nuc_tm_.print(os_); md_updateMasks_tm.print(os_); md_extrapolateOrbitals_tm.print(os_); + md_updateDMandEnergy_tm.print(os_); quench_tm.print(os_); evnl_tm_.print(os_); ions_setupInteractingIons_tm.print(os_); @@ -980,10 +951,8 @@ void MGmol::printTimers() DavidsonSolver, ReplicatedMatrix>::printTimers(os_); ChebyshevApproximation::printTimers(os_); -#ifdef MGMOL_USE_SCALAPACK PowerGen, dist_matrix::DistVector>::printTimers(os_); -#endif BlockVector::printTimers(os_); if (ct.rmatrices) { @@ -991,7 +960,6 @@ void MGmol::printTimers() ReplicatedMatrix>::printTimers(os_); ChebyshevApproximation::printTimers(os_); } -#ifdef MGMOL_USE_SCALAPACK else { DavidsonSolver, @@ -999,7 +967,6 @@ void MGmol::printTimers() ChebyshevApproximation< dist_matrix::DistMatrix>::printTimers(os_); } -#endif MGOrbitalsPreconditioning::printTimers(os_); MGOrbitalsPreconditioning::printTimers(os_); MDfiles::printTimers(os_); @@ -1054,7 +1021,7 @@ double MGmol::get_evnl(const Ions& ions) evnl_tm_.start(); Control& ct = *(Control::instance()); - double val = std::numeric_limits::signaling_NaN(); + double val; if (ct.short_sighted) { std::shared_ptr projmatrices @@ -1075,7 +1042,6 @@ double MGmol::get_evnl(const Ions& ions) val = g_kbpsi_->getEvnl(ions, projmatrices.get()); } -#ifdef MGMOL_USE_SCALAPACK else { std::shared_ptr< @@ -1087,7 +1053,6 @@ double MGmol::get_evnl(const Ions& ions) val = g_kbpsi_->getEvnl(ions, projmatrices.get()); } -#endif } evnl_tm_.stop(); @@ -1167,6 +1132,19 @@ void MGmol::dumpRestart() if (ierr < 0) os_ << "WARNING: writing restart data failed!!!" << std::endl; + +#ifdef MGMOL_HAS_LIBROM + // Save orbital snapshots + if (ct.getROMOptions().save_librom_snapshot > 0 + && ct.AtomsDynamic() == AtomsDynamicType::Quench) + { + ierr = save_orbital_snapshot(filename, *current_orbitals_); + + if (ierr < 0) + os_ << "WARNING: writing ROM snapshot data failed!!!" + << std::endl; + } +#endif } } @@ -1479,6 +1457,35 @@ void MGmol::getAtomicNumbers(std::vector& an) ions_->getAtomicNumbers(an); } +template +void MGmol::updateDMandEnergy( + OrbitalsType& orbitals, Ions& ions, double& eks) +{ + // initialize electronic density + rho_->update(orbitals); + + // initialize potential + update_pot(ions); + + // initialize projected matrices + updateHmatrix(orbitals, ions); + proj_matrices_->updateThetaAndHB(); + + // compute DM + std::shared_ptr> dm_strategy( + DMStrategyFactory>::create(comm_, os_, ions, + rho_.get(), energy_.get(), electrostat_.get(), hamiltonian_.get(), + this, proj_matrices_.get(), &orbitals)); + + dm_strategy->update(orbitals); + + // evaluate energy and forces + double ts = 0.; + eks = energy_->evaluateTotal( + ts, proj_matrices_.get(), ions, orbitals, 2, os_); +} + template double MGmol::evaluateEnergyAndForces( const std::vector& tau, const std::vector& atnumbers, @@ -1554,7 +1561,6 @@ double MGmol::evaluateDMandEnergyAndForces(Orbitals* orbitals, dm_strategy->update(*dorbitals); } -#ifdef MGMOL_USE_SCALAPACK else { std::shared_ptr> dm_strategy( @@ -1565,7 +1571,6 @@ double MGmol::evaluateDMandEnergyAndForces(Orbitals* orbitals, dm_strategy->update(*dorbitals); } -#endif // evaluate energy and forces double ts = 0.; diff --git a/src/MGmol.h b/src/MGmol.h index 3d183f19..a2ffa62f 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -10,6 +10,8 @@ #ifndef MGMOL_H #define MGMOL_H +#include "mgmol_config.h" + #include "Energy.h" #include "GridFuncVector.h" #include "Hamiltonian.h" @@ -42,6 +44,7 @@ class IonicAlgorithm; #include "AOMMprojector.h" #include "ClusterOrbitals.h" #include "DMStrategy.h" +#include "Energy.h" #include "ExtendedGridOrbitals.h" #include "Forces.h" #include "Ions.h" @@ -128,13 +131,11 @@ class MGmol : public MGmolInterface Rho& rho, const bool write_extrapolated_wf, const short count); -#ifdef MGMOL_USE_SCALAPACK void swapColumnsVect(dist_matrix::DistMatrix& evect, const dist_matrix::DistMatrix& hb2N, const std::vector& eval, dist_matrix::DistMatrix& work); void wftransform(OrbitalsType*, OrbitalsType*, Ions&); -#endif int readLRsFromInput(std::ifstream* tfile); void preWFextrapolation(); void postWFextrapolation(OrbitalsType* orbitals); @@ -172,7 +173,7 @@ class MGmol : public MGmolInterface ~MGmol() override; - void setup() override; + void setup(); /* access functions */ OrbitalsType* getOrbitals() { return current_orbitals_; } @@ -181,6 +182,7 @@ class MGmol : public MGmolInterface return hamiltonian_; } std::shared_ptr> getRho() { return rho_; } + std::shared_ptr getIons() { return ions_; } void run() override; @@ -189,8 +191,7 @@ class MGmol : public MGmolInterface * specified by tau (input) */ double evaluateEnergyAndForces(const std::vector& tau, - const std::vector& atnumbers, - std::vector& forces) override; + const std::vector& atnumbers, std::vector& forces); /* * Evaluate the energy and forces for an atomic configuration @@ -199,7 +200,7 @@ class MGmol : public MGmolInterface */ double evaluateEnergyAndForces(Orbitals* orbitals, const std::vector& tau, const std::vector& atnumbers, - std::vector& forces) override; + std::vector& forces); /* * Evaluate the energy and forces for an atomic configuration @@ -208,14 +209,14 @@ class MGmol : public MGmolInterface */ double evaluateDMandEnergyAndForces(Orbitals* orbitals, const std::vector& tau, const std::vector& atnumbers, - std::vector& forces) override; + std::vector& forces); /* * get internal atomic positions */ - void getAtomicPositions(std::vector& tau) override; + void getAtomicPositions(std::vector& tau); - void getAtomicNumbers(std::vector& an) override; + void getAtomicNumbers(std::vector& an); void setupPotentials(Ions& ions); void initKBR(); @@ -272,12 +273,8 @@ class MGmol : public MGmolInterface void addResidualSpreadPenalty(OrbitalsType& phi, OrbitalsType& res); int get_NOLMO(NOLMOTransform& noot, OrbitalsType& orbitals, OrbitalsType& work_orbitals, const double dd, const bool apply_flag); - void adaptLR(const SpreadsAndCenters* spreadf -#ifdef MGMOL_USE_SCALAPACK - , - const OrbitalsTransform* ot -#endif - ); + void adaptLR(const SpreadsAndCenters* spreadf, + const OrbitalsTransform* ot); int update_masks(); void move_orbitals(OrbitalsType** orbitals); int getMLWF2states(const int st1, const int st2, OrbitalsType& orbitals, @@ -300,7 +297,7 @@ class MGmol : public MGmolInterface void sebprintForces(); int nions() { return ions_->getNumIons(); } double getTotalEnergy(); - void cleanup() override; + void cleanup(); void geomOptimSetup(); void geomOptimQuench(); void geomOptimComputeForces(); @@ -331,18 +328,30 @@ class MGmol : public MGmolInterface { forces_->force(orbitals, ions); } + void setPositions(const std::vector& positions, + const std::vector& atnumbers) + { + ions_->setPositions(positions, atnumbers); + } /* * simply dump current state */ - void dumpRestart() override; + void dumpRestart(); void loadRestartFile(const std::string filename); - std::shared_ptr getProjectedMatrices() override + std::shared_ptr getProjectedMatrices() { return proj_matrices_; } + +#ifdef MGMOL_HAS_LIBROM + int save_orbital_snapshot(std::string file_path, OrbitalsType& orbitals); + void project_orbital( + std::string file_path, int rdim, OrbitalsType& orbitals); +#endif + void updateDMandEnergy(OrbitalsType& orbitals, Ions& ions, double& eks); }; // Instantiate static variables here to avoid clang warnings template diff --git a/src/MGmolInterface.h b/src/MGmolInterface.h index ce0ebebc..a0138846 100644 --- a/src/MGmolInterface.h +++ b/src/MGmolInterface.h @@ -41,6 +41,9 @@ class MGmolInterface virtual void getAtomicPositions(std::vector& tau) = 0; virtual void getAtomicNumbers(std::vector& an) = 0; + virtual void setPositions(const std::vector& positions, + const std::vector& atnumbers) + = 0; virtual std::shared_ptr getProjectedMatrices() = 0; virtual void dumpRestart() = 0; diff --git a/src/MGmol_prototypes.h b/src/MGmol_prototypes.h index cedd514a..e0f39301 100644 --- a/src/MGmol_prototypes.h +++ b/src/MGmol_prototypes.h @@ -11,8 +11,10 @@ #define MGMOL_PROTOTYPES_H #include "global.h" +#include "mgmol_config.h" #include +namespace po = boost::program_options; class Ions; class KBPsiMatrixSparse; @@ -24,4 +26,8 @@ int read_config(int argc, char** argv, boost::program_options::variables_map& vm, std::string& input_file, std::string& lrs_filename, std::string& constraints_filename, float& total_spin, bool& with_spin); +#ifdef MGMOL_HAS_LIBROM +void setupROMConfigOption(po::options_description& rom_cfg); +#endif + #endif diff --git a/src/MVPSolver.cc b/src/MVPSolver.cc index a26e08f1..04e4954b 100644 --- a/src/MVPSolver.cc +++ b/src/MVPSolver.cc @@ -10,6 +10,7 @@ #include "MVPSolver.h" #include "Control.h" +#include "DistMatrix.h" #include "Electrostatic.h" #include "Energy.h" #include "Ions.h" @@ -22,10 +23,6 @@ #include "Rho.h" #include "tools.h" -#ifdef MGMOL_USE_SCALAPACK -#include "DistMatrix.h" -#endif - #include template @@ -414,11 +411,10 @@ void MVPSolver::printTimers(std::ostream& os) target_tm_.print(os); } -#ifdef MGMOL_USE_SCALAPACK template class MVPSolver, dist_matrix::DistMatrix>; +template class MVPSolver, ReplicatedMatrix>; + template class MVPSolver, dist_matrix::DistMatrix>; -#endif -template class MVPSolver, ReplicatedMatrix>; template class MVPSolver, ReplicatedMatrix>; diff --git a/src/MVP_DMStrategy.cc b/src/MVP_DMStrategy.cc index 4dcbaf78..09657755 100644 --- a/src/MVP_DMStrategy.cc +++ b/src/MVP_DMStrategy.cc @@ -72,11 +72,10 @@ void MVP_DMStrategy::dressDM() if (use_old_dm_) proj_matrices_->dressupDM(); } -#ifdef MGMOL_USE_SCALAPACK template class MVP_DMStrategy, dist_matrix::DistMatrix>; +template class MVP_DMStrategy, ReplicatedMatrix>; + template class MVP_DMStrategy, dist_matrix::DistMatrix>; -#endif -template class MVP_DMStrategy, ReplicatedMatrix>; template class MVP_DMStrategy, ReplicatedMatrix>; diff --git a/src/OrbitalsExtrapolationOrder2.cc b/src/OrbitalsExtrapolationOrder2.cc index fdf37dec..4fdea147 100644 --- a/src/OrbitalsExtrapolationOrder2.cc +++ b/src/OrbitalsExtrapolationOrder2.cc @@ -9,21 +9,17 @@ #include "OrbitalsExtrapolationOrder2.h" #include "Control.h" +#include "DistMatrixTools.h" #include "ExtendedGridOrbitals.h" #include "LocGridOrbitals.h" #include "ProjectedMatrices.h" -#ifdef MGMOL_USE_SCALAPACK -#include "DistMatrixTools.h" -#endif - template void OrbitalsExtrapolationOrder2::extrapolate_orbitals( OrbitalsType** orbitals, OrbitalsType* new_orbitals) { Control& ct = *(Control::instance()); -#ifdef MGMOL_USE_SCALAPACK bool use_dense_proj_mat = false; if (ct.OuterSolver() != OuterSolverType::ABPG && ct.OuterSolver() != OuterSolverType::NLCG) @@ -35,7 +31,6 @@ void OrbitalsExtrapolationOrder2::extrapolate_orbitals( proj_matrices)) use_dense_proj_mat = true; } -#endif new_orbitals->assign(**orbitals); @@ -47,8 +42,7 @@ void OrbitalsExtrapolationOrder2::extrapolate_orbitals( if (ct.verbose > 1 && onpe0) (*MPIdata::sout) << "Extrapolate orbitals order 2..." << std::endl; - // align orbitals_minus1_ with new_orbitals -#ifdef MGMOL_USE_SCALAPACK + // align orbitals_minus1_ with new_orbitals if (use_dense_proj_mat) { dist_matrix::DistMatrix matQ("Q", ct.numst, ct.numst); @@ -58,13 +52,13 @@ void OrbitalsExtrapolationOrder2::extrapolate_orbitals( "yyt", ct.numst, ct.numst); getProcrustesTransform(matQ, yyt); - orbitals_minus1->multiply_by_matrix(matQ, 0., *orbitals_minus1); + orbitals_minus1->multiply_by_matrix(matQ); orbitals_minus1->axpy((ORBDTYPE)-1., *new_orbitals); - orbitals_minus1->multiply_by_matrix(yyt, 0., *orbitals_minus1); + orbitals_minus1->multiply_by_matrix(yyt); } - else // !use_dense_proj_mat -#endif - { + else + { // !use_dense_proj_mat + new_orbitals->scal(2.); } new_orbitals->axpy((ORBDTYPE)-1., *orbitals_minus1); diff --git a/src/OrbitalsExtrapolationOrder3.cc b/src/OrbitalsExtrapolationOrder3.cc index c66492b2..e0586bab 100644 --- a/src/OrbitalsExtrapolationOrder3.cc +++ b/src/OrbitalsExtrapolationOrder3.cc @@ -8,14 +8,11 @@ // Please also read this link https://github.com/llnl/mgmol/LICENSE #include "OrbitalsExtrapolationOrder3.h" +#include "DistMatrixTools.h" #include "ExtendedGridOrbitals.h" #include "LocGridOrbitals.h" #include "ProjectedMatrices.h" -#ifdef MGMOL_USE_SCALAPACK -#include "DistMatrixTools.h" -#endif - template void OrbitalsExtrapolationOrder3::extrapolate_orbitals( OrbitalsType** orbitals, OrbitalsType* new_orbitals) @@ -24,7 +21,6 @@ void OrbitalsExtrapolationOrder3::extrapolate_orbitals( new_orbitals->assign(**orbitals); -#ifdef MGMOL_USE_SCALAPACK bool use_dense_proj_mat = false; if (ct.OuterSolver() != OuterSolverType::ABPG && ct.OuterSolver() != OuterSolverType::NLCG) @@ -36,7 +32,6 @@ void OrbitalsExtrapolationOrder3::extrapolate_orbitals( proj_matrices)) use_dense_proj_mat = true; } -#endif // do the extrapolation if previous orbitals exist (not at first step) @@ -47,8 +42,7 @@ void OrbitalsExtrapolationOrder3::extrapolate_orbitals( if (ct.verbose > 1 && onpe0) (*MPIdata::sout) << "Extrapolate orbitals using 3rd order scheme..." << std::endl; - // align orbitals_minus1 with new_orbitals -#ifdef MGMOL_USE_SCALAPACK + // align orbitals_minus1 with new_orbitals if (use_dense_proj_mat) { dist_matrix::DistMatrix matQ("Q", ct.numst, ct.numst); @@ -58,30 +52,26 @@ void OrbitalsExtrapolationOrder3::extrapolate_orbitals( // alignement orbitals_minus1_->computeGram(*new_orbitals, matQ); getProcrustesTransform(matQ, yyt); - orbitals_minus1_->multiply_by_matrix(matQ, 0., *orbitals_minus1_); + orbitals_minus1_->multiply_by_matrix(matQ); // compute delta Phi tmp_orbitals_minus1.assign(*orbitals_minus1_); tmp_orbitals_minus1.axpy((ORBDTYPE)-1., *new_orbitals); - tmp_orbitals_minus1.multiply_by_matrix( - yyt, 0., tmp_orbitals_minus1); + tmp_orbitals_minus1.multiply_by_matrix(yyt); if (orbitals_minus2_ != nullptr) { // alignement orbitals_minus2_->computeGram(*new_orbitals, matQ); getProcrustesTransform(matQ, yyt); - orbitals_minus2_->multiply_by_matrix( - matQ, 0., *orbitals_minus2_); + orbitals_minus2_->multiply_by_matrix(matQ); // compute delta Phi orbitals_minus2_->axpy((ORBDTYPE)-1., *orbitals_minus1_); - orbitals_minus2_->multiply_by_matrix( - yyt, 0., *orbitals_minus2_); + orbitals_minus2_->multiply_by_matrix(yyt); } } else -#endif { tmp_orbitals_minus1.assign(*orbitals_minus1_); if (orbitals_minus2_ != nullptr) @@ -113,6 +103,10 @@ void OrbitalsExtrapolationOrder3::extrapolate_orbitals( orbitals_minus1_ = *orbitals; + if (use_dense_proj_mat) + { + } + *orbitals = new_orbitals; (*orbitals)->incrementIterativeIndex(); diff --git a/src/PBdiel.cc b/src/PBdiel.cc index 2fe98f8a..6d592690 100644 --- a/src/PBdiel.cc +++ b/src/PBdiel.cc @@ -6,12 +6,27 @@ // All rights reserved. // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -#include "PBdiel.h" -#include "MGmol_blas1.h" +// $Id$ #include #include +#include "MGmol_blas1.h" +#include "PBdiel.h" + +template class PBdiel>; +// template class PBdiel >; +template class PBdiel>; +// template class PBdiel >; +template class PBdiel>; +// template class PBdiel >; +template class PBdiel>; +// template class PBdiel >; +template class PBdiel>; +// template class PBdiel >; +template class PBdiel>; +// template class PBdiel >; + template void PBdiel::solve( const pb::GridFunc& rho, const pb::GridFunc& rhoc) @@ -68,13 +83,7 @@ void PBdiel::solve( template void PBdiel::set_rhod(pb::GridFunc* rhod) { + //(*MPIdata::sout)<<"set_rhod"<>; -template class PBdiel>; -template class PBdiel>; -template class PBdiel>; -template class PBdiel>; -template class PBdiel>; diff --git a/src/PBdiel.h b/src/PBdiel.h index c8462e26..eb469d37 100644 --- a/src/PBdiel.h +++ b/src/PBdiel.h @@ -6,8 +6,10 @@ // All rights reserved. // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -#ifndef MGMOL_included_PBdiel -#define MGMOL_included_PBdiel + +// $Id$ +#ifndef included_PBdiel +#define included_PBdiel #include "MPIdata.h" #include "Poisson.h" diff --git a/src/PBdiel_CG.cc b/src/PBdiel_CG.cc index 4785300e..c00d1ca7 100644 --- a/src/PBdiel_CG.cc +++ b/src/PBdiel_CG.cc @@ -6,12 +6,13 @@ // All rights reserved. // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -#include "PBdiel_CG.h" -#include "MGmol_blas1.h" #include #include +#include "MGmol_blas1.h" +#include "PBdiel_CG.h" + template void PBdiel_CG::solve( const pb::GridFunc& rho, const pb::GridFunc& rhoc) @@ -68,13 +69,20 @@ void PBdiel_CG::solve( template void PBdiel_CG::set_rhod(pb::GridFunc* rhod) { + //(*MPIdata::sout)<<"set_rhod"<>; +// template class PBdiel_CG >; template class PBdiel_CG>; +// template class PBdiel_CG >; template class PBdiel_CG>; +// template class PBdiel_CG >; template class PBdiel_CG>; +// template class PBdiel_CG >; template class PBdiel_CG>; +// template class PBdiel_CG >; template class PBdiel_CG>; +// template class PBdiel_CG >; diff --git a/src/PBdiel_CG.h b/src/PBdiel_CG.h index cc5cb7b3..13c14fe8 100644 --- a/src/PBdiel_CG.h +++ b/src/PBdiel_CG.h @@ -6,8 +6,9 @@ // All rights reserved. // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -#ifndef MGMOL_PBdiel_CG_H_ -#define MGMOL_PBdiel_CG_H_ + +#ifndef _PBdiel_CG_H_ +#define _PBdiel_CG_H_ #include "MPIdata.h" #include "Poisson.h" @@ -37,10 +38,9 @@ class PBdiel_CG : public Poisson T oper(Poisson::grid_, e0, rho0, drho0); Poisson::vepsilon_ = new pb::GridFunc(Poisson::grid_, bc[0], bc[1], bc[2]); - rhod_ = nullptr; - Control& ct = *(Control::instance()); - poisson_solver_ = new PCGSolver_Diel( - oper, ct.lap_type, bc[0], bc[1], bc[2]); + rhod_ = nullptr; + poisson_solver_ + = new PCGSolver_Diel(oper, bc[0], bc[1], bc[2]); }; // Destructor diff --git a/src/PCGSolver.cc b/src/PCGSolver.cc index b1905ddd..cb22ab24 100644 --- a/src/PCGSolver.cc +++ b/src/PCGSolver.cc @@ -6,11 +6,9 @@ // All rights reserved. // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -#include "PCGSolver.h" -#include "LapFactory.h" +#include "PCGSolver.h" -#include #include #include @@ -118,8 +116,6 @@ void PCGSolver::preconSolve( pb::GridFunc& gf_v, const pb::GridFunc& gf_f, const short level) { - assert(is_precond_setup_); - //(*MPIdata::sout)<<"Preconditioning::mg() at level "<::solve( pb::GridFunc res(finegrid, bc_[0], bc_[1], bc_[2]); oper_.rhs(rhs, res); - // save rhs norm to compute relative norms - const double rhs_norm = res.norm2(); - assert(!std::isnan(rhs_norm)); - // compute r = r - Ax res -= lhs; @@ -200,8 +192,9 @@ bool PCGSolver::solve( assert(!std::isnan(init_rnorm)); // cout<<"init_rnorm="<::solve( // check for convergence rnorm = res.norm2(); - if (rnorm < tol_ * rhs_norm) + if (rnorm <= tol_ * init_rnorm) { converged = true; break; diff --git a/src/PCGSolver.h b/src/PCGSolver.h index 7b38ba71..0782775a 100644 --- a/src/PCGSolver.h +++ b/src/PCGSolver.h @@ -10,7 +10,10 @@ #ifndef MGMOL_PCG_SOLVER_H #define MGMOL_PCG_SOLVER_H +#include "Control.h" #include "Lap.h" +#include "LapFactory.h" +#include "global.h" #include @@ -19,14 +22,13 @@ class PCGSolver { private: std::vector grid_; + short precond_lap_type_; short bc_[3]; bool fully_periodic_; // operator to solve for OperatorType oper_; - const short precond_lap_type_; - // preconditioner operator for each MG level std::vector*> precond_oper_; std::vector*> gf_work_; @@ -52,10 +54,9 @@ class PCGSolver void clear(); public: - PCGSolver(OperatorType& oper, const short precond_lap_type, const short px, - const short py, const short pz) + PCGSolver( + OperatorType& oper, const short px, const short py, const short pz) : oper_(oper), - precond_lap_type_(precond_lap_type), maxiters_(10), tol_(1.e-16), final_residual_(-1.), @@ -70,6 +71,9 @@ class PCGSolver bc_[1] = py; bc_[2] = pz; fully_periodic_ = ((bc_[0] == 1) && (bc_[1] == 1) && (bc_[2] == 1)); + + Control& ct = *(Control::instance()); + precond_lap_type_ = ct.lap_type; }; void setup(const short nu1, const short nu2, const short max_sweeps, diff --git a/src/PCGSolver_Diel.cc b/src/PCGSolver_Diel.cc index 2dc8cdcb..4e7c88a9 100644 --- a/src/PCGSolver_Diel.cc +++ b/src/PCGSolver_Diel.cc @@ -99,17 +99,19 @@ void PCGSolver_Diel::setupPrecon() } } -// MG V-cycle with no mask template +// MG V-cycle with no mask void PCGSolver_Diel::preconSolve(pb::GridFunc& gf_v, const pb::GridFunc& gf_f, const short level) { + //(*MPIdata::sout)<<"Preconditioning::mg() at level "< (nu1_ + nu2_) ? 4 : (nu1_ + nu2_); } + // pb::Lap* myoper=pc_oper_[level]; T* myoper = pc_oper_[level]; // SMOOTHING @@ -146,8 +148,8 @@ void PCGSolver_Diel::preconSolve(pb::GridFunc& gf_v, if (bc_[0] != 1 || bc_[2] != 1 || bc_[2] != 1) gf_v.trade_boundaries(); } -// Left Preconditioned CG template +// Left Preconditioned CG bool PCGSolver_Diel::solve( pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs) { @@ -167,33 +169,20 @@ bool PCGSolver_Diel::solve( pb::GridFunc res(finegrid, bc_[0], bc_[1], bc_[2]); // scale initial guess with epsilon oper_.inv_transform(gf_phi); - // compute initial residual oper_.apply(gf_phi, lhs); pb::GridFunc rhs(gf_rhs); - - // transform r.h.s. to account for dielectric model oper_.transform(rhs); - - // convert to Hartree units + // Hartree units rhs *= (4. * M_PI); - - // save rhs norm to compute relative norms - const double rhs_norm = res.norm2(); - assert(!std::isnan(rhs_norm)); - res.diff(rhs, lhs); double init_rnorm = res.norm2(); double rnorm = init_rnorm; - // Early return if already converged - if (init_rnorm < tol_ * rhs_norm) return true; - // preconditioned residual pb::GridFunc z(finegrid, bc_[0], bc_[1], bc_[2]); // preconditioning step - z.setZero(); - + z = 0.; preconSolve(z, res, 0); // conjugate vectors pb::GridFunc p(z); @@ -216,7 +205,7 @@ bool PCGSolver_Diel::solve( // check for convergence rnorm = res.norm2(); - if (rnorm <= tol_ * rhs_norm) + if (rnorm <= tol_ * init_rnorm) { converged = true; break; @@ -236,8 +225,8 @@ bool PCGSolver_Diel::solve( return converged; } -// Left Preconditioned CG template +// Left Preconditioned CG bool PCGSolver_Diel::solve(pb::GridFunc& gf_phi, pb::GridFunc& gf_rhs, pb::GridFunc& gf_rhod, pb::GridFunc& gf_vks) diff --git a/src/PCGSolver_Diel.h b/src/PCGSolver_Diel.h index 7ae4aa2c..f2001a4c 100644 --- a/src/PCGSolver_Diel.h +++ b/src/PCGSolver_Diel.h @@ -10,6 +10,7 @@ #ifndef MGMOL_PCG_SOLVER_DIEL_H_ #define MGMOL_PCG_SOLVER_DIEL_H_ +#include "Control.h" #include "PB.h" #include "PBh2.h" #include "PBh4.h" @@ -26,11 +27,11 @@ class PCGSolver_Diel private: std::vector grid_; + short lap_type_; short bc_[3]; // operators T oper_; std::vector pc_oper_; - const short lap_type_; std::vector*> gf_work_; std::vector*> gf_rcoarse_; std::vector*> gf_newv_; @@ -52,9 +53,8 @@ class PCGSolver_Diel const pb::GridFunc& gf_f, const short level = 0); public: - PCGSolver_Diel(T& oper, const short lap_type, const short px, - const short py, const short pz) - : oper_(oper), lap_type_(lap_type) + PCGSolver_Diel(T& oper, const short px, const short py, const short pz) + : oper_(oper) { maxiters_ = 10; // default nu1_ = 2; // default @@ -68,6 +68,9 @@ class PCGSolver_Diel bc_[0] = px; bc_[1] = py; bc_[2] = pz; + // fully_periodic_=( (bc_[0]==1) && (bc_[1]==1) && (bc_[2]==1) ); + Control& ct = *(Control::instance()); + lap_type_ = ct.lap_type; }; void setup(const short nu1, const short nu2, const short max_sweeps, diff --git a/src/Poisson.h b/src/Poisson.h index 0e994eb9..7ceabc80 100644 --- a/src/Poisson.h +++ b/src/Poisson.h @@ -88,6 +88,14 @@ class Poisson : public PoissonInterface Int_vhrho_ = vel * vh_->gdot(rho); Int_vhrhoc_ = vel * vh_->gdot(rhoc); } + + virtual void applyOperator( + pb::GridFunc& vh, pb::GridFunc& lhs) + { + std::cerr << "ERROR: Abstract method Poisson::applyOperator()" + << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } }; #endif diff --git a/src/PolakRibiereSolver.cc b/src/PolakRibiereSolver.cc index 9b073cdf..b76d3bf0 100644 --- a/src/PolakRibiereSolver.cc +++ b/src/PolakRibiereSolver.cc @@ -336,6 +336,7 @@ int PolakRibiereSolver::solve(OrbitalsType& orbitals, if (dm_success == 0) // if DM determined at previous step is good { + // turn on PB solver if necessary dielON(); @@ -353,11 +354,8 @@ int PolakRibiereSolver::solve(OrbitalsType& orbitals, // (to be used for energy and gradient computation) proj_matrices_->updateThetaAndHB(); - if (step == max_steps) - { - solve_tm_.stop(); - break; - } + if (step == max_steps) break; + // Output the eigenvalues and occupations bool flag = false; if (iprint) flag = (!(step % iprint) && step < max_steps - 1); @@ -441,7 +439,6 @@ int PolakRibiereSolver::solve(OrbitalsType& orbitals, << " PolakRibiereSolver: convergence achieved for " "delta E..." << std::endl; - solve_tm_.stop(); break; } @@ -452,7 +449,8 @@ int PolakRibiereSolver::solve(OrbitalsType& orbitals, { double beta = computeBeta(work_orbitals); if (beta < 0.) beta = 0.; - if (onpe0 && ct.verbose > 0) + // const double beta=0.; + if (onpe0 && ct.verbose > 1) os_ << " PolakRibiereSolver: beta=" << beta << std::endl; p_k_->scal(beta); p_k_->axpy((ORBDTYPE)1., *z_k_); @@ -501,7 +499,6 @@ int PolakRibiereSolver::solve(OrbitalsType& orbitals, orbitals.computeGramAndInvS(); } -#ifdef MGMOL_USE_SCALAPACK // rotate pairs if smallest eigenvalue of overlap matrix below threshold if (ct.getThresholdEigenvalueGramQuench() > 0. && wolfe) { @@ -520,7 +517,6 @@ int PolakRibiereSolver::solve(OrbitalsType& orbitals, << std::endl; } } -#endif // rebuild dm with new overlap matrix dm_strategy_->dressDM(); diff --git a/src/Potentials.cc b/src/Potentials.cc index a4f0b958..67769bd3 100644 --- a/src/Potentials.cc +++ b/src/Potentials.cc @@ -943,6 +943,98 @@ void Potentials::initBackground() if (fabs(background_charge_) < 1.e-10) background_charge_ = 0.; } +void Potentials::evalIonDensityOnSamplePts(Ions& ions, + const std::vector& local_idx, std::vector& sampled_rhoc) +{ + Mesh* mymesh = Mesh::instance(); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const pb::Grid& mygrid = mymesh->grid(); + + const char flag_filter = pot_type(0); + if (flag_filter == 's') + { + if (onpe0) + { + std::cout << "Potentials::evalIonDensityOnSamplePts - flag_filter " + "s is not supported" + << std::endl; + } + mmpi.abort(); + } + + // initialize output vector + sampled_rhoc.resize(local_idx.size()); + for (int d = 0; d < sampled_rhoc.size(); d++) + sampled_rhoc[d] = 0.0; + + // Loop over ions + for (auto& ion : ions.overlappingVL_ions()) + { + const Species& sp(ion->getSpecies()); + + Vector3D position(ion->position(0), ion->position(1), ion->position(2)); + + initializeRadialDataOnSampledPts(position, sp, local_idx, sampled_rhoc); + } + + return; +} + +void Potentials::initializeRadialDataOnSampledPts(const Vector3D& position, + const Species& sp, const std::vector& local_idx, + std::vector& sampled_rhoc) +{ + assert(local_idx.size() == sampled_rhoc.size()); + + Control& ct = *(Control::instance()); + + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + + const int dim0 = mygrid.dim(0); + const int dim1 = mygrid.dim(1); + const int dim2 = mygrid.dim(2); + + const double start0 = mygrid.start(0); + const double start1 = mygrid.start(1); + const double start2 = mygrid.start(2); + + const double h0 = mygrid.hgrid(0); + const double h1 = mygrid.hgrid(1); + const double h2 = mygrid.hgrid(2); + + Vector3D point(0., 0., 0.); + + const double lrad = sp.lradius(); + + const RadialInter& lpot(sp.local_pot()); + const Vector3D lattice(mygrid.ll(0), mygrid.ll(1), mygrid.ll(2)); + + for (int k = 0; k < local_idx.size(); k++) + { + /* + local_idx provides offset. + offset = iz + iy * dim2 + ix * dim1 * dim2; + evaluate x,y,z indices backward. + */ + int iz = local_idx[k] % dim2; + int ix = local_idx[k] / dim2; + int iy = ix % dim1; + ix /= dim1; + + /* compute grid point position */ + point[0] = start0 + ix * h0; + point[1] = start1 + iy * h1; + point[2] = start2 + iz * h2; + + /* accumulate ion species density */ + const double r = position.minimage(point, lattice, ct.bcPoisson); + if (r < lrad) sampled_rhoc[k] += sp.getRhoComp(r); + } + + return; +} + int Potentials::read(HDFrestart& h5f_file) { Control& ct = *(Control::instance()); diff --git a/src/Potentials.h b/src/Potentials.h index 538fe314..bfef2628 100644 --- a/src/Potentials.h +++ b/src/Potentials.h @@ -110,6 +110,10 @@ class Potentials void initializeSupersampledRadialDataOnMesh( const Vector3D& position, const Species& sp); + void initializeRadialDataOnSampledPts(const Vector3D& position, + const Species& sp, const std::vector& local_idx, + std::vector& sampled_rhoc); + void rescaleRhoComp(); void addBackgroundToRhoComp(); @@ -178,6 +182,8 @@ class Potentials double getChargeInCell() const { return charge_in_cell_; } + const double getBackgroundCharge() const { return background_charge_; } + void getVofRho(std::vector& vrho) const; double getEfield(short int dir) { return efield_[dir]; } @@ -220,6 +226,9 @@ class Potentials void resetVhRho2Backup() { vh_rho_ = vh_rho_backup_; } + void evalIonDensityOnSamplePts(Ions& ions, + const std::vector& local_idx, std::vector& sampled_rhoc); + #ifdef HAVE_TRICUBIC void readExternalPot(const string filename, const char type); void setupVextTricubic(); diff --git a/src/Power.cc b/src/Power.cc index b2855b29..58e268d0 100644 --- a/src/Power.cc +++ b/src/Power.cc @@ -15,11 +15,6 @@ #include "mputils.h" #include "random.h" -#ifdef MGMOL_USE_SCALAPACK -#include "DistMatrix.h" -#include "DistVector.h" -#endif - // compute sum of squares of elements of vector y-theta*v template double diff2(VECTOR& y, VECTOR& v, const double theta, const bool verbose) @@ -102,8 +97,6 @@ void Power::computeEigenInterval(const MATRIX& A, double& emin, template class Power, SquareLocalMatrices>; -#ifdef MGMOL_USE_SCALAPACK template class Power, dist_matrix::DistMatrix>; -#endif // template class Power; diff --git a/src/Power.h b/src/Power.h index b296c7b5..7a929941 100644 --- a/src/Power.h +++ b/src/Power.h @@ -7,6 +7,7 @@ #ifndef MGMOL_POWER_H #define MGMOL_POWER_H +#include "DistVector.h" #include "Timer.h" #include "random.h" #include diff --git a/src/PowerGen.cc b/src/PowerGen.cc index f2eaf6c5..3bd9dc74 100644 --- a/src/PowerGen.cc +++ b/src/PowerGen.cc @@ -8,17 +8,14 @@ #include "PowerGen.h" #include "Control.h" +#include "DistMatrix.h" +#include "DistVector.h" #include "GramMatrix.h" #include "ReplicatedMatrix.h" #include "ReplicatedVector.h" #include "mputils.h" #include "random.h" -#ifdef MGMOL_USE_SCALAPACK -#include "DistMatrix.h" -#include "DistVector.h" -#endif - /* Use the power method to compute the extents of the spectrum of the * generalized eigenproblem. In order to use a residual-based convergence * criterion in an efficient way, we delay normalization of the vectors to avoid @@ -188,8 +185,6 @@ void PowerGen::computeGenEigenInterval(MatrixType& mat, compute_tm_.stop(); } -#ifdef MGMOL_USE_SCALAPACK template class PowerGen, dist_matrix::DistVector>; -#endif template class PowerGen; diff --git a/src/ProjectedMatrices.cc b/src/ProjectedMatrices.cc index 50a6cd21..58ef38d1 100644 --- a/src/ProjectedMatrices.cc +++ b/src/ProjectedMatrices.cc @@ -11,7 +11,10 @@ #include "Control.h" #include "DensityMatrix.h" +#include "DistMatrix2SquareLocalMatrices.h" +#include "DistMatrixTools.h" #include "HDFrestart.h" +#include "LocalMatrices2DistMatrix.h" #include "LocalMatrices2ReplicatedMatrix.h" #include "MGmol_MPI.h" #include "Power.h" @@ -21,19 +24,10 @@ #include "ReplicatedVector.h" #include "ReplicatedWorkSpace.h" #include "SP2.h" -#include "fermi.h" -#include "hdf_tools.h" - -#ifdef MGMOL_USE_SCALAPACK -#include "DistMatrix2SquareLocalMatrices.h" -#include "DistMatrixTools.h" -#include "DistVector.h" -#include "LocalMatrices2DistMatrix.h" #include "SparseDistMatrix.h" #include "SquareSubMatrix2DistMatrix.h" -#else -typedef double DISTMATDTYPE; -#endif +#include "fermi.h" +#include "hdf_tools.h" #include #include @@ -47,9 +41,7 @@ template GramMatrix* ProjectedMatrices::gram_4dotProducts_ = nullptr; -#ifdef MGMOL_USE_SCALAPACK static int sparse_distmatrix_nb_partitions = 128; -#endif template <> std::string ProjectedMatrices::getMatrixType() @@ -57,15 +49,12 @@ std::string ProjectedMatrices::getMatrixType() return "ReplicatedMatrix"; } -#ifdef MGMOL_USE_SCALAPACK template <> std::string ProjectedMatrices>::getMatrixType() { return "DistMatrix"; } -#endif -#ifdef MGMOL_USE_SCALAPACK // // conversion functions from one matrix format into another // @@ -89,7 +78,6 @@ void convert_matrix(const dist_matrix::DistMatrix& src, dst.assign(tmp); } #endif -#endif #ifndef HAVE_MAGMA void convert_matrix(const ReplicatedMatrix& src, @@ -157,7 +145,6 @@ ProjectedMatrices::~ProjectedMatrices() n_instances_--; } -#ifdef MGMOL_USE_SCALAPACK template <> void ProjectedMatrices>::convert( const SquareLocalMatrices& src, @@ -166,7 +153,6 @@ void ProjectedMatrices>::convert( LocalMatrices2DistMatrix* sl2dm = LocalMatrices2DistMatrix::instance(); sl2dm->accumulate(src, dst); } -#endif template <> void ProjectedMatrices::convert( @@ -179,7 +165,6 @@ void ProjectedMatrices::convert( sl2rm->accumulate(src, dst); } -#ifdef MGMOL_USE_SCALAPACK template <> void ProjectedMatrices>:: setupGlobalIndexes(const std::vector>& global_indexes) @@ -191,7 +176,6 @@ void ProjectedMatrices>:: comm, global_indexes, gm_->getMatrix()); LocalMatrices2DistMatrix::setup(comm, global_indexes); } -#endif template <> void ProjectedMatrices::setupGlobalIndexes( @@ -1203,7 +1187,6 @@ ProjectedMatrices::computeChemicalPotentialAndDMwithChebyshev( /* Use the power method to compute the extents of the spectrum of the * generalized eigenproblem. */ -#ifdef MGMOL_USE_SCALAPACK template <> void ProjectedMatrices>:: computeGenEigenInterval( @@ -1217,7 +1200,6 @@ void ProjectedMatrices>:: power.computeGenEigenInterval(mat, *gm_, interval, maxits, pad); } -#endif template <> void ProjectedMatrices::computeGenEigenInterval( @@ -1230,7 +1212,6 @@ void ProjectedMatrices::computeGenEigenInterval( power.computeGenEigenInterval(mat, *gm_, interval, maxits, pad); } -#ifdef MGMOL_USE_SCALAPACK template <> void ProjectedMatrices>::consolidateH() { @@ -1252,7 +1233,6 @@ void ProjectedMatrices>::consolidateH() consolidate_H_tm_.stop(); } -#endif template <> void ProjectedMatrices::consolidateH() @@ -1275,7 +1255,6 @@ void ProjectedMatrices::updateSubMatX(const MatrixType& dm) convert_matrix(dm, *localX_); } -#ifdef MGMOL_USE_SCALAPACK template <> SquareLocalMatrices ProjectedMatrices>::getReplicatedDM() @@ -1286,7 +1265,6 @@ ProjectedMatrices>::getReplicatedDM() return sldm; } -#endif template <> SquareLocalMatrices @@ -1299,7 +1277,5 @@ ProjectedMatrices::getReplicatedDM() return sldm; } -#ifdef MGMOL_USE_SCALAPACK template class ProjectedMatrices>; -#endif template class ProjectedMatrices; diff --git a/src/ProjectedMatrices.h b/src/ProjectedMatrices.h index 9827c7ef..2b5699a1 100644 --- a/src/ProjectedMatrices.h +++ b/src/ProjectedMatrices.h @@ -321,9 +321,9 @@ class ProjectedMatrices : public ProjectedMatricesInterface double computeEntropyWithCheb(const double kbt); double checkCond(const double tol, const bool flag = true) override; int writeDM(HDFrestart& h5f_file) override; - int writeSavedDM(HDFrestart& h5f_file) override; + int writeSavedDM(HDFrestart& h5f_file); int readDM(HDFrestart& h5f_file) override; - int readWFDM(HDFrestart& h5f_file) override; + int readWFDM(HDFrestart& h5f_file); void printEigenvalues(std::ostream& os) const; void updateDM() override; void updateDMwithEigenstates(); @@ -429,12 +429,12 @@ class ProjectedMatrices : public ProjectedMatricesInterface template Timer ProjectedMatrices::init_gram_matrix_tm_( - "ProjectedMatrices::initialize_Gram_Matrix"); + "ProjectedMatrices::initialize_Gram_Matrix"); template Timer ProjectedMatrices::sygv_tm_("ProjectedMatrices::sygv"); template Timer ProjectedMatrices::compute_inverse_tm_( - "ProjectedMatrices::computeInverse"); + "ProjectedMatrices::computeInverse"); template Timer ProjectedMatrices::compute_invB_tm_( "ProjectedMatrices::computeInvB"); @@ -454,5 +454,5 @@ Timer ProjectedMatrices::consolidate_H_tm_( "ProjectedMatrices::consolidate_sH"); template Timer ProjectedMatrices::compute_entropy_tm_( - "ProjectedMatrices::compute_entropy"); + "ProjectedMatrices::compute_entropy"); #endif diff --git a/src/ProjectedMatrices2N.cc b/src/ProjectedMatrices2N.cc index 60f2cc80..d13196b2 100644 --- a/src/ProjectedMatrices2N.cc +++ b/src/ProjectedMatrices2N.cc @@ -8,11 +8,8 @@ // Please also read this link https://github.com/llnl/mgmol/LICENSE #include "ProjectedMatrices2N.h" -#include "ReplicatedMatrix.h" - -#ifdef MGMOL_USE_SCALAPACK #include "DistMatrix.h" -#endif +#include "ReplicatedMatrix.h" template ProjectedMatrices2N::ProjectedMatrices2N( @@ -70,7 +67,5 @@ void ProjectedMatrices2N::iterativeUpdateDMwithEigenstates( ProjectedMatrices::buildDM(*work2N_); } -#ifdef MGMOL_USE_SCALAPACK template class ProjectedMatrices2N>; -#endif template class ProjectedMatrices2N; diff --git a/src/ProjectedMatricesMehrstellen.cc b/src/ProjectedMatricesMehrstellen.cc index 2f890487..b12aee94 100644 --- a/src/ProjectedMatricesMehrstellen.cc +++ b/src/ProjectedMatricesMehrstellen.cc @@ -8,12 +8,9 @@ // Please also read this link https://github.com/llnl/mgmol/LICENSE #include "ProjectedMatricesMehrstellen.h" -#include "ReplicatedMatrix.h" - -#ifdef MGMOL_USE_SCALAPACK #include "DistMatrix.h" #include "DistMatrixTools.h" -#endif +#include "ReplicatedMatrix.h" template ProjectedMatricesMehrstellen::ProjectedMatricesMehrstellen( @@ -100,8 +97,6 @@ void ProjectedMatricesMehrstellen::rotateAll( ProjectedMatrices::dm_->rotate(rotation_matrix, flag_eigen); } -#ifdef MGMOL_USE_SCALAPACK template class ProjectedMatricesMehrstellen< dist_matrix::DistMatrix>; -#endif template class ProjectedMatricesMehrstellen; diff --git a/src/ProjectedMatricesMehrstellen.h b/src/ProjectedMatricesMehrstellen.h index c5b1ec86..5adb32c7 100644 --- a/src/ProjectedMatricesMehrstellen.h +++ b/src/ProjectedMatricesMehrstellen.h @@ -48,7 +48,7 @@ class ProjectedMatricesMehrstellen : public ProjectedMatrices const MatrixType& rotation_matrix, const bool flag_eigen) override; void initializeMatB( - const SquareLocalMatrices& ss) override + const SquareLocalMatrices& ss) { ProjectedMatrices::convert(ss, *matB_); } diff --git a/src/ReplicatedMatrix.cc b/src/ReplicatedMatrix.cc index 04c56e0d..885372d1 100644 --- a/src/ReplicatedMatrix.cc +++ b/src/ReplicatedMatrix.cc @@ -600,8 +600,7 @@ void ReplicatedMatrix::syev( &lwork, &info); #endif if (info != 0) - std::cerr << "ReplicatedMatrix::syev() failed, info = " << info - << std::endl; + std::cerr << "magma_dsyevd_gpu failed, info = " << info << std::endl; // for(auto& d : evals)std::cout<::setUpperTriangularSquareMatrixToZero() } } -#ifdef MGMOL_USE_SCALAPACK template void ReplicatedWorkSpace::initSquareMatrix( const dist_matrix::DistMatrix& distmat) { distmat.allgather(square_matrix_, ndim_); } -#endif template void ReplicatedWorkSpace::initSquareMatrix( diff --git a/src/ReplicatedWorkSpace.h b/src/ReplicatedWorkSpace.h index d5eaddcc..c4d66f59 100644 --- a/src/ReplicatedWorkSpace.h +++ b/src/ReplicatedWorkSpace.h @@ -10,13 +10,10 @@ #ifndef MGMOL_REPLICATED_WORKSPACE_H #define MGMOL_REPLICATED_WORKSPACE_H +#include "DistMatrix.h" #include "ReplicatedMatrix.h" #include "Timer.h" -#ifdef MGMOL_USE_SCALAPACK -#include "DistMatrix.h" -#endif - #include class ReplicatedMatrix; @@ -76,9 +73,7 @@ class ReplicatedWorkSpace void setUpperTriangularSquareMatrixToZero(); -#ifdef MGMOL_USE_SCALAPACK void initSquareMatrix(const dist_matrix::DistMatrix& tmat); -#endif void initSquareMatrix(const ReplicatedMatrix& mat); int getDim() { return ndim_; } diff --git a/src/Rho.cc b/src/Rho.cc index 680f9e0f..7b104573 100644 --- a/src/Rho.cc +++ b/src/Rho.cc @@ -16,15 +16,12 @@ #include "ProjectedMatrices.h" #include "ReplicatedMatrix.h" #include "SquareLocalMatrices.h" +#include "SubMatrices.h" #include "magma_singleton.h" #include "memory_space.h" #include "mputils.h" #include "numerical_kernels.h" -#ifdef MGMOL_USE_SCALAPACK -#include "SubMatrices.h" -#endif - template Timer Rho::update_tm_("Rho::update"); template @@ -579,8 +576,6 @@ template double Rho>::dotWithRho( const double* const func) const; template double Rho>::dotWithRho( const double* const func) const; - -#ifdef MGMOL_USE_SCALAPACK template void Rho>::computeRho< dist_matrix::DistMatrix>(ExtendedGridOrbitals&, ExtendedGridOrbitals&, const dist_matrix::DistMatrix&, @@ -593,7 +588,6 @@ template void Rho>::computeRho< template void Rho>::computeRho>( LocGridOrbitals&, const dist_matrix::DistMatrix&); -#endif #ifdef MGMOL_USE_MIXEDP template double Rho>::dotWithRho( const float* const func) const; diff --git a/src/Rho.h b/src/Rho.h index ae96dd68..0c7642c1 100644 --- a/src/Rho.h +++ b/src/Rho.h @@ -71,6 +71,8 @@ class Rho Rho(); ~Rho() {}; + const OrthoType getOrthoType() { return orbitals_type_; } + void setup( const OrthoType orbitals_type, const std::vector>&); void setVerbosityLevel(const int vlevel) { verbosity_level_ = vlevel; } diff --git a/src/SP2.cc b/src/SP2.cc index 7e83eb0c..6ff7625a 100644 --- a/src/SP2.cc +++ b/src/SP2.cc @@ -6,15 +6,12 @@ // Please also read this link https://github.com/llnl/mgmol/LICENSE #include "SP2.h" +#include "DistMatrix.h" #include "MGmol_MPI.h" #include "MPIdata.h" #include "ReplicatedMatrix.h" #include "linear_algebra/blas3_c.h" -#ifdef MGMOL_USE_SCALAPACK -#include "DistMatrix.h" -#endif - Timer SP2::getdm_tm_("SP2::getDM"); #ifdef HAVE_BML @@ -217,7 +214,6 @@ void SP2::initializeLocalMat( reduceSumTrace(); } -#ifdef MGMOL_USE_SCALAPACK template <> void SP2::getDM(dist_matrix::DistMatrix& submatM, // output const dist_matrix::DistMatrix& invS) @@ -234,7 +230,6 @@ void SP2::getDM(dist_matrix::DistMatrix& submatM, // output getdm_tm_.stop(); } -#endif template <> void SP2::getDM(ReplicatedMatrix& submatM, // output diff --git a/src/jade.cc b/src/jade.cc index 757e8125..6da9cac7 100644 --- a/src/jade.cc +++ b/src/jade.cc @@ -6,9 +6,7 @@ // All rights reserved. // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -#ifdef MGMOL_USE_SCALAPACK #include "DistMatrix.h" -#endif #include "MGmol_MPI.h" #include "MGmol_blas1.h" #include "fc_mangle.h" @@ -26,7 +24,6 @@ // The input matrices are given in rmat[k], k = 0,..,m-1 // the element a(i,j) of matrix a_k is in rmat[k][i+m_loc*j]; // the orthogonal transformation is returned in tmat[i+m_loc*j] -#ifdef MGMOL_USE_SCALAPACK double jade(std::vector*>& rmat, const int n_loc, const int m_loc, const int offset_, const int maxsweep, const double tol, MPI_Comm comm, std::vector& tmat, @@ -37,7 +34,7 @@ double jade(std::vector*>& rmat, const int m = 2 * NDIM; const int npecol = rmat[0]->npcol(); -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK MGmol_MPI& mmpi = *(MGmol_MPI::instance()); int mype = mmpi.mypeSpin(); @@ -282,7 +279,7 @@ double jade(std::vector*>& rmat, std::vector remote_actualv(n_loc); for (int pe = 0; pe < npecol; pe++) { -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK if (mype == pe) { v_small = u_small; @@ -323,7 +320,7 @@ double jade(std::vector*>& rmat, } // pe -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK for (int i = 0; i < 8; i++) req[i] = MPI_REQUEST_NULL; @@ -457,7 +454,7 @@ double jade(std::vector*>& rmat, } } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK double in = sum; MPI_Allreduce(&in, &sum, 1, MPI_DOUBLE, MPI_SUM, comm); in = sum_off; @@ -531,7 +528,7 @@ double jade(std::vector*>& rmat, for (int ip = 0; ip < n_loc; ip++) { int lip = actual[ip]; -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK mmpi.bcast(&lip, 1, root); #endif if (lip < m_loc) @@ -541,7 +538,7 @@ double jade(std::vector*>& rmat, memcpy(&tmat[m_loc * lip], &u_loc[m_loc * ip], m_loc * sizeof(double)); } -#ifdef MGMOL_USE_SCALAPACK +#ifdef SCALAPACK mmpi.bcast(&tmat[m_loc * lip], m_loc, root); #endif } @@ -558,5 +555,3 @@ double jade(std::vector*>& rmat, return delta; } - -#endif diff --git a/src/linear_algebra/gemm_impl.cc b/src/linear_algebra/gemm_impl.cc index e1418620..3500410c 100644 --- a/src/linear_algebra/gemm_impl.cc +++ b/src/linear_algebra/gemm_impl.cc @@ -244,3 +244,8 @@ template void gemm_impl(const char transa, const double alpha, const float* const a, const int lda, const float* const b, const int ldb, const double beta, double* const c, const int ldc); +template void gemm_impl(const char transa, + const char transb, const int m, const int n, const int k, + const double alpha, const float* const a, const int lda, + const float* const b, const int ldb, const double beta, float* const c, + const int ldc); diff --git a/src/linear_algebra/mputils.cc b/src/linear_algebra/mputils.cc index b965fe0f..794de6ca 100644 --- a/src/linear_algebra/mputils.cc +++ b/src/linear_algebra/mputils.cc @@ -872,6 +872,8 @@ template void LAU_H::MPgemmNN(const int m, const int n, const int k, const double alpha, const double* const a, const int lda, const double* const b, const int ldb, const double beta, double* const c, const int ldc); +template double LAU_H::MPdot( + const int len, const double* const xptr, const double* const yptr); template double LAU_H::MPdot(const int len, const float* __restrict__ xptr, const float* __restrict__ yptr); template double LAU_H::MPdot( diff --git a/src/local_matrices/LocalVector.h b/src/local_matrices/LocalVector.h index c8bfc84e..16ca3463 100644 --- a/src/local_matrices/LocalVector.h +++ b/src/local_matrices/LocalVector.h @@ -2,7 +2,6 @@ #define MGMOL_LOCALVECTOR #include "memory_space.h" -#include "mputils.h" #include diff --git a/src/md.cc b/src/md.cc index 69a56e4a..66d94317 100644 --- a/src/md.cc +++ b/src/md.cc @@ -34,6 +34,12 @@ #include "mgmol_Signal.h" #include "tools.h" +#ifdef MGMOL_HAS_LIBROM +#include "KBPsiMatrixSparse.h" +#include "PinnedH2O.h" +#include "rom_workflows.h" +#endif + #include #include #include @@ -43,6 +49,7 @@ Timer md_tau_tm("md_tau"); Timer md_moveVnuc_tm("md_moveVnuc"); Timer md_updateMasks_tm("md_updateMasks"); Timer md_extrapolateOrbitals_tm("md_extrapolateOrbitals"); +Timer md_updateDMandEnergy_tm("md_updateDMandEnergy"); #define DUMP_MAX_NUM_TRY 5 @@ -369,8 +376,29 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) h5f_file_.reset(); } + bool ROM_MVP + = (ct.getROMOptions().rom_stage == ROMStage::ONLINE_PINNED_H2O_3DOF); +#ifdef MGMOL_HAS_LIBROM + // ROM - initialize orbitals and density matrix + if (ROM_MVP) + { + if (onpe0) os_ << "Setup ROM MVP solver..." << std::endl; + ExtendedGridOrbitals** extended_orbitals + = reinterpret_cast**>(orbitals); + (*extended_orbitals)->set(ct.getROMOptions().basis_file, ct.numst); + (*extended_orbitals)->orthonormalizeLoewdin(); + (*extended_orbitals)->setDataWithGhosts(true); + (*extended_orbitals)->setIterativeIndex(10); + + std::shared_ptr projmatrices + = getProjectedMatrices(); + projmatrices->setDMuniform(ct.getNelSpin()); + projmatrices->printDM(os_); + } +#endif + // additional SC steps to compensate random start - if (ct.restart_info < 3) + if (ct.restart_info < 3 && !ROM_MVP) { double eks = 0.; quench(**orbitals, ions, ct.max_electronic_steps, 20, eks); @@ -381,6 +409,9 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) bool extrapolated_flag = true; if (ct.dt <= 0.) extrapolated_flag = false; + int librom_snapshot_freq = ct.getROMOptions().librom_snapshot_freq; + if (librom_snapshot_freq == -1) librom_snapshot_freq = ct.md_print_freq; + MDfiles md_files; // main MD iteration loop @@ -399,20 +430,53 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) int retval = 0; bool small_move = true; bool last_move_is_small = true; - do + + bool force_on_ions = true; +#ifdef MGMOL_HAS_LIBROM + // Ions for ROM + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + const double lattice[3] = { mygrid.ll(0), mygrid.ll(1), mygrid.ll(2) }; + std::vector positions; + std::vector anumbers; + getAtomicPositions(positions); + getAtomicNumbers(anumbers); + Ions* ROM_ions; + + // Pinned H2O 3 DOF + PinnedH2O H2O_molecule; + if (ct.getROMOptions().rom_stage == ROMStage::ONLINE_PINNED_H2O_3DOF) + { + if (onpe0) + os_ << "Rotate Pinned H2O molecule in timestep " << mdstep + << std::endl; + H2O_molecule.rotate(positions, anumbers); + if (onpe0) H2O_molecule.print(os_); + ROM_ions + = new Ions(positions, anumbers, lattice, ions_->getSpecies()); + setupPotentials(*ROM_ions); + force_on_ions = false; + } +#endif + + if (ROM_MVP) + { + md_updateDMandEnergy_tm.start(); + updateDMandEnergy(**orbitals, *ROM_ions, eks); + md_updateDMandEnergy_tm.stop(); + } + else { retval = quench(**orbitals, ions, ct.max_electronic_steps, 0, eks); + } + do + { // update localization regions if (ct.adaptiveLRs()) { assert(lrs_); - adaptLR(spreadf_.get() -#ifdef MGMOL_USE_SCALAPACK - , - nullptr -#endif - ); + adaptLR(spreadf_.get(), nullptr); last_move_is_small = lrs_->moveIsSmall(); @@ -467,7 +531,63 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) << std::endl; // Compute forces - force(**orbitals, ions); + if (force_on_ions) force(**orbitals, ions); + +#ifdef MGMOL_HAS_LIBROM + if (ct.getROMOptions().rom_stage == ROMStage::ONLINE_PINNED_H2O_3DOF) + { + force(**orbitals, *ROM_ions); + // Pinned H2O 3 DOF + if (onpe0) + os_ << "Transpose rotate the PinnedH2O molecule" << std::endl; + std::vector forces; + ROM_ions->getForces(forces); + H2O_molecule.transpose_rotate(positions, anumbers, forces); + ions.setLocalForces(forces, positions); + delete ROM_ions; + } + + if (ct.getROMOptions().rom_stage == ROMStage::TEST_ORBITAL) + { + if (onpe0) + { + os_ << "Projecting orbitals onto ROM subspaces to compare " + << ((ct.getROMOptions().compare_md) ? "MD dynamics" + : "force") + << std::endl; + os_ << "Loading ROM basis " << ct.getROMOptions().basis_file + << std::endl; + os_ << "ROM basis dimension = " + << ct.getROMOptions().num_orbbasis << std::endl; + } + + // Project orbitals to ROM subspace + project_orbital(ct.getROMOptions().basis_file, + ct.getROMOptions().num_orbbasis, **orbitals); + if (ct.getROMOptions().compare_md) + { + // overwrite ions force for MD time stepping + force(**orbitals, ions); + } + else + { + // write ROM_ions force for one-step comparison + ROM_ions = new Ions( + positions, anumbers, lattice, ions_->getSpecies()); + force(**orbitals, *ROM_ions); + std::string zero = "0"; + if (ions_->getNumIons() < 256 || ct.verbose > 2) + { + if (ct.verbose > 0) ROM_ions->printForcesGlobal(os_); + } + else if (zero.compare(ct.md_print_filename) == 0) + { + ROM_ions->printForcesLocal(os_); + } + delete ROM_ions; + } + } +#endif // set fion ions.getLocalForces(fion); @@ -615,6 +735,21 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) printWithTimeStamp("dumped restart file...", std::cout); } +#ifdef MGMOL_HAS_LIBROM + // Save orbital snapshots + if (ct.getROMOptions().save_librom_snapshot > 0 + && md_iteration_ % librom_snapshot_freq == 0) + { + int ierr = save_orbital_snapshot( + ct.md_print_filename + "_mdstep" + std::to_string(mdstep), + **orbitals); + + if (ierr < 0) + os_ << "WARNING md(): writing ROM snapshot data failed!!!" + << std::endl; + } +#endif + md_iterations_tm.stop(); } // md loop diff --git a/src/mgmol_config.h b/src/mgmol_config.h new file mode 100644 index 00000000..e126ea18 --- /dev/null +++ b/src/mgmol_config.h @@ -0,0 +1,19 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +// Description: A configuration header for mgmol. +// + +#ifndef INCLUDED_MGMOL_CONFIG_H +#define INCLUDED_MGMOL_CONFIG_H + +/* Have google test library. */ +/* #undef MGMOL_HAS_LIBROM */ + +#endif diff --git a/src/mgmol_config.h.in b/src/mgmol_config.h.in new file mode 100644 index 00000000..0b099f6c --- /dev/null +++ b/src/mgmol_config.h.in @@ -0,0 +1,19 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +// Description: A configuration header for mgmol. +// + +#ifndef INCLUDED_MGMOL_CONFIG_H +#define INCLUDED_MGMOL_CONFIG_H + +/* Have google test library. */ +#cmakedefine MGMOL_HAS_LIBROM + +#endif diff --git a/src/mgmol_run.cc b/src/mgmol_run.cc index 9240d507..92390e83 100644 --- a/src/mgmol_run.cc +++ b/src/mgmol_run.cc @@ -14,7 +14,7 @@ #include "Mesh.h" #include "PackedCommunicationBuffer.h" #include "ReplicatedWorkSpace.h" -// #include "SparseDistMatrix.h" +#include "SparseDistMatrix.h" #include "tools.h" #include @@ -43,24 +43,6 @@ int mgmol_init(MPI_Comm comm) assert(mype > -1); MPIdata::onpe0 = (MPIdata::mype == 0); - if (mype == 0) - { -#if defined(__GNUC__) && !defined(__clang__) - std::cout << "GCC " << __GNUC__ << "." << __GNUC_MINOR__ << "." - << __GNUC_PATCHLEVEL__ << std::endl; -#elif defined(__clang__) - std::cout << "Clang " << __clang_major__ << "." << __clang_minor__ - << "." << __clang_patchlevel__ << std::endl; -#else - std::cout << "Unknown Compiler" << std::endl; -#endif - - int version, subversion; - MPI_Get_version(&version, &subversion); - std::cout << "MPI Version: " << version << "." << subversion - << std::endl; - } - #ifdef MGMOL_USE_BLIS bli_init(); #endif @@ -165,10 +147,8 @@ void mgmol_finalize() if (!ct.short_sighted) { -#ifdef MGMOL_USE_SCALAPACK // need to destroy any MPI based object befor calling MPI_Finalize MatricesBlacsContext::instance().clear(); -#endif } // release memory for static arrays diff --git a/src/mlwf.cc b/src/mlwf.cc index 23805ffc..dc920134 100644 --- a/src/mlwf.cc +++ b/src/mlwf.cc @@ -11,18 +11,15 @@ #include "LocGridOrbitals.h" #include "LocalizationRegions.h" #include "MGmol.h" +#include "MLWFTransform.h" #include "Mesh.h" +#include "NOLMOTransform.h" #include "ProjectedMatrices.h" #include "ReplicatedWorkSpace.h" #include "SinCosOps.h" #include "blas3_c.h" #include "mputils.h" -#ifdef MGMOL_USE_SCALAPACK -#include "MLWFTransform.h" -#include "NOLMOTransform.h" -#endif - #include Timer get_NOLMO_tm("get_NOLMO"); @@ -35,7 +32,6 @@ void dtrsm_c(const char side, const char uplo, const char transa, DTRSM(&side, &uplo, &transa, &diag, &m, &n, &alpha, a, &lda, b, &ldb); } -#ifdef MGMOL_USE_SCALAPACK void distributeColumns( std::vector& vmm, dist_matrix::DistMatrix& mat) { @@ -342,7 +338,6 @@ int MGmol::get_NOLMO(NOLMOTransform& noot, OrbitalsType& orbitals, return 0; } -#endif template class MGmol>; template class MGmol>; diff --git a/src/pb/DielFunc.h b/src/pb/DielFunc.h index 7d4ccff3..05d991b0 100644 --- a/src/pb/DielFunc.h +++ b/src/pb/DielFunc.h @@ -90,7 +90,7 @@ class DielFunc : public GridFunc drho0_ = A.drho0_; } - ~DielFunc() {}; + ~DielFunc() override {}; DielFunc& operator=(const T); diff --git a/src/pb/GridFunc.cc b/src/pb/GridFunc.cc index d1cfdf17..fe5ac4ed 100644 --- a/src/pb/GridFunc.cc +++ b/src/pb/GridFunc.cc @@ -27,6 +27,19 @@ const double inv64 = 1. / 64.; namespace pb { +Timer GridFuncInterface::trade_bc_tm_("GridFunc::trade_bc"); +Timer GridFuncInterface::restrict3D_tm_("GridFunc::restrict3D"); +Timer GridFuncInterface::extend3D_tm_("GridFunc::extend3D"); +Timer GridFuncInterface::prod_tm_("GridFunc::prod"); +Timer GridFuncInterface::gather_tm_("GridFunc::gather"); +Timer GridFuncInterface::scatter_tm_("GridFunc::scatter"); +Timer GridFuncInterface::all_gather_tm_("GridFunc::all_gather"); +Timer GridFuncInterface::finishExchangeNorthSouth_tm_( + "GridFunc::finishExNorthSouth"); +Timer GridFuncInterface::finishExchangeUpDown_tm_("GridFunc::finishExUpDown"); +Timer GridFuncInterface::finishExchangeEastWest_tm_( + "GridFunc::finishExEastWest"); + template std::vector GridFunc::buf1_; template @@ -651,7 +664,7 @@ int GridFunc::count_threshold(const T threshold) if (rc != MPI_SUCCESS) { std::cout << "MPI_Allreduce double sum failed!!!" << std::endl; - mmpi.abort(); + mype_env().globalExit(); } icount = sum; } @@ -2719,9 +2732,72 @@ void GridFunc::setBoundaryValues( } // dot product on the global grid (distributed) -template -template -double GridFunc::gdot(const GridFunc& vv) const +/* This is split into the double-type argument and float-type argument + * below. This is necessary to ensure that the underlying MPdot routine + * that is called returns the same result whether T=float and vv is double + * or T=double and vv is float. Otherwise the GridFunc copy constructor + * would use a copy of vv of the same type as "this" GridFunc object (i.e. + * type T). This could lead to different results for float-double and + * double-float combinations of this function. + */ +template +double GridFunc::gdot(const GridFunc& vv) const +{ + const int nghosts = ghost_pt(); + + int dimz = grid_.dim(2); + + const int endx = (nghosts + dim_[0]) * incx_; + const int endy = (nghosts + dim_[1]) * incy_; + + int initx = nghosts * incx_; + int inity = nghosts * incy_; + int initz = nghosts; + + // remove "layers" belonging to BC + if (((bc_[0] != 1) || (vv.bc(0) != 1)) && mype_env().my_mpi(0) == 0) + initx += incx_; + if (((bc_[1] != 1) || (vv.bc(1) != 1)) && mype_env().my_mpi(1) == 0) + inity += incy_; + if (((bc_[2] != 1) || (vv.bc(2) != 1)) && mype_env().my_mpi(2) == 0) + { + dimz--; + initz++; + } + + const double* const vv1 = vv.uu(); + const T* const vv2 = uu_; + double my_dot = 0.; + for (int ix = initx; ix < endx; ix += incx_) + { + for (int iy = inity; iy < endy; iy += incy_) + { + int iz = ix + iy + initz; + my_dot += LinearAlgebraUtils::MPdot( + dimz, &vv1[iz], &vv2[iz]); + } + } + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + if (mype_env().n_mpi_tasks() > 1) + { + double sum = 0.; + int rc = mmpi.allreduce(&my_dot, &sum, 1, MPI_SUM); + if (rc != MPI_SUCCESS) + { + std::cout << "MPI_Allreduce double sum failed in gdot!!!" + << std::endl; + mype_env().globalExit(); + } + my_dot = sum; + } + + return my_dot; +} + +// dot product on the global grid (distributed) +template +double GridFunc::gdot(const GridFunc& vv) const { const int nghosts = ghost_pt(); @@ -2745,9 +2821,9 @@ double GridFunc::gdot(const GridFunc& vv) const initz++; } - const ScalarType2* const vv1 = vv.uu(); - const ScalarType1* const vv2 = uu_; - double my_dot = 0.; + const float* const vv1 = vv.uu(); + const T* const vv2 = uu_; + double my_dot = 0.; for (int ix = initx; ix < endx; ix += incx_) { for (int iy = inity; iy < endy; iy += incy_) @@ -2767,7 +2843,7 @@ double GridFunc::gdot(const GridFunc& vv) const { std::cout << "MPI_Allreduce double sum failed in gdot!!!" << std::endl; - mmpi.abort(); + mype_env().globalExit(); } my_dot = sum; } @@ -2791,13 +2867,13 @@ bool GridFunc::def_const() const if ((!bc_[0] || !bc_[1] || !bc_[2])) tmp = 1; MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - if (mmpi.size() > 1) + if (mype_env().n_mpi_tasks() > 1) { int rc = mmpi.allreduce(&tmp, &sum, 1, MPI_SUM); if (rc != MPI_SUCCESS) { std::cout << "MPI_Allreduce double sum failed!!!" << std::endl; - mmpi.abort(); + mype_env().globalExit(); } } @@ -2840,7 +2916,7 @@ double GridFunc::get_average() if (rc != MPI_SUCCESS) { std::cout << "MPI_Allreduce double sum failed!!!" << std::endl; - mmpi.abort(); + mype_env().globalExit(); } sum = tmp / ((double)mype_env().n_mpi_tasks()); } @@ -2946,7 +3022,7 @@ double GridFunc::integral() const if (rc != MPI_SUCCESS) { std::cout << "MPI_Allreduce double sum failed!!!" << std::endl; - mmpi.abort(); + mype_env().globalExit(); } integral = tmp; } @@ -3143,7 +3219,7 @@ void GridFunc::test_newgrid() if (rc != MPI_SUCCESS) { std::cout << "MPI_Allreduce double sum failed!!!" << std::endl; - mmpi.abort(); + mype_env().globalExit(); } my_dot = sum; s2 = sqrt(grid_.vel() * my_dot); @@ -3302,7 +3378,7 @@ double GridFunc::get_bias() if (rc != MPI_SUCCESS) { std::cout << "MPI_Bcast failed in get_bias()!!!" << std::endl; - mmpi.abort(); + mype_env().globalExit(); } for (unsigned int j = 0; j < grid_.dim(1); j++) diff --git a/src/pb/GridFunc.h b/src/pb/GridFunc.h index 78c048ed..1e0930a3 100644 --- a/src/pb/GridFunc.h +++ b/src/pb/GridFunc.h @@ -11,6 +11,7 @@ #define PB_GRIDFUNC_H #include "Grid.h" +#include "GridFuncInterface.h" #include "Timer.h" #include @@ -25,8 +26,8 @@ namespace pb class PEenv; -template -class GridFunc +template +class GridFunc : public GridFuncInterface { bool directionDirichlet_[3]; bool directionPeriodic_[3]; @@ -53,20 +54,19 @@ class GridFunc // private functions bool def_const() const; - void setBoundaryValuesNeumannX(const ScalarType alpha); - void setBoundaryValuesNeumannY(const ScalarType alpha); - void setBoundaryValuesNeumannZ(const ScalarType alpha); + void setBoundaryValuesNeumannX(const T alpha); + void setBoundaryValuesNeumannY(const T alpha); + void setBoundaryValuesNeumannZ(const T alpha); - ScalarType one(const double r, const double a) + T one(const double r, const double a) { (void)r; // unused (void)a; // unused return 1.; } - ScalarType ran0(); - ScalarType radial_func( - const double r, const double a, const short ftype = 0); + T ran0(); + T radial_func(const double r, const double a, const short ftype = 0); // allocate memory for function void alloc(); @@ -75,45 +75,34 @@ class GridFunc void resizeBuffers(); - static Timer trade_bc_tm_; - static Timer extend3D_tm_; - static Timer restrict3D_tm_; - static Timer prod_tm_; - static Timer gather_tm_; - static Timer scatter_tm_; - static Timer all_gather_tm_; - static Timer finishExchangeNorthSouth_tm_; - static Timer finishExchangeUpDown_tm_; - static Timer finishExchangeEastWest_tm_; - protected: const Grid& grid_; // data storage - std::unique_ptr memory_; + std::unique_ptr memory_; // raw pointer to data - ScalarType* uu_; + T* uu_; bool updated_boundaries_; short bc_[3]; - ScalarType valuesNeumann_[3]; + T valuesNeumann_[3]; - static GridFunc* bc_func_; + static GridFunc* bc_func_; - static std::vector buf1_; - static std::vector buf2_; - static std::vector buf3_; - static std::vector buf4_; + static std::vector buf1_; + static std::vector buf2_; + static std::vector buf3_; + static std::vector buf4_; - void setValues(const int n, const ScalarType* src); + void setValues(const int n, const T* src); public: // Constructors GridFunc(const Grid&, const short, const short, const short); // constructor with pointer to data allocation - GridFunc(const Grid&, const short, const short, const short, ScalarType*, + GridFunc(const Grid&, const short, const short, const short, T*, const bool updated_boundaries = false); // copy constructor @@ -121,26 +110,26 @@ class GridFunc GridFunc(const GridFunc& A); // copy constructor on different grid - GridFunc(const GridFunc&, const Grid&); + GridFunc(const GridFunc&, const Grid&); - void setValues(const GridFunc& src); - void setValues(const ScalarType val); + void setValues(const GridFunc& src); + void setValues(const T val); void setZero(); int inc(const short dir) const { return grid_.inc(dir); } - void assign(const GridFunc& src, const char dis); - void scatterFrom(const GridFunc& src); + void assign(const GridFunc& src, const char dis); + void scatterFrom(const GridFunc& src); double fmax(); - template - void getValues(ScalarType2*) const; + template + void getValues(T2*) const; - void init_vect(ScalarType*, const char) const; - void init_vect_shift(ScalarType* global_func) const; - void gather(ScalarType*) const; - void allGather(ScalarType*) const; + void init_vect(T*, const char) const; + void init_vect_shift(T* global_func) const; + void gather(T*) const; + void allGather(T*) const; short bc(const short dir) const { @@ -161,14 +150,14 @@ class GridFunc bc_[2] = pz; updated_boundaries_ = false; } - void setValuesNeumann(const ScalarType valuesNeumann[3]) + void setValuesNeumann(const T valuesNeumann[3]) { valuesNeumann_[0] = valuesNeumann[0]; valuesNeumann_[1] = valuesNeumann[1]; valuesNeumann_[2] = valuesNeumann[2]; updated_boundaries_ = false; } - ScalarType* uu(const int i = 0) const + T* uu(const int i = 0) const { assert(uu_ != 0); return uu_ + i; @@ -177,86 +166,85 @@ class GridFunc void set_updated_boundaries(const bool i) { updated_boundaries_ = i; } - void copyFrom(GridFunc* src) + void copyFrom(GridFunc* src) { - memcpy(uu_, src->uu_, grid_.sizeg() * sizeof(ScalarType)); + memcpy(uu_, src->uu_, grid_.sizeg() * sizeof(T)); } - void copyFrom(GridFunc& src) + void copyFrom(GridFunc& src) { - memcpy(uu_, src.uu_, grid_.sizeg() * sizeof(ScalarType)); + memcpy(uu_, src.uu_, grid_.sizeg() * sizeof(T)); } - GridFunc& operator=(const GridFunc& func); - GridFunc& operator=(const ScalarType val); + GridFunc& operator=(const GridFunc& func); + GridFunc& operator=(const T val); - GridFunc operator+(const GridFunc& A); - GridFunc operator-(const GridFunc& A); - GridFunc operator*(const double val); + GridFunc operator+(const GridFunc& A); + GridFunc operator-(const GridFunc& A); + GridFunc operator*(const double val); - GridFunc& operator+=(const GridFunc& func); - GridFunc& operator+=(ScalarType alpha); + GridFunc& operator+=(const GridFunc& func); + GridFunc& operator+=(T alpha); - GridFunc& operator-=(const ScalarType alpha); + GridFunc& operator-=(const T alpha); - GridFunc& operator*=(const double alpha); - GridFunc& operator*=(const GridFunc& B); + GridFunc& operator*=(const double alpha); + GridFunc& operator*=(const GridFunc& B); - GridFunc& operator/=(const GridFunc& B); + GridFunc& operator/=(const GridFunc& B); - void axpy(const ScalarType alpha, const GridFunc& vv); + void axpy(const T alpha, const GridFunc& vv); void scal(const double alpha); - void prod(const GridFunc& A, const GridFunc& B); - void diff(const GridFunc& A, const GridFunc& B); + void prod(const GridFunc& A, const GridFunc& B); + void diff(const GridFunc& A, const GridFunc& B); - void set_max(const ScalarType val); + void set_max(const T val); - virtual ~GridFunc(); + ~GridFunc() override; - int count_threshold(const ScalarType); + int count_threshold(const T); - void jacobi(const GridFunc& v, - const GridFunc& epsilon, const double c0); - void add_prod( - const GridFunc& v1, const GridFunc& v2); - void substract_prod( - const GridFunc& v1, const GridFunc& v2); + void jacobi( + const GridFunc& v, const GridFunc& epsilon, const double c0); + void add_prod(const GridFunc& v1, const GridFunc& v2); + void substract_prod(const GridFunc& v1, const GridFunc& v2); void init_radial(const double[3], const double, const short ftype = 0); void initTrigo3d(const short bc[3], const int n[3]); - void initCos3d(const ScalarType k[3]); + void initCos3d(const T k[3]); void print_radial(const char[]); void defaultTrade_boundaries(); virtual void trade_boundaries() { defaultTrade_boundaries(); } - void set_bc_func(GridFunc* bc_func) { bc_func_ = bc_func; } + void set_bc_func(GridFunc* bc_func) { bc_func_ = bc_func; } - void setBoundaryValues(const ScalarType, const bool direction[3]); - void setBoundaryValues( - const GridFunc&, const bool direction[3]); - void setBoundaryValuesNeumann( - const ScalarType alpha[3], const bool direction[3]); + void setBoundaryValues(const T, const bool direction[3]); + void setBoundaryValues(const GridFunc&, const bool direction[3]); + void setBoundaryValuesNeumann(const T alpha[3], const bool direction[3]); bool isZero(const double tol = 1.e-16, const bool wghosts = false); void test_setBoundaryValues(); - template - double gdot(const GridFunc&) const; + /* gdot is split into two for consistency between float-double and + * double-float calls to gdot. See function definition for more details. + */ + double gdot(const GridFunc&) const; + double gdot(const GridFunc&) const; double norm2() const; - void extend3D(GridFunc&); - void restrict3D(GridFunc&); + void extend3D(GridFunc&); + void restrict3D(GridFunc&); void test_grid_transfer(); void init_rand(); - void initTriLin(const ScalarType a[4], const bool wghosts = true); + void initTriLin(const T a[4], const bool wghosts = true); double integral() const; double get_average(); double average0(); - template - void assign(const ScalarType2* const, const char dis = 'd'); + template + void assign(const T2* const, const char dis = 'd'); void test_newgrid(); void print(std::ostream&); void write_plt(const char[]) const; void write_global_x(const char str[]); - void global_xyz_task0(ScalarType*); + void global_xyz_task0(T*); void write_xyz(std::ofstream&) const; void write_zyx(std::ofstream&) const; void write_global_xyz(std::ofstream&); @@ -274,10 +262,10 @@ class GridFunc int size() const { return grid_.size(); } const Grid& grid() const { return grid_; } - GridFunc& operator-=(const GridFunc& func); + GridFunc& operator-=(const GridFunc& func); void resetData() { - memset(uu_, 0, grid_.sizeg() * sizeof(ScalarType)); + memset(uu_, 0, grid_.sizeg() * sizeof(T)); updated_boundaries_ = true; } void getCellCornersValues( @@ -291,66 +279,21 @@ class GridFunc void finishExchangeEastWest(); void setBoundaryValuesBeforeTrade(); - - static void printTimers(std::ostream& os) - { - trade_bc_tm_.print(os); - finishExchangeNorthSouth_tm_.print(os); - finishExchangeUpDown_tm_.print(os); - finishExchangeEastWest_tm_.print(os); - extend3D_tm_.print(os); - restrict3D_tm_.print(os); - prod_tm_.print(os); - gather_tm_.print(os); - scatter_tm_.print(os); - all_gather_tm_.print(os); - } }; -template -double dot(const GridFunc& A, const GridFunc& B) +template +double dot(const GridFunc& A, const GridFunc& B) { return A.gdot(B) * A.grid().vel(); } -template -double norm(const GridFunc& A) +template +double norm(const GridFunc& A) { return A.norm2(); } -template -GridFunc* GridFunc::bc_func_ = nullptr; - -template -Timer GridFunc::trade_bc_tm_( - "GridFunc::trade_bc_" + std::to_string(sizeof(ScalarType) * 8)); -template -Timer GridFunc::restrict3D_tm_( - "GridFunc::restrict3D_" + std::to_string(sizeof(ScalarType) * 8)); -template -Timer GridFunc::extend3D_tm_( - "GridFunc::extend3D_" + std::to_string(sizeof(ScalarType) * 8)); -template -Timer GridFunc::prod_tm_( - "GridFunc::prod_" + std::to_string(sizeof(ScalarType) * 8)); -template -Timer GridFunc::gather_tm_( - "GridFunc::gather_" + std::to_string(sizeof(ScalarType) * 8)); -template -Timer GridFunc::scatter_tm_( - "GridFunc::scatter_" + std::to_string(sizeof(ScalarType) * 8)); -template -Timer GridFunc::all_gather_tm_( - "GridFunc::all_gather_" + std::to_string(sizeof(ScalarType) * 8)); -template -Timer GridFunc::finishExchangeNorthSouth_tm_( - "GridFunc::finishExNorthSouth_" + std::to_string(sizeof(ScalarType) * 8)); -template -Timer GridFunc::finishExchangeUpDown_tm_( - "GridFunc::finishExUpDown_" + std::to_string(sizeof(ScalarType) * 8)); -template -Timer GridFunc::finishExchangeEastWest_tm_( - "GridFunc::finishExEastWest_" + std::to_string(sizeof(ScalarType) * 8)); +template +GridFunc* GridFunc::bc_func_ = nullptr; } // namespace pb diff --git a/src/pb/GridFuncInterface.h b/src/pb/GridFuncInterface.h new file mode 100644 index 00000000..b9e908da --- /dev/null +++ b/src/pb/GridFuncInterface.h @@ -0,0 +1,51 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#ifndef GRIDFUNCINTERFACE_H_ +#define GRIDFUNCINTERFACE_H_ + +#include "Timer.h" + +namespace pb +{ + +class GridFuncInterface +{ +protected: + static Timer trade_bc_tm_; + static Timer extend3D_tm_; + static Timer restrict3D_tm_; + static Timer prod_tm_; + static Timer gather_tm_; + static Timer scatter_tm_; + static Timer all_gather_tm_; + static Timer finishExchangeNorthSouth_tm_; + static Timer finishExchangeUpDown_tm_; + static Timer finishExchangeEastWest_tm_; + +public: + virtual ~GridFuncInterface() { } + + static void printTimers(std::ostream& os) + { + trade_bc_tm_.print(os); + finishExchangeNorthSouth_tm_.print(os); + finishExchangeUpDown_tm_.print(os); + finishExchangeEastWest_tm_.print(os); + extend3D_tm_.print(os); + restrict3D_tm_.print(os); + prod_tm_.print(os); + gather_tm_.print(os); + scatter_tm_.print(os); + all_gather_tm_.print(os); + } +}; + +} // namespace pb +#endif diff --git a/src/pb/GridFuncVector.h b/src/pb/GridFuncVector.h index a807d594..88f115d8 100644 --- a/src/pb/GridFuncVector.h +++ b/src/pb/GridFuncVector.h @@ -503,36 +503,34 @@ class GridFuncVector template Timer GridFuncVector::trade_bc_tm_( - "GridFuncVector::trade_bc_" + std::to_string(sizeof(ScalarType) * 8)); + "GridFuncVector::trade_bc_" + std::to_string(sizeof(ScalarType))); template Timer GridFuncVector::trade_bc_colors_tm_( - "GridFuncVector::trade_bc_colors" + std::to_string(sizeof(ScalarType) * 8)); + "GridFuncVector::trade_bc_colors"); template Timer GridFuncVector::prod_tm_( - "GridFuncVector::prod" + std::to_string(sizeof(ScalarType) * 8)); + "GridFuncVector::prod"); template Timer GridFuncVector::finishExchangeNorthSouth_tm_( - "GridFuncVector::finishExNorthSouth_" - + std::to_string(sizeof(ScalarType) * 8)); + "GridFuncVector::finishExNorthSouth_" + std::to_string(sizeof(ScalarType))); template Timer GridFuncVector::finishExchangeUpDown_tm_( - "GridFuncVector::finishExUpDown_" + std::to_string(sizeof(ScalarType) * 8)); + "GridFuncVector::finishExUpDown_" + std::to_string(sizeof(ScalarType))); template Timer GridFuncVector::finishExchangeEastWest_tm_( - "GridFuncVector::finishExEastWest_" - + std::to_string(sizeof(ScalarType) * 8)); + "GridFuncVector::finishExEastWest_" + std::to_string(sizeof(ScalarType))); template Timer GridFuncVector::wait_north_south_tm_( - "GridFuncVector::waitNS_" + std::to_string(sizeof(ScalarType) * 8)); + "GridFuncVector::waitNS_" + std::to_string(sizeof(ScalarType))); template Timer GridFuncVector::wait_up_down_tm_( - "GridFuncVector::waitUD_" + std::to_string(sizeof(ScalarType) * 8)); + "GridFuncVector::waitUD_" + std::to_string(sizeof(ScalarType))); template Timer GridFuncVector::wait_east_west_tm_( - "GridFuncVector::waitEW_" + std::to_string(sizeof(ScalarType) * 8)); + "GridFuncVector::waitEW_" + std::to_string(sizeof(ScalarType))); template Timer GridFuncVector::copy_tm_( - "GridFuncVector::copy_" + std::to_string(sizeof(ScalarType) * 8)); + "GridFuncVector::copy_" + std::to_string(sizeof(ScalarType))); template Map2Masks* GridFuncVector::map2masks_(nullptr); diff --git a/src/quench.cc b/src/quench.cc index b6acec5d..695c4863 100644 --- a/src/quench.cc +++ b/src/quench.cc @@ -33,6 +33,7 @@ #include "MPIdata.h" #include "MasksSet.h" #include "Mesh.h" +#include "OrbitalsTransform.h" #include "PolakRibiereSolver.h" #include "Potentials.h" #include "ProjectedMatricesInterface.h" @@ -60,12 +61,8 @@ Timer updateCenters_tm("MGmol::updateCenters"); template <> void MGmol>::adaptLR( - const SpreadsAndCenters>* /*spreadf*/ -#ifdef MGMOL_USE_SCALAPACK - , - const OrbitalsTransform* /*ot*/ -#endif -) + const SpreadsAndCenters>* /*spreadf*/, + const OrbitalsTransform* /*ot*/) { } @@ -75,12 +72,8 @@ void MGmol>::adaptLR( // 1 -> radius only // 2 -> center and radius template -void MGmol::adaptLR(const SpreadsAndCenters* spreadf -#ifdef MGMOL_USE_SCALAPACK - , - const OrbitalsTransform* ot -#endif -) +void MGmol::adaptLR( + const SpreadsAndCenters* spreadf, const OrbitalsTransform* ot) { assert(lrs_); @@ -120,7 +113,6 @@ void MGmol::adaptLR(const SpreadsAndCenters* spreadf } double avg; -#ifdef MGMOL_USE_SCALAPACK // if ct.lr_volume_calc true, calculate volume base on spreads if (ct.lr_volume_calc == 1) { @@ -146,7 +138,6 @@ void MGmol::adaptLR(const SpreadsAndCenters* spreadf avg = lrs_->updateRadii(ot, ratio); } else -#endif { if (onpe0) os_ << " Adapt with constant LR volume" << std::endl; avg = lrs_->updateRadiiConstVol(*spreadf); @@ -196,7 +187,6 @@ void MGmol::resetProjectedMatricesAndDM( dm_strategy_->initialize(orbitals); } -#ifdef MGMOL_USE_SCALAPACK // try to use some rotations to avoid degeneracies template bool MGmol::rotateStatesPairsCommonCenter( @@ -413,7 +403,6 @@ void MGmol::disentangleOrbitals(OrbitalsType& orbitals, ct.num_MD_steps--; } } -#endif template <> void MGmol>::applyAOMMprojection( @@ -529,7 +518,6 @@ int MGmol::outerSolve(OrbitalsType& orbitals, retval = solver.solve(orbitals, work_orbitals); } -#ifdef MGMOL_USE_SCALAPACK else { DavidsonSolver::outerSolve(OrbitalsType& orbitals, retval = solver.solve(orbitals, work_orbitals); } -#endif break; } @@ -588,9 +575,7 @@ int MGmol::quench(OrbitalsType& orbitals, Ions& ions, orbitals.setDataWithGhosts(); orbitals.trade_boundaries(); -#ifdef MGMOL_USE_SCALAPACK disentangleOrbitals(orbitals, work_orbitals, ions, max_steps); -#endif // setup "kernel" functions for AOMM algorithm if (ct.use_kernel_functions) diff --git a/src/read_config.cc b/src/read_config.cc index 614ad5f0..0aeae89e 100644 --- a/src/read_config.cc +++ b/src/read_config.cc @@ -7,6 +7,7 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE +#include "MGmol_prototypes.h" #include #include #include @@ -191,11 +192,7 @@ int read_config(int argc, char** argv, po::variables_map& vm, po::value()->default_value(false), "print projected matrices in MM format")( "ProjectedMatrices.replicated", -#ifdef MGMOL_USE_SCALAPACK po::value()->default_value(false), -#else - po::value()->default_value(true), -#endif "use replicated projected matrices")("LocalizationRegions.radius", po::value()->default_value(1000.), "Localization regions radius")("LocalizationRegions.adaptive", @@ -219,6 +216,10 @@ int read_config(int argc, char** argv, po::variables_map& vm, po::value()->default_value(""), "Output file for dumping cluster information in vtk format"); +#ifdef MGMOL_HAS_LIBROM + setupROMConfigOption(config); +#endif + // Hidden options, will be allowed in config file, but will not be // shown to the user. po::options_description hidden("Hidden options"); @@ -425,3 +426,37 @@ int read_config(int argc, char** argv, po::variables_map& vm, return 0; } + +#ifdef MGMOL_HAS_LIBROM +void setupROMConfigOption(po::options_description& rom_cfg) +{ + rom_cfg.add_options()("ROM.stage", + po::value()->default_value("none"), + "ROM workflow stage: offline; build; online; none.")( + "ROM.offline.restart_filefmt", + po::value()->default_value(""), + "File name format to read for snapshots.")( + "ROM.offline.restart_min_idx", po::value()->default_value(-1), + "Minimum index for snapshot file format.")( + "ROM.offline.restart_max_idx", po::value()->default_value(-1), + "Maximum index for snapshot file format.")("ROM.offline.basis_file", + po::value()->default_value(""), + "File name for libROM snapshot/POD matrices.")( + "ROM.offline.save_librom_snapshot", + po::value()->default_value(false), + "Save libROM snapshot file at FOM simulation.")( + "ROM.offline.librom_snapshot_freq", po::value()->default_value(-1), + "Frequency of saving libROM snapshot file at FOM simulation.")( + "ROM.offline.variable", po::value()->default_value(""), + "FOM variable to perform POD: either orbitals or potential.")( + "ROM.basis.compare_md", po::value()->default_value(false), + "Compare MD or single-step force.")("ROM.basis.number_of_orbital_basis", + po::value()->default_value(-1), + "Number of orbital POD basis.")("ROM.basis.number_of_potential_basis", + po::value()->default_value(-1), + "Number of potential POD basis to build Hartree potential ROM " + "operator.")("ROM.potential_rom_file", + po::value()->default_value(""), + "File name to save/load potential ROM operators."); +} +#endif diff --git a/src/rom.cc b/src/rom.cc new file mode 100644 index 00000000..00bc0ee0 --- /dev/null +++ b/src/rom.cc @@ -0,0 +1,127 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "mgmol_config.h" +#ifdef MGMOL_HAS_LIBROM + +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" + +#include "librom.h" + +#include +#include +#include +#include + +// Save the wavefunction snapshots +template +int MGmol::save_orbital_snapshot( + std::string file_path, OrbitalsType& orbitals) +{ + std::string snapshot_filename = file_path; + struct stat s; + if (stat(file_path.c_str(), &s) == 0) + { + if (s.st_mode & S_IFDIR) + { + snapshot_filename = file_path + "/orbital"; + } + else if (s.st_mode & S_IFREG) + { + snapshot_filename = file_path + "_orbital"; + } + else + { + std::cout << file_path + << " exists but is not a directory or a file." + << std::endl; + return 1; + } + } + + const int dim = orbitals.getLocNumpt(); + const int totalSamples = orbitals.chromatic_number(); + + CAROM::Options svd_options(dim, totalSamples, 1); + CAROM::BasisGenerator basis_generator( + svd_options, false, snapshot_filename); + + for (int i = 0; i < totalSamples; ++i) + basis_generator.takeSample(orbitals.getPsi(i)); + + basis_generator.writeSnapshot(); + + return 0; +} + +template +void MGmol::project_orbital( + std::string file_path, int rdim, OrbitalsType& orbitals) +{ + const int dim = orbitals.getLocNumpt(); + const int totalSamples = orbitals.chromatic_number(); + + CAROM::Options svd_options(dim, totalSamples, 1); + CAROM::BasisGenerator basis_generator(svd_options, false, "foo"); + + for (int i = 0; i < totalSamples; ++i) + basis_generator.takeSample(orbitals.getPsi(i)); + const CAROM::Matrix* orbital_snapshots + = basis_generator.getSnapshotMatrix(); + + CAROM::BasisReader reader(file_path); + CAROM::Matrix* orbital_basis = reader.getSpatialBasis(rdim); + + CAROM::Matrix* proj_orbital_coeff + = orbital_basis->transposeMult(orbital_snapshots); + CAROM::Matrix* proj_orbital_snapshots + = orbital_basis->mult(proj_orbital_coeff); + + Control& ct = *(Control::instance()); + Mesh* mesh = Mesh::instance(); + pb::GridFunc gf_psi( + mesh->grid(), ct.bcWF[0], ct.bcWF[1], ct.bcWF[2]); + CAROM::Vector snapshot, proj_snapshot; + for (int i = 0; i < totalSamples; ++i) + { + orbital_snapshots->getColumn(i, snapshot); + proj_orbital_snapshots->getColumn(i, proj_snapshot); + gf_psi.assign(proj_snapshot.getData()); + orbitals.setPsi(gf_psi, i); + snapshot -= proj_snapshot; + std::cout << "Error for orbital " << i << " = " << snapshot.norm() + << std::endl; + } +} + +// template +// void ExtendedGridOrbitals::set(std::string file_path, int rdim) +//{ +// const int dim = getLocNumpt(); + +// CAROM::BasisReader reader(file_path); +// CAROM::Matrix* orbital_basis = reader.getSpatialBasis(rdim); + +// Control& ct = *(Control::instance()); +// Mesh* mymesh = Mesh::instance(); +// pb::GridFunc gf_psi(mymesh->grid(), ct.bcWF[0], ct.bcWF[1], +// ct.bcWF[2]); CAROM::Vector psi; for (int i = 0; i < rdim; ++i) +// { +// orbital_basis->getColumn(i, psi); +// gf_psi.assign(psi.getData()); +// setPsi(gf_psi, i); +// } +//} + +template class MGmol>; +template class MGmol>; + +#endif // MGMOL_HAS_LIBROM diff --git a/src/rom_Control.h b/src/rom_Control.h new file mode 100644 index 00000000..0e852522 --- /dev/null +++ b/src/rom_Control.h @@ -0,0 +1,63 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#ifndef ROM_CONTROL_H +#define ROM_CONTROL_H + +#include +#include +#include +#include +#include + +enum class ROMStage +{ + OFFLINE, + ONLINE, + RESTORE, // TODO(kevin): what stage is this? + BUILD, + ONLINE_PINNED_H2O_3DOF, + TEST_ORBITAL, + ONLINE_POISSON, + TEST_POISSON, + TEST_RHO, + TEST_ION, + UNSUPPORTED +}; + +enum class ROMVariable +{ + ORBITALS, + POTENTIAL, + NONE +}; + +/* Stored as a private member variable of Control class */ +struct ROMPrivateOptions +{ + ROMStage rom_stage = ROMStage::UNSUPPORTED; + + std::string restart_file_fmt = ""; + int restart_file_minidx = -1; + int restart_file_maxidx = -1; + std::string basis_file = ""; + ROMVariable variable = ROMVariable::NONE; + + /* save librom orbital snapshot matrix at FOM simulation. */ + bool save_librom_snapshot = false; + int librom_snapshot_freq = -1; + + /* options for ROM building */ + bool compare_md = false; + int num_orbbasis = -1; + int num_potbasis = -1; + std::string pot_rom_file = ""; +}; + +#endif // ROM_CONTROL_H diff --git a/src/rom_main.cc b/src/rom_main.cc new file mode 100644 index 00000000..f9014bee --- /dev/null +++ b/src/rom_main.cc @@ -0,0 +1,180 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +// +// main.cc +// +// Description: +// Real grid, finite difference, molecular dynamics program +// for with nonorthogonal localized orbitals. +// +// Uses Mehrstellen operators, multigrid accelerations, and +// non-local pseudopotentials. +// +// Includes LDA and PBE exchange and correlation functionals. +// +// Units: +// Potentials, eigenvalues and operators in Rydberg +// Energies in Hartree +// +#include "rom_workflows.h" + +// A helper function +template +std::ostream& operator<<(std::ostream& os, const std::vector& v) +{ + copy(v.begin(), v.end(), std::ostream_iterator(std::cout, " ")); + return os; +} + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + mgmol_init(comm); + + // read runtime parameters + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // use configure file if it can be found + // std::string config_filename("mgmol.cfg"); + + // read options from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + unsigned ngpts[3] = { ct.ngpts_[0], ct.ngpts_[1], ct.ngpts_[2] }; + double origin[3] = { ct.ox_, ct.oy_, ct.oz_ }; + const double cell[3] = { ct.lx_, ct.ly_, ct.lz_ }; + Mesh::setup(mmpi.commSpin(), ngpts, origin, cell, ct.lap_type); + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + /* Get ROM driver mode */ + ROMPrivateOptions rom_options = ct.getROMOptions(); + ROMStage rom_stage = rom_options.rom_stage; + + // Enter main scope + { + MGmolInterface* mgmol; + if (ct.isLocMode()) + mgmol = new MGmol>(global_comm, + *MPIdata::sout, input_filename, lrs_filename, + constraints_filename); + else + mgmol = new MGmol>(global_comm, + *MPIdata::sout, input_filename, lrs_filename, + constraints_filename); + + mgmol->setup(); + + switch (rom_stage) + { + case (ROMStage::OFFLINE): + if (ct.isLocMode()) + readRestartFiles>(mgmol); + else + readRestartFiles>(mgmol); + break; + + case (ROMStage::BUILD): + if (ct.isLocMode()) + buildROMPoissonOperator>(mgmol); + else + buildROMPoissonOperator>( + mgmol); + break; + + case (ROMStage::ONLINE_POISSON): + if (ct.isLocMode()) + runPoissonROM>(mgmol); + else + runPoissonROM>(mgmol); + break; + + case (ROMStage::TEST_POISSON): + if (ct.isLocMode()) + testROMPoissonOperator>(mgmol); + else + testROMPoissonOperator>( + mgmol); + break; + + case (ROMStage::TEST_RHO): + if (ct.isLocMode()) + testROMRhoOperator>(mgmol); + else + testROMRhoOperator>(mgmol); + + case (ROMStage::TEST_ION): + if (ct.isLocMode()) + testROMIonDensity>(mgmol); + else + testROMIonDensity>(mgmol); + + break; + + default: + std::cerr << "rom_main error: Unknown ROM stage" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + break; + } + + delete mgmol; + + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + time_t tt; + time(&tt); + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; + + // MemTrack::TrackDumpBlocks(); + + // MemTrack::TrackListMemoryUsage(); + + return 0; +} diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc new file mode 100644 index 00000000..73584eec --- /dev/null +++ b/src/rom_workflows.cc @@ -0,0 +1,1015 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "rom_workflows.h" +#include "Electrostatic.h" +#include +#include +#include + +namespace CAROM +{ + +/* + This is not implemented in libROM yet. + A temporary placeholder until it is merged into libROM. +*/ +void mult(const Matrix& A, const std::vector& sample_row_A, + const Matrix& B, Matrix& AB) +{ + CAROM_VERIFY(!B.distributed()); + CAROM_VERIFY(A.distributed() == AB.distributed()); + CAROM_VERIFY(A.numColumns() == B.numRows()); + CAROM_VERIFY(sample_row_A.size() <= A.numRows()); + + const int num_rows = sample_row_A.size(); + const int num_cols = B.numColumns(); + AB.setSize(num_rows, num_cols); + + // Do the multiplication. + const int Acol = A.numColumns(); + for (int r = 0; r < num_rows; ++r) + { + double* d_Arow = A.getData() + sample_row_A[r] * Acol; + for (int c = 0; c < num_cols; ++c) + { + double* d_Aptr = d_Arow; + double* d_Bptr = B.getData() + c; + double result_val = 0.0; + for (int ac = 0; ac < Acol; ++ac, d_Aptr++) + { + result_val += (*d_Aptr) * (*d_Bptr); + d_Bptr += num_cols; + } + AB.item(r, c) = result_val; + } + } +} + +} + +template +std::string string_format(const std::string& format, Args... args) +{ + int size_s = std::snprintf(nullptr, 0, format.c_str(), args...) + + 1; // Extra space for '\0' + if (size_s <= 0) + { + throw std::runtime_error("Error during formatting."); + } + auto size = static_cast(size_s); + std::unique_ptr buf(new char[size]); + std::snprintf(buf.get(), size, format.c_str(), args...); + return std::string( + buf.get(), buf.get() + size - 1); // We don't want the '\0' inside +} + +template +void readRestartFiles(MGmolInterface* mgmol_) +{ + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + + ROMPrivateOptions rom_options = ct.getROMOptions(); + /* type of variable we intend to run POD */ + ROMVariable rom_var = rom_options.variable; + + /* number of restart files, start/end indices */ + assert(rom_options.restart_file_minidx >= 0); + assert(rom_options.restart_file_maxidx >= 0); + const int minidx = rom_options.restart_file_minidx; + const int maxidx = rom_options.restart_file_maxidx; + const int num_restart = maxidx - minidx + 1; + + MGmol* mgmol = static_cast*>(mgmol_); + OrbitalsType* orbitals = mgmol->getOrbitals(); + Potentials& pot = mgmol->getHamiltonian()->potential(); + std::string filename; + + /* Determine basis prefix, dimension, and sample size */ + std::string basis_prefix = rom_options.basis_file; + int dim; + int totalSamples = num_restart; + const int chrom_num = orbitals->chromatic_number(); + switch (rom_var) + { + case ROMVariable::ORBITALS: + basis_prefix += "_orbitals"; + dim = orbitals->getLocNumpt(); + /* if orbitals, each sample have chromatic number of wave functions + */ + totalSamples *= orbitals->chromatic_number(); + break; + + case ROMVariable::POTENTIAL: + basis_prefix += "_potential"; + dim = pot.size(); + break; + + default: + ct.global_exit(); + break; + } + + /* Initialize libROM classes */ + CAROM::Options svd_options(dim, totalSamples, 1); + CAROM::BasisGenerator basis_generator(svd_options, false, basis_prefix); + + /* Collect the restart files */ + for (int k = minidx; k <= maxidx; k++) + { + filename = string_format(rom_options.restart_file_fmt, k); + mgmol->loadRestartFile(filename); + assert(dim == orbitals->getLocNumpt()); + assert(chrom_num == orbitals->chromatic_number()); + + switch (rom_var) + { + case ROMVariable::ORBITALS: + for (int i = 0; i < chrom_num; ++i) + basis_generator.takeSample(orbitals->getPsi(i)); + break; + + case ROMVariable::POTENTIAL: + basis_prefix += "_potential"; + /* we save hartree potential */ + // TODO: consider create overloaded takeSample with const + basis_generator.takeSample( + const_cast(pot.vh_rho().data())); + break; + } + } + basis_generator.writeSnapshot(); + basis_generator.endSamples(); +} + +template +void buildROMPoissonOperator(MGmolInterface* mgmol_) +{ + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const int rank = mmpi.mypeGlobal(); + const int nprocs = mmpi.size(); + + ROMPrivateOptions rom_options = ct.getROMOptions(); + /* type of variable we intend to run POD */ + ROMVariable rom_var = rom_options.variable; + if (rom_var != ROMVariable::POTENTIAL) + { + std::cerr << "buildROMPoissonOperator error: ROM variable must be " + "POTENTIAL to run this stage!\n" + << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + /* Load Hartree potential basis matrix */ + std::string basis_file = rom_options.basis_file; + const int num_pot_basis = rom_options.num_potbasis; + CAROM::BasisReader basis_reader(basis_file); + CAROM::Matrix* pot_basis = basis_reader.getSpatialBasis(num_pot_basis); + + /* Load PoissonSolver pointer */ + MGmol* mgmol = static_cast*>(mgmol_); + Poisson* poisson = mgmol->electrostat_->getPoissonSolver(); + + /* GridFunc initialization inputs */ + const pb::Grid& grid(poisson->vh().grid()); + short bc[3]; + for (int d = 0; d < 3; d++) + bc[d] = poisson->vh().bc(d); + + /* Initialize ROM matrix (undistributed) */ + CAROM::Matrix pot_rom(num_pot_basis, num_pot_basis, false); + + pb::GridFunc col_gf(grid, bc[0], bc[1], bc[2]); + pb::GridFunc opcol_gf(grid, bc[0], bc[1], bc[2]); + CAROM::Vector op_col(pot_basis->numRows(), true); + CAROM::Vector rom_col(num_pot_basis, false); + for (int c = 0; c < num_pot_basis; c++) + { + /* copy c-th column librom vector to GridFunc gf_col */ + CAROM::Vector* col = pot_basis->getColumn(c); + col_gf.assign(col->getData(), 'd'); + + /* apply Laplace operator */ + poisson->applyOperator(col_gf, opcol_gf); + + /* get librom view-vector of gf_opcol */ + opcol_gf.init_vect(op_col.getData(), 'd'); + + /* Compute basis projection of the column */ + /* Resulting vector is undistributed */ + pot_basis->transposeMult(op_col, rom_col); + + /* libROM matrix is row-major, so data copy is necessary */ + for (int r = 0; r < num_pot_basis; r++) + pot_rom(r, c) = rom_col(r); + + delete col; + } // for (int c = 0; c < num_pot_basis; c++) + + /* DEIM hyperreduction */ + CAROM::Matrix pot_rhs_rom(num_pot_basis, num_pot_basis, false); + std::vector global_sampled_row(num_pot_basis), + sampled_rows_per_proc(nprocs); + DEIM(pot_basis, num_pot_basis, global_sampled_row, sampled_rows_per_proc, + pot_rhs_rom, rank, nprocs); + + /* ROM rescaleTotalCharge operator */ + CAROM::Vector fom_ones(pot_basis->numRows(), true); + CAROM::Vector rom_ones(num_pot_basis, false); + fom_ones = mymesh->grid().vel(); // volume element + pot_basis->transposeMult(fom_ones, rom_ones); + + /* Save ROM operator */ + // write the file from PE0 only + if (MPIdata::onpe0) + { + std::string rom_oper = rom_options.pot_rom_file; + CAROM::HDFDatabase h5_helper; + h5_helper.create(rom_oper); + h5_helper.putInteger("number_of_potential_basis", num_pot_basis); + h5_helper.putDoubleArray("potential_rom_operator", pot_rom.getData(), + num_pot_basis * num_pot_basis, false); + + /* save the inverse as well */ + pot_rom.inverse(); + h5_helper.putDoubleArray("potential_rom_inverse", pot_rom.getData(), + num_pot_basis * num_pot_basis, false); + + /* save right-hand side hyper-reduction operator */ + h5_helper.putDoubleArray("potential_rhs_rom_inverse", + pot_rhs_rom.getData(), num_pot_basis * num_pot_basis, false); + + /* save right-hand side rescaling operator */ + h5_helper.putDoubleArray( + "potential_rhs_rescaler", rom_ones.getData(), num_pot_basis, false); + + h5_helper.close(); + } +} + +template +void runPoissonROM(MGmolInterface* mgmol_) +{ + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const int rank = mmpi.mypeGlobal(); + const int nprocs = mmpi.size(); + + ROMPrivateOptions rom_options = ct.getROMOptions(); + /* type of variable we intend to run POD */ + ROMVariable rom_var = rom_options.variable; + if (rom_var != ROMVariable::POTENTIAL) + { + std::cerr << "runPoissonROM error: ROM variable must be POTENTIAL to " + "run this stage!\n" + << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + /* Load Hartree potential basis matrix */ + std::string basis_file = rom_options.basis_file; + const int num_pot_basis = rom_options.num_potbasis; + CAROM::BasisReader basis_reader(basis_file); + CAROM::Matrix* pot_basis = basis_reader.getSpatialBasis(num_pot_basis); + + /* initialize rom operator variables */ + CAROM::Matrix pot_rom(num_pot_basis, num_pot_basis, false); + CAROM::Matrix pot_rom_inv(num_pot_basis, num_pot_basis, false); + CAROM::Matrix pot_rhs_rom(num_pot_basis, num_pot_basis, false); + CAROM::Vector pot_rhs_rescaler(num_pot_basis, false); + + /* Load ROM operator */ + // read the file from PE0 only + if (MPIdata::onpe0) + { + std::string rom_oper = rom_options.pot_rom_file; + CAROM::HDFDatabase h5_helper; + h5_helper.open(rom_oper, "r"); + int num_pot_basis_file = -1; + h5_helper.getInteger("number_of_potential_basis", num_pot_basis_file); + CAROM_VERIFY(num_pot_basis_file == num_pot_basis); + + h5_helper.getDoubleArray("potential_rom_operator", pot_rom.getData(), + num_pot_basis * num_pot_basis, false); + + /* load the inverse as well */ + h5_helper.getDoubleArray("potential_rom_inverse", pot_rom_inv.getData(), + num_pot_basis * num_pot_basis, false); + + /* load right-hand side hyper-reduction operator */ + h5_helper.getDoubleArray("potential_rhs_rom_inverse", + pot_rhs_rom.getData(), num_pot_basis * num_pot_basis, false); + + /* load right-hand side rescaling operator */ + h5_helper.getDoubleArray("potential_rhs_rescaler", + pot_rhs_rescaler.getData(), num_pot_basis, false); + + h5_helper.close(); + } +} + +/* test routines */ + +template +void testROMPoissonOperator(MGmolInterface* mgmol_) +{ + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + + ROMPrivateOptions rom_options = ct.getROMOptions(); + + /* Load MGmol pointer and Potentials */ + MGmol* mgmol = static_cast*>(mgmol_); + Poisson* poisson = mgmol->electrostat_->getPoissonSolver(); + Potentials& pot = mgmol->getHamiltonian()->potential(); + const int dim = pot.size(); + printf("pot size: %d\n", dim); + + /* GridFunc initialization inputs */ + const pb::Grid& grid(poisson->vh().grid()); + short bc[3]; + for (int d = 0; d < 3; d++) + bc[d] = poisson->vh().bc(d); + + /* fictitious snapshot numbers */ + const int nsnapshot = 3; + + /* Set compensating charges to zero now */ + pb::GridFunc rhoc(grid, bc[0], bc[1], bc[2]); + rhoc = 0.0; + + /* Generate fictitious right-hand sides and snapshots */ + std::vector> rhs(nsnapshot), fom_sol(nsnapshot); + for (int s = 0; s < nsnapshot; s++) + { + rhs[s].resize(dim); + for (int d = 0; d < dim; d++) + rhs[s][d] = ran0(); + + /* average out for periodic bc */ + pb::GridFunc rhs_gf(grid, bc[0], bc[1], bc[2]); + rhs_gf.assign(rhs[s].data(), 'd'); + double avg = rhs_gf.get_average(); + rhs_gf -= avg; + + /* copy back to rhs */ + rhs_gf.init_vect(rhs[s].data(), 'd'); + + poisson->solve(rhs_gf, rhoc); + + fom_sol[s].resize(dim); + poisson->vh().init_vect(fom_sol[s].data(), 'd'); + + /* check if the solution is correct */ + pb::GridFunc res(grid, bc[0], bc[1], bc[2]); + pb::GridFunc sol_gf(grid, bc[0], bc[1], bc[2]); + sol_gf.assign(fom_sol[s].data()); + /* apply Laplace operator */ + poisson->applyOperator(sol_gf, res); + /* FD operator scales rhs by 4pi */ + res.axpy(-4. * M_PI, rhs_gf); + printf("FOM res norm: %.3e\n", res.norm2()); + } + + /* Initialize libROM classes */ + std::string basis_prefix = "test_poisson"; + CAROM::Options svd_options(dim, nsnapshot, 1); + CAROM::BasisGenerator basis_generator(svd_options, false, basis_prefix); + + /* Collect snapshots and train POD basis */ + for (int s = 0; s < nsnapshot; s++) + basis_generator.takeSample(fom_sol[s].data()); + basis_generator.endSamples(); + + /* Load POD basis. We use the maximum number of basis vectors. */ + const CAROM::Matrix* pot_basis = basis_generator.getSpatialBasis(); + + /* Check if full projection preserves FOM solution */ + for (int c = 0; c < nsnapshot; c++) + { + CAROM::Vector* fom_sol_vec = nullptr; + /* get librom view-vector of fom_sol */ + fom_sol_vec = new CAROM::Vector( + fom_sol[c].data(), pot_basis->numRows(), true, false); + + CAROM::Vector* rom_proj = pot_basis->transposeMult(*fom_sol_vec); + CAROM::Vector* reconstruct = pot_basis->mult(*rom_proj); + + /* error on libROM side */ + CAROM::Vector* librom_error = reconstruct->minus(fom_sol_vec); + printf("librom reconstruction error: %.3e\n", librom_error->norm()); + + /* error on mgmol side */ + pb::GridFunc recon_gf(grid, bc[0], bc[1], bc[2]); + pb::GridFunc fom_gf(grid, bc[0], bc[1], bc[2]); + recon_gf.assign(reconstruct->getData(), 'd'); + fom_gf.assign(fom_sol[c].data(), 'd'); + recon_gf -= fom_gf; + printf("mgmol reconstruction error: %.3e\n", recon_gf.norm2()); + + delete fom_sol_vec; + delete rom_proj; + delete reconstruct; + delete librom_error; + } + + /* Check FOM axpy is equivalent to ROM axpy */ + for (int s = 0; s < nsnapshot; s++) + { + CAROM::Vector fom_res(pot_basis->numRows(), true); + CAROM::Vector rom_res(nsnapshot, false); + CAROM::Vector fom_rhs(pot_basis->numRows(), true); + CAROM::Vector rom_rhs(nsnapshot, false); + + pb::GridFunc res(grid, bc[0], bc[1], bc[2]); + pb::GridFunc fomsol_gf(grid, bc[0], bc[1], bc[2]); + pb::GridFunc mgmol_rhs(grid, bc[0], bc[1], bc[2]); + fomsol_gf.assign(fom_sol[s].data(), 'd'); + mgmol_rhs.assign(rhs[s].data(), 'd'); + + /* apply Laplace operator */ + poisson->applyOperator(fomsol_gf, res); + + /* get librom view-vector of fom_res */ + res.init_vect(fom_res.getData(), 'd'); + pot_basis->transposeMult(fom_res, rom_res); + + /* get librom view-vector of fom_rhs */ + mgmol_rhs.init_vect(fom_rhs.getData(), 'd'); + pot_basis->transposeMult(fom_rhs, rom_rhs); + + /* ROM residual: FD operator scales rhs by 4pi */ + rom_rhs *= 4. * M_PI; + rom_res -= rom_rhs; + printf("ROM res norm: %.3e\n", rom_res.norm()); + + /* FOM residual: FD operator scales rhs by 4pi */ + res.axpy(-4. * M_PI, mgmol_rhs); + printf("FOM res norm: %.3e\n", res.norm2()); + + /* projection of the residual */ + res.init_vect(fom_res.getData(), 'd'); + CAROM::Vector* res_proj = pot_basis->transposeMult(fom_res); + printf("FOM res projection norm: %.3e\n", res_proj->norm()); + + delete res_proj; + } + + /* Initialize Projection ROM matrix (undistributed) */ + CAROM::Matrix pot_rom(nsnapshot, nsnapshot, false); + + /* Build Projection of Poisson operator */ + for (int c = 0; c < nsnapshot; c++) + { + pb::GridFunc col_gf(grid, bc[0], bc[1], bc[2]); + pb::GridFunc opcol_gf(grid, bc[0], bc[1], bc[2]); + CAROM::Vector op_col(pot_basis->numRows(), true); + + /* copy c-th column librom vector to GridFunc gf_col */ + CAROM::Vector* col = pot_basis->getColumn(c); + col_gf.assign(col->getData(), 'd'); + + /* apply Laplace operator */ + poisson->applyOperator(col_gf, opcol_gf); + + /* get librom view-vector of gf_opcol */ + opcol_gf.init_vect(op_col.getData(), 'd'); + + /* Compute basis projection of the column */ + /* Resulting vector is undistributed */ + CAROM::Vector* rom_col = pot_basis->transposeMult(op_col); + + /* libROM matrix is row-major, so data copy is necessary */ + for (int r = 0; r < nsnapshot; r++) + pot_rom(r, c) = (*rom_col)(r); + + delete col; + delete rom_col; + } // for (int c = 0; c < num_pot_basis; c++) + + /* Inverse of the projection ROM matrix */ + CAROM::Matrix pot_rom_inv(pot_rom); + pot_rom_inv.inverse(); + + /* Check the inverse */ + CAROM::Matrix* identity = pot_rom_inv.mult(pot_rom); + printf("pot_rom_inv * pot_rom = identity\n"); + for (int i = 0; i < nsnapshot; i++) + { + for (int j = 0; j < nsnapshot; j++) + printf("%.3e\t", identity->item(i, j)); + printf("\n"); + } + delete identity; + + /* Test with sample RHS. ROM must be able to 100% reproduce the FOM + * solution. */ + std::vector rom_sol(0), rom_rhs(0); + std::vector> test_sol(nsnapshot); + for (int s = 0; s < nsnapshot; s++) + { + /* get librom view-vector of rhs[s] */ + CAROM::Vector fom_rhs(rhs[s].data(), dim, true, false); + + /* project onto POD basis */ + rom_rhs.push_back(pot_basis->transposeMult(fom_rhs)); + + /* FOM FD operator scales rhs by 4pi */ + *rom_rhs.back() *= 4. * M_PI; + + /* solve ROM */ + rom_sol.push_back(pot_rom_inv.mult(*rom_rhs.back())); + + /* check ROM solution */ + CAROM::Vector& res(*pot_rom.mult(*rom_sol.back())); + res -= *rom_rhs.back(); + printf("rom res norm: %.3e\n", res.norm()); + + /* initialize lift-up FOM solution */ + test_sol[s].resize(dim); + /* get librom view-vector of test_sol[s] */ + CAROM::Vector test_sol_vec(test_sol[s].data(), dim, true, false); + pot_basis->mult(*rom_sol.back(), test_sol_vec); + } + + /* Compute relative errors */ + for (int s = 0; s < nsnapshot; s++) + { + pb::GridFunc testsol_gf(grid, bc[0], bc[1], bc[2]); + pb::GridFunc fomsol_gf(grid, bc[0], bc[1], bc[2]); + + testsol_gf.assign(test_sol[s].data(), 'd'); + fomsol_gf.assign(fom_sol[s].data(), 'd'); + + testsol_gf -= fomsol_gf; + double rel_error = testsol_gf.norm2() / fomsol_gf.norm2(); + printf("%d-th sample relative error: %.3e\n", s, rel_error); + + if (rel_error > 1.0e-9) abort(); + } + + /* clean up pointers */ + for (int s = 0; s < nsnapshot; s++) + { + delete rom_sol[s]; + delete rom_rhs[s]; + } +} + +template +void testROMRhoOperator(MGmolInterface* mgmol_) +{ + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const int subdivx = mymesh->subdivx(); + const pb::PEenv& myPEenv = mymesh->peenv(); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const int rank = mmpi.mypeGlobal(); + const int nprocs = mmpi.size(); + + // if (ct.isLocMode()) + // printf("LocMode is On!\n"); + // else + // printf("LocMode is Off!\n"); + + ROMPrivateOptions rom_options = ct.getROMOptions(); + + /* Load MGmol pointer and Potentials */ + MGmol* mgmol = static_cast*>(mgmol_); + Poisson* poisson = mgmol->electrostat_->getPoissonSolver(); + Potentials& pot = mgmol->getHamiltonian()->potential(); + std::shared_ptr> rho = NULL; // mgmol->getRho(); + const OrthoType ortho_type = rho->getOrthoType(); + assert(ortho_type == OrthoType::Nonorthogonal); + + /* potential should have the same size as rho */ + const int dim = pot.size(); + + /* number of restart files, start/end indices */ + assert(rom_options.restart_file_minidx >= 0); + assert(rom_options.restart_file_maxidx >= 0); + const int minidx = rom_options.restart_file_minidx; + const int maxidx = rom_options.restart_file_maxidx; + const int num_restart = maxidx - minidx + 1; + + /* Initialize libROM classes */ + /* + In practice, we do not produce rho POD basis. + This rho POD basis is for the sake of verification. + */ + CAROM::Options svd_options(dim, num_restart, 1); + svd_options.static_svd_preserve_snapshot = true; + CAROM::BasisGenerator basis_generator(svd_options, false, "test_rho"); + + /* Collect the restart files */ + std::string filename; + for (int k = minidx; k <= maxidx; k++) + { + filename = string_format(rom_options.restart_file_fmt, k); + mgmol->loadRestartFile(filename); + basis_generator.takeSample(&rho->rho_[0][0]); + } + // basis_generator.writeSnapshot(); + const CAROM::Matrix rho_snapshots(*basis_generator.getSnapshotMatrix()); + basis_generator.endSamples(); + + const CAROM::Matrix* rho_basis = basis_generator.getSpatialBasis(); + CAROM::Matrix* proj_rho = rho_basis->transposeMult(rho_snapshots); + + /* DEIM hyperreduction */ + CAROM::Matrix rho_basis_inv(num_restart, num_restart, false); + std::vector global_sampled_row(num_restart), + sampled_rows_per_proc(nprocs); + DEIM(rho_basis, num_restart, global_sampled_row, sampled_rows_per_proc, + rho_basis_inv, rank, nprocs); + if (rank == 0) + { + int num_sample_rows = 0; + for (int k = 0; k < sampled_rows_per_proc.size(); k++) + num_sample_rows += sampled_rows_per_proc[k]; + printf("number of sampled row: %d\n", num_sample_rows); + } + + /* get local sampled row */ + std::vector offsets, sampled_row(sampled_rows_per_proc[rank]); + int num_global_sample + = CAROM::get_global_offsets(sampled_rows_per_proc[rank], offsets); + for (int s = 0, gs = offsets[rank]; gs < offsets[rank + 1]; gs++, s++) + sampled_row[s] = global_sampled_row[gs]; + + /* load only the first restart file for now */ + const int test_idx = 2; + + filename = string_format(rom_options.restart_file_fmt, test_idx + minidx); + /* + currently, this does not update rho. + computeRhoOnSamplePts computes with the new density matrix, + while mgmol rho remains the same as the initial condition. + Commenting line 516 gives a consistent result, both for the initial + condition. + */ + mgmol->loadRestartFile(filename); + + const int nrows = mymesh->locNumpt(); + // printf("mesh::locNumpt: %d\n", nrows); + + OrbitalsType* orbitals = mgmol->getOrbitals(); + // printf("orbitals::locNumpt: %d\n", orbitals->getLocNumpt()); + + /* NOTE(kevin): we assume we only use ProjectedMatrices class */ + // ProjectedMatrices> *proj_matrices = + // static_cast> + // *>(orbitals->getProjMatrices()); + ProjectedMatricesInterface* proj_matrices = orbitals->getProjMatrices(); + proj_matrices->updateSubMatX(); + SquareLocalMatrices& localX( + proj_matrices->getLocalX()); + + // printf("localX nmat: %d\n", localX.nmat()); + // printf("localX n: %d\n", localX.n()); + // printf("localX m: %d\n", localX.m()); + + bool dm_distributed = (localX.nmat() > 1); + assert(!dm_distributed); + + /* copy density matrix */ + CAROM::Matrix dm( + localX.getRawPtr(), localX.m(), localX.n(), dm_distributed, true); + + // /* random density matrix */ + // /* NOTE(kevin): Due to rescaleTotalCharge, the result is slightly + // inconsistent. */ CAROM::Matrix dm(localX.m(), localX.n(), dm_distributed, + // true); dm = 0.0; dist_matrix::DistMatrix + // dm_mgmol(proj_matrices->dm()); for (int i = 0; i < dm_mgmol.m(); i++) + // { + // for (int j = 0; j < dm_mgmol.m(); j++) + // { + // if (dm_mgmol.getVal(i, j) == 0.0) + // continue; + + // dm(i, j) = dm_mgmol.getVal(i, j) * (0.975 + 0.05 * ran0()); + // dm_mgmol.setVal(i, j, dm(i, j)); + // } + // } + + /* update rho first */ + // rho->computeRho(*orbitals, dm_mgmol); + // rho->update(*orbitals); + + const int chrom_num = orbitals->chromatic_number(); + CAROM::Matrix psi(dim, chrom_num, true); + for (int c = 0; c < chrom_num; c++) + { + ORBDTYPE* d_psi = orbitals->getPsi(c); + for (int d = 0; d < dim; d++) + psi.item(d, c) = *(d_psi + d); + } + + CAROM::Matrix rom_psi(chrom_num, chrom_num, false); + for (int i = 0; i < chrom_num; i++) + for (int j = 0; j < chrom_num; j++) + rom_psi(i, j) = (i == j) ? 1 : 0; + + /* this will be resized in computeRhoOnSamplePts */ + CAROM::Vector sample_rho(1, true); + + computeRhoOnSamplePts(dm, psi, rom_psi, sampled_row, sample_rho); + + for (int s = 0; s < sampled_row.size(); s++) + { + const double error = abs(rho->rho_[0][sampled_row[s]] - sample_rho(s)); + if (error > 1.0e-4) + printf("rank %d, rho[%d]: %.5e, sample_rho: %.5e, librom_snapshot: " + "%.5e\n", + rank, sampled_row[s], rho->rho_[0][sampled_row[s]], + sample_rho(s), rho_snapshots(sampled_row[s], test_idx)); + CAROM_VERIFY(error < 1.0e-4); + } + + sample_rho.gather(); + + CAROM::Vector* rom_rho = rho_basis_inv.mult(sample_rho); + for (int d = 0; d < rom_rho->dim(); d++) + { + if ((rank == 0) + && (abs(proj_rho->item(d, test_idx) - rom_rho->item(d)) > 1.0e-3)) + printf("rom_rho error: %.3e\n", + abs(proj_rho->item(d, test_idx) - rom_rho->item(d))); + CAROM_VERIFY( + abs(proj_rho->item(d, test_idx) - rom_rho->item(d)) < 1.0e-3); + } + + CAROM::Vector* fom_rho = rho_basis->mult(*rom_rho); + + CAROM_VERIFY(fom_rho->dim() == rho->rho_[0].size()); + for (int d = 0; d < fom_rho->dim(); d++) + CAROM_VERIFY(abs(fom_rho->item(d) - rho->rho_[0][d]) < 1.0e-4); + + delete rom_rho; + delete fom_rho; +} + +/* + dm: density matrix converted to CAROM::Matrix + phi_basis: POD basis matrix of orbitals, or orbitals themselves + rom_phi: ROM coefficients of POD Basis. If phi_basis is orbitals themselves, + then simply an identity. local_idx: sampled local grid indices on this + processor. sampled_rho: FOM density on sampled grid points. +*/ +void computeRhoOnSamplePts(const CAROM::Matrix& dm, + const CAROM::Matrix& phi_basis, const CAROM::Matrix& rom_phi, + const std::vector& local_idx, CAROM::Vector& sampled_rho) +{ + assert(!dm.distributed()); + assert(sampled_rho.distributed() == phi_basis.distributed()); + + // this will be resized. + CAROM::Matrix sampled_phi(1, 1, phi_basis.distributed()); + + CAROM::mult(phi_basis, local_idx, rom_phi, sampled_phi); + + /* same product as in computeRhoSubdomainUsingBlas3 */ + CAROM::Matrix product(1, 1, sampled_phi.distributed()); + sampled_phi.mult(dm, product); + + sampled_rho.setSize(sampled_phi.numRows()); + double* d_product = product.getData(); + double* d_phi = sampled_phi.getData(); + for (int d = 0; d < sampled_rho.dim(); d++) + { + double val = 0.0; + /* CAROM Matrices are row-major */ + for (int c = 0; c < sampled_phi.numColumns(); c++, d_product++, d_phi++) + val += (*d_product) * (*d_phi); + + sampled_rho(d) = val; + } + + /* + TODO(kevin): need to figure out what these functions do. + and probably need to make ROM-equivalent functions with another + hyper-reduction? + */ + // gatherSpin(); + + /* + rescaleTotalCharge is handled after hyperreduction. + */ + // rescaleTotalCharge(); +} + +template +void testROMIonDensity(MGmolInterface* mgmol_) +{ + /* random number generator */ + static std::random_device + rd; // Will be used to obtain a seed for the random number engine + static std::mt19937 gen( + rd()); // Standard mersenne_twister_engine seeded with rd(){} + static std::uniform_real_distribution<> dis(0.0, 1.0); + + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + const int subdivx = mymesh->subdivx(); + const pb::PEenv& myPEenv = mymesh->peenv(); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + const int rank = mmpi.mypeGlobal(); + const int nprocs = mmpi.size(); + + ROMPrivateOptions rom_options = ct.getROMOptions(); + + /* Load MGmol pointer and Potentials */ + MGmol* mgmol = static_cast*>(mgmol_); + Poisson* poisson = mgmol->electrostat_->getPoissonSolver(); + Potentials& pot = mgmol->getHamiltonian()->potential(); + const int dim = pot.size(); + std::shared_ptr ions = mgmol->getIons(); + + /* get the extent of global domain */ + const double origin[3] + = { mygrid.origin(0), mygrid.origin(1), mygrid.origin(2) }; + const double lattice[3] = { mygrid.ll(0), mygrid.ll(1), mygrid.ll(2) }; + if (rank == 0) + { + printf("origin: (%.3e, %.3e, %.3e)\n", origin[0], origin[1], origin[2]); + printf("lattice: (%.3e, %.3e, %.3e)\n", lattice[0], lattice[1], + lattice[2]); + } + + /* get global atomic numbers */ + const int num_ions = ions->getNumIons(); + std::vector atnumbers(num_ions); + ions->getAtomicNumbers(atnumbers); + + assert(!(mgmol->electrostat_->isDielectric())); + assert(pot.getBackgroundCharge() <= 0.0); + + const int num_snap = 3; + + /* 3 fictitious ion configurations */ + std::vector> cfgs(num_snap); + for (int idx = 0; idx < num_snap; idx++) + { + cfgs[idx].resize(3 * num_ions); + if (rank == 0) + for (int k = 0; k < num_ions; k++) + for (int d = 0; d < 3; d++) + cfgs[idx][3 * k + d] = origin[d] + lattice[d] * dis(gen); + + mmpi.bcastGlobal(cfgs[idx].data(), 3 * num_ions, 0); + } + + /* Artificial ions object to avoid repeated setPositions */ + Ions* new_ions; + /* Collect fictitious ion density based on each configuration */ + std::vector> fom_rhoc(num_snap); + /* Sanity check for overlappingVL_ions */ + std::vector>> fom_overlap_ions(num_snap); + for (int idx = 0; idx < num_snap; idx++) + { + /* set ion positions */ + new_ions = new Ions(cfgs[idx], atnumbers, lattice, ions->getSpecies()); + + /* save overlapping ions for sanity check */ + fom_overlap_ions[idx].resize(new_ions->overlappingVL_ions().size()); + for (int k = 0; k < new_ions->overlappingVL_ions().size(); k++) + { + fom_overlap_ions[idx][k].resize(3); + for (int d = 0; d < 3; d++) + fom_overlap_ions[idx][k][d] + = new_ions->overlappingVL_ions()[k]->position(d); + } + + /* compute resulting ion density */ + /* NOTE: we exclude rescaling for the sake of verification */ + mgmol->setupPotentials(*new_ions); + + fom_rhoc[idx].resize(dim); + mgmol->electrostat_->getRhoc()->init_vect(fom_rhoc[idx].data(), 'd'); + + delete new_ions; + } + + /* Initialize libROM classes */ + /* + In practice, we do not produce rhoc POD basis. + This rhoc POD basis is for the sake of verification. + */ + CAROM::Options svd_options(dim, num_snap, 1); + svd_options.static_svd_preserve_snapshot = true; + CAROM::BasisGenerator basis_generator(svd_options, false, "test_rhoc"); + for (int idx = 0; idx < num_snap; idx++) + basis_generator.takeSample(&fom_rhoc[idx][0]); + + const CAROM::Matrix rhoc_snapshots(*basis_generator.getSnapshotMatrix()); + basis_generator.endSamples(); + + const CAROM::Matrix* rhoc_basis = basis_generator.getSpatialBasis(); + CAROM::Matrix* proj_rhoc = rhoc_basis->transposeMult(rhoc_snapshots); + + /* DEIM hyperreduction */ + CAROM::Matrix rhoc_basis_inv(num_snap, num_snap, false); + std::vector global_sampled_row(num_snap), + sampled_rows_per_proc(nprocs); + DEIM(rhoc_basis, num_snap, global_sampled_row, sampled_rows_per_proc, + rhoc_basis_inv, rank, nprocs); + if (rank == 0) + { + int num_sample_rows = 0; + for (int k = 0; k < sampled_rows_per_proc.size(); k++) + num_sample_rows += sampled_rows_per_proc[k]; + printf("number of sampled row: %d\n", num_sample_rows); + } + + /* get local sampled row */ + std::vector offsets, sampled_row(sampled_rows_per_proc[rank]); + int num_global_sample + = CAROM::get_global_offsets(sampled_rows_per_proc[rank], offsets); + for (int s = 0, gs = offsets[rank]; gs < offsets[rank + 1]; gs++, s++) + sampled_row[s] = global_sampled_row[gs]; + + /* test one solution */ + std::uniform_int_distribution<> distrib(0, num_snap - 1); + int test_idx = distrib(gen); + mmpi.bcastGlobal(&test_idx); + if (rank == 0) printf("test index: %d\n", test_idx); + + /* set ion positions */ + new_ions = new Ions(cfgs[test_idx], atnumbers, lattice, ions->getSpecies()); + + /* Sanity check for overlapping ions */ + CAROM_VERIFY(fom_overlap_ions[test_idx].size() + == new_ions->overlappingVL_ions().size()); + for (int k = 0; k < new_ions->overlappingVL_ions().size(); k++) + for (int d = 0; d < 3; d++) + CAROM_VERIFY(abs(fom_overlap_ions[test_idx][k][d] + - new_ions->overlappingVL_ions()[k]->position(d)) + < 1.0e-12); + + /* set up potentials */ + mgmol->setupPotentials(*new_ions); + + /* eval ion density on sample grid points */ + std::vector sampled_rhoc(sampled_row.size()); + pot.evalIonDensityOnSamplePts(*new_ions, sampled_row, sampled_rhoc); + + // For now, we relax the threshold to allow the slight difference of the + // values at the sampled indices. This is because the rescaling procedure + // after getting the radial data on mesh requires global information, thus + // cannot be done in the ROM level only with sampled values. After we have + // the DEIM reconstruction, we will do rescaling and revisit this. + for (int d = 0; d < sampled_row.size(); d++) + { + printf("rank %d, fom rhoc[%d]: %.3e, rom rhoc: %.3e\n", rank, + sampled_row[d], fom_rhoc[test_idx][sampled_row[d]], + sampled_rhoc[d]); + CAROM_VERIFY( + abs(fom_rhoc[test_idx][sampled_row[d]] - sampled_rhoc[d]) < 1.0e-2); + } + + delete new_ions; +} + +template void readRestartFiles>( + MGmolInterface* mgmol_); +template void readRestartFiles>( + MGmolInterface* mgmol_); + +template void buildROMPoissonOperator>( + MGmolInterface* mgmol_); +template void buildROMPoissonOperator>( + MGmolInterface* mgmol_); + +template void runPoissonROM>(MGmolInterface* mgmol_); +template void runPoissonROM>( + MGmolInterface* mgmol_); + +template void testROMPoissonOperator>( + MGmolInterface* mgmol_); +template void testROMPoissonOperator>( + MGmolInterface* mgmol_); + +template void testROMRhoOperator>( + MGmolInterface* mgmol_); +template void testROMRhoOperator>( + MGmolInterface* mgmol_); + +template void testROMIonDensity>( + MGmolInterface* mgmol_); +template void testROMIonDensity>( + MGmolInterface* mgmol_); diff --git a/src/rom_workflows.h b/src/rom_workflows.h new file mode 100644 index 00000000..3dcaf9ba --- /dev/null +++ b/src/rom_workflows.h @@ -0,0 +1,64 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#ifndef ROM_WORKFLOWS_H +#define ROM_WORKFLOWS_H + +#include "Control.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "Mesh.h" +#include "Potentials.h" +#include "ProjectedMatrices.h" +#include "mgmol_run.h" +#include "tools.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +namespace po = boost::program_options; + +#include "librom.h" +#include "utils/HDFDatabase.h" +#include "utils/mpi_utils.h" + +template +void readRestartFiles(MGmolInterface* mgmol_); + +template +void buildROMPoissonOperator(MGmolInterface* mgmol_); + +template +void runPoissonROM(MGmolInterface* mgmol_); + +template +void testROMPoissonOperator(MGmolInterface* mgmol_); + +template +void testROMRhoOperator(MGmolInterface* mgmol_); + +template +void testROMIonDensity(MGmolInterface* mgmol_); + +void computeRhoOnSamplePts(const CAROM::Matrix& dm, + const CAROM::Matrix& phi_basis, const CAROM::Matrix& rom_phi, + const std::vector& local_idx, CAROM::Vector& sampled_rho); + +#endif // ROM_WORKFLOWS_H diff --git a/src/setup.cc b/src/setup.cc index 32e9ef37..c7b67c60 100644 --- a/src/setup.cc +++ b/src/setup.cc @@ -14,9 +14,7 @@ #include "MGmol.h" #include "Potentials.h" #include "ReplicatedWorkSpace.h" -#ifdef MGMOL_USE_SCALAPACK #include "SparseDistMatrix.h" -#endif #include "mgmol_run.h" @@ -86,7 +84,6 @@ int MGmol::setupFromInput(const std::string filename) { ReplicatedWorkSpace::instance().setup(ct.numst); -#ifdef MGMOL_USE_SCALAPACK if (!ct.rmatrices) { MatricesBlacsContext::instance().setup(mmpi.commSpin(), ct.numst); @@ -102,7 +99,6 @@ int MGmol::setupFromInput(const std::string filename) int npes = mmpi.size(); setSparseDistMatriConsolidationNumber(npes); } -#endif } if (ct.rmatrices) ReplicatedMatrix::setMPIcomm(mmpi.commSpin()); diff --git a/src/tools.cc b/src/tools.cc index 8dd0f403..aff34933 100644 --- a/src/tools.cc +++ b/src/tools.cc @@ -12,11 +12,8 @@ #include "MGmol_MPI.h" #include "MPIdata.h" #include "Mesh.h" -#include "Vector3D.h" - -#ifdef MGMOL_USE_SCALAPACK #include "SparseDistMatrix.h" -#endif +#include "Vector3D.h" #include #include @@ -106,7 +103,6 @@ void read_comments(std::ifstream& tfile) } } -#ifdef MGMOL_USE_SCALAPACK void setSparseDistMatriConsolidationNumber(const int npes) { int consolidation_number = 9; @@ -135,7 +131,6 @@ void setSparseDistMatriConsolidationNumber(const int npes) dist_matrix::SparseDistMatrix::printConsolidationNumber( *MPIdata::sout); } -#endif void reduceBytes(std::vector& val, const MPI_Comm comm) { diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index baae6cda..ca187a0e 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -8,6 +8,7 @@ set(SOURCES mgmol_mpi_tools.cc random.cc coloring.cc SymmetricMatrix.cc + PinnedH2O.cc ) add_library(mgmol_tools ${SOURCES}) diff --git a/src/tools/PinnedH2O.cc b/src/tools/PinnedH2O.cc new file mode 100644 index 00000000..540babb8 --- /dev/null +++ b/src/tools/PinnedH2O.cc @@ -0,0 +1,273 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "PinnedH2O.h" +#include + +PinnedH2O::PinnedH2O() + : flipped_bond(false), + O1_idx(-1), + H1_idx(-1), + H2_idx(-1), + planar_rotation_angle(0.0) +{ + for (int i = 0; i < 3; ++i) + { + for (int j = 0; j < 3; ++j) + { + out_of_plane_rotation_matrix[i][j] = 0.0; + } + } +} + +double PinnedH2O::calculate_bondlength( + const double atom1[3], const double atom2[3]) const +{ + return sqrt(pow(atom1[0] - atom2[0], 2) + pow(atom1[1] - atom2[1], 2) + + pow(atom1[2] - atom2[2], 2)); +} + +double PinnedH2O::calculate_bondangle( + const double atom1[3], const double atom2[3], const double atom3[3]) const +{ + double vector1[3] + = { atom1[0] - atom2[0], atom1[1] - atom2[1], atom1[2] - atom2[2] }; + double vector2[3] + = { atom3[0] - atom2[0], atom3[1] - atom2[1], atom3[2] - atom2[2] }; + + double dot_product = vector1[0] * vector2[0] + vector1[1] * vector2[1] + + vector1[2] * vector2[2]; + double magnitude_product + = sqrt(pow(vector1[0], 2) + pow(vector1[1], 2) + pow(vector1[2], 2)) + * sqrt(pow(vector2[0], 2) + pow(vector2[1], 2) + pow(vector2[2], 2)); + double angle = acos(dot_product / magnitude_product); + + return angle; +} + +void PinnedH2O::rotation_matrix( + const double axis[3], double angle, double matrix[3][3]) const +{ + double cos_theta = cos(angle); + double sin_theta = sin(angle); + double ux = axis[0], uy = axis[1], uz = axis[2]; + + matrix[0][0] = cos_theta + ux * ux * (1 - cos_theta); + matrix[0][1] = ux * uy * (1 - cos_theta) - uz * sin_theta; + matrix[0][2] = ux * uz * (1 - cos_theta) + uy * sin_theta; + + matrix[1][0] = uy * ux * (1 - cos_theta) + uz * sin_theta; + matrix[1][1] = cos_theta + uy * uy * (1 - cos_theta); + matrix[1][2] = uy * uz * (1 - cos_theta) - ux * sin_theta; + + matrix[2][0] = uz * ux * (1 - cos_theta) - uy * sin_theta; + matrix[2][1] = uz * uy * (1 - cos_theta) + ux * sin_theta; + matrix[2][2] = cos_theta + uz * uz * (1 - cos_theta); +} + +void PinnedH2O::normalize(double vec[3]) const +{ + double norm = sqrt(vec[0] * vec[0] + vec[1] * vec[1] + vec[2] * vec[2]); + vec[0] /= norm; + vec[1] /= norm; + vec[2] /= norm; +} + +void PinnedH2O::cross( + const double a[3], const double b[3], double result[3]) const +{ + result[0] = a[1] * b[2] - a[2] * b[1]; + result[1] = a[2] * b[0] - a[0] * b[2]; + result[2] = a[0] * b[1] - a[1] * b[0]; +} + +void PinnedH2O::apply_rotation( + const double matrix[3][3], const double vec[3], double result[3]) const +{ + result[0] + = matrix[0][0] * vec[0] + matrix[0][1] * vec[1] + matrix[0][2] * vec[2]; + result[1] + = matrix[1][0] * vec[0] + matrix[1][1] * vec[1] + matrix[1][2] * vec[2]; + result[2] + = matrix[2][0] * vec[0] + matrix[2][1] * vec[1] + matrix[2][2] * vec[2]; +} + +void PinnedH2O::apply_transpose_rotation( + const double matrix[3][3], const double vec[3], double result[3]) const +{ + result[0] + = matrix[0][0] * vec[0] + matrix[1][0] * vec[1] + matrix[2][0] * vec[2]; + result[1] + = matrix[0][1] * vec[0] + matrix[1][1] * vec[1] + matrix[2][1] * vec[2]; + result[2] + = matrix[0][2] * vec[0] + matrix[1][2] * vec[1] + matrix[2][2] * vec[2]; +} + +void PinnedH2O::rotate( + std::vector& positions, std::vector& anumbers) +{ + for (int i = 0; i < 3; i++) + { + if (positions[3 * i] == 0.0 && positions[3 * i + 1] == 0.0 + && positions[3 * i + 2] == 0.0) + { + O1_idx = i; + break; + } + } + if (O1_idx == -1) return; + + H1_idx = (O1_idx + 1) % 3; + H2_idx = (O1_idx + 2) % 3; + + double O1[3] = { positions[3 * O1_idx], positions[3 * O1_idx + 1], + positions[3 * O1_idx + 2] }; + double H1[3] = { positions[3 * H1_idx], positions[3 * H1_idx + 1], + positions[3 * H1_idx + 2] }; + double H2[3] = { positions[3 * H2_idx], positions[3 * H2_idx + 1], + positions[3 * H2_idx + 2] }; + + bondlength1 = calculate_bondlength(H1, O1); + bondlength2 = calculate_bondlength(H2, O1); + bondangle = calculate_bondangle(H1, O1, H2); + + double H1_temp[3], H2_temp[3]; + double H1_rotated[3], H2_rotated[3]; + + double plane_normal[3]; + cross(H2, H1, plane_normal); + normalize(plane_normal); + double target_plane_normal[3] = { 0, 0, 1 }; + double dot_product = plane_normal[0] * target_plane_normal[0] + + plane_normal[1] * target_plane_normal[1] + + plane_normal[2] * target_plane_normal[2]; + double angle_to_align + = std::acos(std::min(std::max(dot_product, -1.0), 1.0)); + double axis_to_align[3]; + if (abs(dot_product) > 1.0 - 1e-8) + { + axis_to_align[0] = 1.0; + axis_to_align[1] = 0.0; + axis_to_align[2] = 0.0; + } + else + { + cross(plane_normal, target_plane_normal, axis_to_align); + normalize(axis_to_align); + } + + rotation_matrix( + axis_to_align, angle_to_align, out_of_plane_rotation_matrix); + apply_rotation(out_of_plane_rotation_matrix, H1, H1_temp); + apply_rotation(out_of_plane_rotation_matrix, H2, H2_temp); + + double theta1 = std::atan2(H1_temp[1], H1_temp[0]); + planar_rotation_angle = -theta1 + bondangle / 2.0; + double planar_rotation_matrix[3][3]; + rotation_matrix( + target_plane_normal, planar_rotation_angle, planar_rotation_matrix); + apply_rotation(planar_rotation_matrix, H1_temp, H1_rotated); + apply_rotation(planar_rotation_matrix, H2_temp, H2_rotated); + + flipped_bond = (bondlength1 < bondlength2); + if (flipped_bond) + { + H1_rotated[1] *= -1.0; + H2_rotated[1] *= -1.0; + std::swap(H1_rotated, H2_rotated); + std::swap(bondlength1, bondlength2); + } + + positions[0] = H2_rotated[0]; + positions[1] = H2_rotated[1]; + positions[2] = H2_rotated[2]; + positions[3] = 0.0; + positions[4] = 0.0; + positions[5] = 0.0; + positions[6] = H1_rotated[0]; + positions[7] = H1_rotated[1]; + positions[8] = H1_rotated[2]; + + anumbers[0] = 1; + anumbers[1] = 8; + anumbers[2] = 1; +} + +void PinnedH2O::transpose_rotate(std::vector& positions, + std::vector& anumbers, std::vector& forces) +{ + double H2_rotated[3] = { positions[0], positions[1], positions[2] }; + double O1_rotated[3] = { positions[3], positions[4], positions[5] }; + double H1_rotated[3] = { positions[6], positions[7], positions[8] }; + + double f_H2_rotated[3] = { forces[0], forces[1], forces[2] }; + double f_O1_rotated[3] = { forces[3], forces[4], forces[5] }; + double f_H1_rotated[3] = { forces[6], forces[7], forces[8] }; + + if (flipped_bond) + { + H1_rotated[1] *= -1.0; + H2_rotated[1] *= -1.0; + f_O1_rotated[1] *= -1.0; + f_H1_rotated[1] *= -1.0; + f_H2_rotated[1] *= -1.0; + } + + double planar_rotation_matrix[3][3]; + double target_plane_normal[3] = { 0, 0, 1 }; + rotation_matrix( + target_plane_normal, planar_rotation_angle, planar_rotation_matrix); + + double H1_temp[3], H2_temp[3]; + apply_transpose_rotation(planar_rotation_matrix, H1_rotated, H1_temp); + apply_transpose_rotation(planar_rotation_matrix, H2_rotated, H2_temp); + + double f_O1_temp[3], f_H1_temp[3], f_H2_temp[3]; + apply_transpose_rotation(planar_rotation_matrix, f_O1_rotated, f_O1_temp); + apply_transpose_rotation(planar_rotation_matrix, f_H1_rotated, f_H1_temp); + apply_transpose_rotation(planar_rotation_matrix, f_H2_rotated, f_H2_temp); + + double H1_restored[3], H2_restored[3]; + apply_transpose_rotation( + out_of_plane_rotation_matrix, H1_temp, H1_restored); + apply_transpose_rotation( + out_of_plane_rotation_matrix, H2_temp, H2_restored); + + double f_O1_restored[3], f_H1_restored[3], f_H2_restored[3]; + apply_transpose_rotation( + out_of_plane_rotation_matrix, f_O1_temp, f_O1_restored); + apply_transpose_rotation( + out_of_plane_rotation_matrix, f_H1_temp, f_H1_restored); + apply_transpose_rotation( + out_of_plane_rotation_matrix, f_H2_temp, f_H2_restored); + + positions[3 * H2_idx] = H2_restored[0]; + positions[3 * H2_idx + 1] = H2_restored[1]; + positions[3 * H2_idx + 2] = H2_restored[2]; + positions[3 * O1_idx] = 0.0; + positions[3 * O1_idx + 1] = 0.0; + positions[3 * O1_idx + 2] = 0.0; + positions[3 * H1_idx] = H1_restored[0]; + positions[3 * H1_idx + 1] = H1_restored[1]; + positions[3 * H1_idx + 2] = H1_restored[2]; + + anumbers[H2_idx] = 1; + anumbers[O1_idx] = 8; + anumbers[H1_idx] = 1; + + forces[3 * H2_idx] = f_H2_restored[0]; + forces[3 * H2_idx + 1] = f_H2_restored[1]; + forces[3 * H2_idx + 2] = f_H2_restored[2]; + forces[3 * O1_idx] = f_O1_restored[0]; + forces[3 * O1_idx + 1] = f_O1_restored[1]; + forces[3 * O1_idx + 2] = f_O1_restored[2]; + forces[3 * H1_idx] = f_H1_restored[0]; + forces[3 * H1_idx + 1] = f_H1_restored[1]; + forces[3 * H1_idx + 2] = f_H1_restored[2]; +} diff --git a/src/tools/PinnedH2O.h b/src/tools/PinnedH2O.h new file mode 100644 index 00000000..4f8b9ecd --- /dev/null +++ b/src/tools/PinnedH2O.h @@ -0,0 +1,62 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#ifndef PINNED_H2O_H +#define PINNED_H2O_H + +#include +#include +#include +#include + +class PinnedH2O +{ + +public: + PinnedH2O(); + ~PinnedH2O() = default; + + void rotate(std::vector& positions, std::vector& anumbers); + void transpose_rotate(std::vector& positions, + std::vector& anumbers, std::vector& forces); + void print(std::ostream& os) + { + os << "Bondlengths = " << bondlength1 << ", " << bondlength2 + << " Bohrs; " + << "Bondangle = " << bondangle * 180.0 / M_PI << " degrees." + << std::endl; + } + +private: + double calculate_bondlength( + const double atom1[3], const double atom2[3]) const; + double calculate_bondangle(const double atom1[3], const double atom2[3], + const double atom3[3]) const; + void rotation_matrix( + const double axis[3], double angle, double matrix[3][3]) const; + void normalize(double vec[3]) const; + void cross(const double a[3], const double b[3], double result[3]) const; + void apply_rotation( + const double matrix[3][3], const double vec[3], double result[3]) const; + void apply_transpose_rotation( + const double matrix[3][3], const double vec[3], double result[3]) const; + + double bondlength1; + double bondlength2; + double bondangle; + + bool flipped_bond; + int O1_idx; + int H1_idx; + int H2_idx; + double planar_rotation_angle; + double out_of_plane_rotation_matrix[3][3]; +}; + +#endif // PINNED_H2O_H diff --git a/src/tools/SaveData.h b/src/tools/SaveData.h index 5fb61503..ade177a5 100644 --- a/src/tools/SaveData.h +++ b/src/tools/SaveData.h @@ -10,11 +10,9 @@ #ifndef SAVEDATA_H_ #define SAVEDATA_H_ -#ifdef SCALAPACK #include "BlacsContext.h" #include "DistMatrix.h" #include "MatricesBlacsContext.h" -#endif #include #include @@ -71,10 +69,10 @@ void saveData(std::vector data, const char* filename) } } -#ifdef SCALAPACK template void saveData(dist_matrix::DistMatrix data, const char* filename) { + dist_matrix::BlacsContext bc(MPI_COMM_WORLD, 1, 1); std::ofstream outputFile(filename); @@ -83,9 +81,9 @@ void saveData(dist_matrix::DistMatrix data, const char* filename) if (outputFile.is_open()) { + data.print(outputFile); } } -#endif #endif /* SAVEDATA_H_ */ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 906ae297..d6d017b5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -8,7 +8,6 @@ endif() add_executable(testMPI ${CMAKE_SOURCE_DIR}/tests/testMPI.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) -if(${MGMOL_WITH_SCALAPACK}) add_executable(testBlacsContext ${CMAKE_SOURCE_DIR}/tests/testBlacsContext.cc ${CMAKE_SOURCE_DIR}/src/DistMatrix/BlacsContext.cc @@ -115,7 +114,6 @@ add_executable(testPowerDistMatrix ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/src/tools/random.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) -endif(${MGMOL_WITH_SCALAPACK}) add_executable(testDirectionalReduce ${CMAKE_SOURCE_DIR}/tests/testDirectionalReduce.cc ${CMAKE_SOURCE_DIR}/src/sparse_linear_algebra/DirectionalReduce.cc @@ -220,30 +218,6 @@ add_executable(testBatchLaph4 ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) -add_executable(testPoisson - ${CMAKE_SOURCE_DIR}/tests/testPoisson.cc - ${CMAKE_SOURCE_DIR}/src/PCGSolver.cc - ${CMAKE_SOURCE_DIR}/src/pb/Laph4.cc - ${CMAKE_SOURCE_DIR}/src/pb/Lap.cc - ${CMAKE_SOURCE_DIR}/src/pb/Laph2.cc - ${CMAKE_SOURCE_DIR}/src/pb/Laph6.cc - ${CMAKE_SOURCE_DIR}/src/pb/Laph8.cc - ${CMAKE_SOURCE_DIR}/src/pb/Laph4M.cc - ${CMAKE_SOURCE_DIR}/src/pb/FDoper.cc - ${CMAKE_SOURCE_DIR}/src/magma_singleton.cc - ${CMAKE_SOURCE_DIR}/src/pb/Grid.cc - ${CMAKE_SOURCE_DIR}/src/pb/PEenv.cc - ${CMAKE_SOURCE_DIR}/src/pb/GridFunc.cc - ${CMAKE_SOURCE_DIR}/src/Map2Masks.cc - ${CMAKE_SOURCE_DIR}/src/pb/MGkernels.cc - ${CMAKE_SOURCE_DIR}/src/pb/FDkernels.cc - ${CMAKE_SOURCE_DIR}/src/tools/MGmol_MPI.cc - ${CMAKE_SOURCE_DIR}/src/linear_algebra/mputils.cc - ${CMAKE_SOURCE_DIR}/src/linear_algebra/gemm_impl.cc - ${CMAKE_SOURCE_DIR}/src/linear_algebra/syrk_impl.cc - ${CMAKE_SOURCE_DIR}/src/tools/mgmol_mpi_tools.cc - ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc - ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) add_executable(testMGkernels ${CMAKE_SOURCE_DIR}/tests/testMGkernels.cc ${CMAKE_SOURCE_DIR}/src/Map2Masks.cc @@ -262,9 +236,7 @@ add_executable(testMGkernels ${CMAKE_SOURCE_DIR}/src/tools/Timer.cc ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) add_executable(testIons - ${CMAKE_SOURCE_DIR}/tests/testIons.cc - ${CMAKE_SOURCE_DIR}/tests/ut_main.cc) -if(${MGMOL_WITH_SCALAPACK}) + ${CMAKE_SOURCE_DIR}/tests/testIons.cc) add_executable(testGramMatrix ${CMAKE_SOURCE_DIR}/tests/testGramMatrix.cc ${CMAKE_SOURCE_DIR}/src/GramMatrix.cc @@ -310,7 +282,6 @@ add_executable(testDensityMatrix ${CMAKE_SOURCE_DIR}/src/ReplicatedWorkSpace.cc ${CMAKE_SOURCE_DIR}/src/hdf_tools.cc ${CMAKE_SOURCE_DIR}/tests/ut_magma_main.cc) -endif(${MGMOL_WITH_SCALAPACK}) add_executable(testRhoVhRestart ${CMAKE_SOURCE_DIR}/tests/RhoVhRestart/testRhoVhRestart.cc) add_executable(testEnergyAndForces @@ -321,6 +292,9 @@ add_executable(testDMandEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/DMandEnergyAndForces/testDMandEnergyAndForces.cc) add_executable(testRestartEnergyAndForces ${CMAKE_SOURCE_DIR}/tests/RestartEnergyAndForces/testRestartEnergyAndForces.cc) +add_executable(testPinnedH2O_3DOF + ${CMAKE_SOURCE_DIR}/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc + ${CMAKE_SOURCE_DIR}/src/tools/PinnedH2O.cc) if(${MAGMA_FOUND}) add_executable(testOpenmpOffload @@ -355,7 +329,6 @@ endif() add_test(NAME testMPI COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testMPI) -if(${MGMOL_WITH_SCALAPACK}) add_test(NAME testBlacsContext COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testBlacsContext) @@ -374,12 +347,11 @@ add_test(NAME testConditionDistMatrix add_test(NAME testConditionDistMatrixPower COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testConditionDistMatrixPower) +add_test(NAME testPower + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/testPower) add_test(NAME testPowerDistMatrix COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testPowerDistMatrix) -add_test(NAME testPower - COMMAND ${CMAKE_CURRENT_BINARY_DIR}/testPower) -endif(${MGMOL_WITH_SCALAPACK}) add_test(NAME testDirectionalReduce COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testDirectionalReduce) @@ -402,26 +374,19 @@ add_test(NAME testLaph4 add_test(NAME testBatchLaph4 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testBatchLaph4) -add_test(NAME testPoisson - COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} - ${CMAKE_CURRENT_BINARY_DIR}/testPoisson) add_test(NAME testtMGkernels COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testMGkernels) add_test(NAME testIons COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} - ${CMAKE_CURRENT_BINARY_DIR}/testIons) -set_tests_properties(testIons PROPERTIES - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/potentials) - -if(${MGMOL_WITH_SCALAPACK}) + ${CMAKE_CURRENT_BINARY_DIR}/testIons + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME testGramMatrix COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testGramMatrix) add_test(NAME testDensityMatrix COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/testDensityMatrix) -endif(${MGMOL_WITH_SCALAPACK}) add_test(NAME testEnergyAndForces COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/EnergyAndForces/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} @@ -454,6 +419,14 @@ add_test(NAME testRestartEnergyAndForces ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/restart.cfg ${CMAKE_CURRENT_SOURCE_DIR}/RestartEnergyAndForces/h2o.xyz ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) +add_test(NAME testPinnedH2O_3DOF + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/PinnedH2O_3DOF/test.py + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/testPinnedH2O_3DOF + ${CMAKE_CURRENT_SOURCE_DIR}/PinnedH2O_3DOF/mgmol.cfg + ${CMAKE_CURRENT_SOURCE_DIR}/PinnedH2O_3DOF/coords.in + ${CMAKE_CURRENT_SOURCE_DIR}/PinnedH2O_3DOF/lrs.in + ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME testRhoVhRestart COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/RhoVhRestart/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} @@ -557,20 +530,6 @@ add_test(NAME DavidsonReplicated ${CMAKE_CURRENT_SOURCE_DIR}/DavidsonReplicated/davidson.cfg ${CMAKE_CURRENT_SOURCE_DIR}/Davidson/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) -add_test(NAME DavidsonMixing - COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/DavidsonMixing/test.py - ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} - ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt - ${CMAKE_CURRENT_SOURCE_DIR}/DavidsonMixing/davidson.cfg - ${CMAKE_CURRENT_SOURCE_DIR}/DavidsonMixing/sih4.xyz - ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) -add_test(NAME PolakRibiere - COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/PolakRibiere/test.py - ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} - ${CMAKE_CURRENT_BINARY_DIR}/../src/mgmol-opt - ${CMAKE_CURRENT_SOURCE_DIR}/PolakRibiere/mgmol.cfg - ${CMAKE_CURRENT_SOURCE_DIR}/PolakRibiere/sih4.xyz - ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) add_test(NAME testSpinO2 COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/SpinO2/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} @@ -585,7 +544,6 @@ add_test(NAME testSpinO2LDA ${CMAKE_CURRENT_SOURCE_DIR}/SpinO2LDA/mgmol.cfg ${CMAKE_CURRENT_SOURCE_DIR}/SpinO2LDA/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) -if(${MGMOL_WITH_SCALAPACK}) add_test(NAME testMVP COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/MVP/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} @@ -593,7 +551,6 @@ add_test(NAME testMVP ${CMAKE_CURRENT_SOURCE_DIR}/MVP/mvp.cfg ${CMAKE_CURRENT_SOURCE_DIR}/MVP/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) -endif(${MGMOL_WITH_SCALAPACK}) add_test(NAME testMVPReplicated COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/MVP/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} @@ -698,7 +655,6 @@ add_test(NAME testFIRE ${CMAKE_CURRENT_SOURCE_DIR}/FIRE/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/FIRE/lrs.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) -if(${MGMOL_WITH_SCALAPACK}) add_test(NAME ChebyshevMVP COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/Chebyshev/test.py ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} @@ -707,7 +663,6 @@ add_test(NAME ChebyshevMVP ${CMAKE_CURRENT_SOURCE_DIR}/Chebyshev/coords.in ${CMAKE_CURRENT_SOURCE_DIR}/../potentials) -endif(${MGMOL_WITH_SCALAPACK}) if(NOT ${MGMOL_WITH_MAGMA}) add_test(NAME testShortSighted COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/ShortSighted/test.py @@ -728,7 +683,6 @@ endif() target_include_directories(testAndersonMix PRIVATE ${CMAKE_SOURCE_DIR}/tests/Anderson ${HDF5_INCLUDE_DIRS}) -if(${MGMOL_WITH_SCALAPACK}) target_include_directories(testDistVector PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testReplicated2DistMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testDistMatrix PRIVATE ${Boost_INCLUDE_DIRS}) @@ -738,24 +692,22 @@ target_include_directories(testPower PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testPowerDistMatrix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testDensityMatrix PRIVATE ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS}) target_include_directories(testGramMatrix PRIVATE ${Boost_INCLUDE_DIRS}) - -target_link_libraries(testBlacsContext PRIVATE ${SCALAPACK_LIBRARIES} - ${BLAS_LIBRARIES} MPI::MPI_CXX) -target_link_libraries(testDensityMatrix PRIVATE ${HDF5_LIBRARIES}) -endif(${MGMOL_WITH_SCALAPACK}) - target_include_directories(testAndersonMix PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(testIons PRIVATE ${Boost_INCLUDE_DIRS} ${HDF5_INCLUDE_DIRS}) target_include_directories(testRhoVhRestart PRIVATE ${Boost_INCLUDE_DIRS}) target_link_libraries(testMPI PRIVATE MPI::MPI_CXX) +target_link_libraries(testBlacsContext PRIVATE ${SCALAPACK_LIBRARIES} + ${BLAS_LIBRARIES} MPI::MPI_CXX) target_link_libraries(testSuperSampling PRIVATE MPI::MPI_CXX) target_link_libraries(testDirectionalReduce PRIVATE MPI::MPI_CXX) target_link_libraries(testEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testWFEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testDMandEnergyAndForces PRIVATE mgmol_src) target_link_libraries(testRestartEnergyAndForces PRIVATE mgmol_src) +target_link_libraries(testPinnedH2O_3DOF PRIVATE mgmol_src) target_link_libraries(testIons PRIVATE mgmol_src) +target_link_libraries(testDensityMatrix PRIVATE ${HDF5_LIBRARIES}) target_link_libraries(testRhoVhRestart mgmol_src) if(${MAGMA_FOUND}) @@ -803,29 +755,23 @@ if(${MAGMA_FOUND}) MPI::MPI_CXX OpenMP::OpenMP_CXX PkgConfig::MAGMA) endif() else() - if(${MGMOL_WITH_SCALAPACK}) - target_link_libraries(testDistVector PRIVATE ${SCALAPACK_LIBRARIES} - ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) - target_link_libraries(testReplicated2DistMatrix PRIVATE ${SCALAPACK_LIBRARIES} - ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) - target_link_libraries(testDistMatrix PRIVATE ${SCALAPACK_LIBRARIES} - ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) - target_link_libraries(testConditionDistMatrix PRIVATE ${SCALAPACK_LIBRARIES} - ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) - target_link_libraries(testConditionDistMatrixPower PRIVATE - ${SCALAPACK_LIBRARIES} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} - MPI::MPI_CXX OpenMP::OpenMP_CXX) - target_link_libraries(testPowerDistMatrix PRIVATE ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES} - ${SCALAPACK_LIBRARIES} MPI::MPI_CXX - OpenMP::OpenMP_CXX) - target_link_libraries(testPower PRIVATE ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} - ${SCALAPACK_LIBRARIES} - MPI::MPI_CXX OpenMP::OpenMP_CXX) - target_link_libraries(testGramMatrix PRIVATE ${SCALAPACK_LIBRARIES} - ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) - target_link_libraries(testDensityMatrix PRIVATE ${SCALAPACK_LIBRARIES} - ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) - endif(${MGMOL_WITH_SCALAPACK}) + target_link_libraries(testDistVector PRIVATE ${SCALAPACK_LIBRARIES} + ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) + target_link_libraries(testReplicated2DistMatrix PRIVATE ${SCALAPACK_LIBRARIES} + ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) + target_link_libraries(testDistMatrix PRIVATE ${SCALAPACK_LIBRARIES} + ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) + target_link_libraries(testConditionDistMatrix PRIVATE ${SCALAPACK_LIBRARIES} + ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) + target_link_libraries(testConditionDistMatrixPower PRIVATE + ${SCALAPACK_LIBRARIES} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} + MPI::MPI_CXX OpenMP::OpenMP_CXX) + target_link_libraries(testPower PRIVATE ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} + ${SCALAPACK_LIBRARIES} + MPI::MPI_CXX OpenMP::OpenMP_CXX) + target_link_libraries(testPowerDistMatrix PRIVATE ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES} + ${SCALAPACK_LIBRARIES} MPI::MPI_CXX + OpenMP::OpenMP_CXX) target_link_libraries(testAndersonMix PRIVATE ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) target_link_libraries(testVariableSizeMatrix PRIVATE ${BLAS_LIBRARIES} @@ -838,10 +784,12 @@ else() MPI::MPI_CXX OpenMP::OpenMP_CXX) target_link_libraries(testBatchLaph4 PRIVATE ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) - target_link_libraries(testPoisson PRIVATE ${BLAS_LIBRARIES} - MPI::MPI_CXX OpenMP::OpenMP_CXX) target_link_libraries(testMGkernels PRIVATE ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) + target_link_libraries(testGramMatrix PRIVATE ${SCALAPACK_LIBRARIES} + ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) + target_link_libraries(testDensityMatrix PRIVATE ${SCALAPACK_LIBRARIES} + ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} MPI::MPI_CXX OpenMP::OpenMP_CXX) endif() set_tests_properties(testSiH4 PROPERTIES REQUIRED_FILES diff --git a/tests/PinnedH2O_3DOF/README.md b/tests/PinnedH2O_3DOF/README.md new file mode 100644 index 00000000..90551b47 --- /dev/null +++ b/tests/PinnedH2O_3DOF/README.md @@ -0,0 +1 @@ +python3 test.py srun -p debug -n 64 ../../build_quartz/tests/testPinnedH2O_3DOF mgmol_online.cfg coords_1.00_1.00_0.0.in ../../potentials diff --git a/tests/PinnedH2O_3DOF/assemble_results.sh b/tests/PinnedH2O_3DOF/assemble_results.sh new file mode 100644 index 00000000..6a9a4d0d --- /dev/null +++ b/tests/PinnedH2O_3DOF/assemble_results.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +rm -rf PinnedH2O_3DOF_assembled_results +mkdir PinnedH2O_3DOF_assembled_results + +for d in results_*; do + echo "Processing $d" + rm $d/energy_and_forces_*.out + rm $d/*.png + + suffix=$(echo "$d" | cut -d'_' -f2,3,4) + N_l=$(echo "$suffix" | cut -d'_' -f1) + N_theta=$(echo "$suffix" | cut -d'_' -f2) + rdim=$(echo "$suffix" | cut -d'_' -f3) + python3 compare_energy_and_forces.py --N_l "$N_l" --N_theta "$N_theta" --rdim "$rdim" + + cp "$d/Eks_difference_histogram_reproductive.png" "PinnedH2O_3DOF_assembled_results/Eks_difference_histogram_${suffix}_reproductive.png" + cp "$d/f_H1_difference_histogram_reproductive.png" "PinnedH2O_3DOF_assembled_results/f_H1_difference_histogram_${suffix}_reproductive.png" + cp "$d/f_H2_difference_histogram_reproductive.png" "PinnedH2O_3DOF_assembled_results/f_H2_difference_histogram_${suffix}_reproductive.png" + cp "$d/f_O1_difference_histogram_reproductive.png" "PinnedH2O_3DOF_assembled_results/f_O1_difference_histogram_${suffix}_reproductive.png" + + cp "$d/Eks_difference_histogram_predictive.png" "PinnedH2O_3DOF_assembled_results/Eks_difference_histogram_${suffix}_predictive.png" + cp "$d/f_H1_difference_histogram_predictive.png" "PinnedH2O_3DOF_assembled_results/f_H1_difference_histogram_${suffix}_predictive.png" + cp "$d/f_H2_difference_histogram_predictive.png" "PinnedH2O_3DOF_assembled_results/f_H2_difference_histogram_${suffix}_predictive.png" + cp "$d/f_O1_difference_histogram_predictive.png" "PinnedH2O_3DOF_assembled_results/f_O1_difference_histogram_${suffix}_predictive.png" +done + +tar -cvf PinnedH2O_3DOF_assembled_results.tar PinnedH2O_3DOF_assembled_results +mv PinnedH2O_3DOF_assembled_results.tar /p/lustre2/cheung26/scp_local + +echo "Finished assembling results." diff --git a/tests/PinnedH2O_3DOF/aux/PinnedH2O_3DOF.png b/tests/PinnedH2O_3DOF/aux/PinnedH2O_3DOF.png new file mode 100644 index 0000000000000000000000000000000000000000..b484f4c9eba4bfa6f73c8b448826f601c213b4c1 GIT binary patch literal 23439 zcmeFZcTiMYv_5zm5Cl}rC<-V+Bumagy$Vg1oH2o9keoqGS5S~9G#S(eBr7?Cq9Qrx zs3ZZ&p^43`RrAkG)!g!yHk>}^?7i1s;rqU|UudW)9HwEQK@j9HR`D-Q z1R)DR5X{&?O8ALL`_KgZCE;>I-$l#8!o~fLlR2V#$Hmds!Nu0++pM%o_{=Pdam2eYkM@0i?1fjcw{*QStmu7_^hP&9mu4;S4 z&kwu%XzP&nmx+YeWY~Zwub+gRJfMA4%SHCqRAiX+(WQ)7+Lv6Dx5@aJAld zY_p}57;lr$(|8X@`Sy@x9RF(F7p)#qbxOK%pIw8my zmuO=ij~+dyPU7LuC-BHZzdf&E$J)c}3#)6L>T~(6vYjK9-n2ch7^QUgiT?X(&2eH< zncDD8mNlg3N_u)ajqc^Wmnyvpw0y*n{}-T!s*MWpA#cwJ~Oo!oA+L)F$4ZpZZe zz{(=UW!@%qCgIz;USSAQ?6kRbN5p=puiLz){$PN}mv^DE8LhHAD(6IPOD6ok-@0{6 zNllF}Z((zJd}Vc2%frLtMV(5bw804uH6F{f)YJ^C&eZLhw4{cLq2Knu-O77%OfqGC zx4(~$j&@IK|DD#OHr#S!xMe!MFiFf!A8QLZNjxSGG~H z8n0nN$%MdgxqEN*{@xwP{=@Shd z9q~QCb-lAqn*H=!`{~t*fs~Hi-DHdLx7E+RC&Cm8?$;mZH7V1+c<~~PLf6SDuO&&& z>_h8BeYl8q*ZaVYzK{1@f3;mir{9~n(CxignRbBov}FeYo3nWeHgWhdL}qArYc(=9 zHUk!$TT-%YtkAYUFVGi$lmrzkRBkYntsk*OPv)F2rm3EB*Sna}Jij+TQ$o zi-**GwUG}P>YmclpEl~v)Z?G;`}mH53rut&gh?J;1^;z~0zsApDgIyoVSIWPUb}>? z5ho~=vO1koAzfdmKtqEgBWI6m?(W7gJx2~Ajt(&~9WPFF^yh2Z=TQ55dg5|(wbMen zk{?kAA`P$5E6m`E_IH}PQPkdk^t%q+912S6_`Lj-o39Q&l0V=`cLAx9F^HPkYHYyG z_V6;0*gA8JdTr$%e*R6DY=~^3puCvj5ff#Fiv6 z;;zZis=I)h+02l{OrHQJ&r1x8yaeil7dZV391P&?wbBTqenO7~xP$-mS9N(R(NfOe zC?w;WDN1R**Ws#(`$7lVF34w?jSii-EFe!&La~6!!id`TC4YKM#eM10P0={jj~_pp zFI3wPOZM9q-Ocsg*^o%Pja*i?NOhU`+hw=JWrZ*^yfZJBztAU;Kb)u1YUWmS@;py> zhK}$A-0|`WnE;WRQs-%x^=`enM^sF0ZEvnO`Wj!m7I^LYb=rZ$48re2R*>XfK_S(p z#s)zdFA0e$QdOIVv)ovzGj7=av&qe8+T}$CdgX5Ql|J-Gj#AS&ZRe7gaFg9We8q80 z{_uJ651;F9+`O4#PntZAI2Hz{`Gyge98YX^bf~DCS-v+=QxbgCmZ!34=Hao7VU98n zk?|c^JNWFmkT}IABR-%~KTR#+BqL*XoP8PArZ>mcRRXDqS9dF#X)bphz}Lugay6_i zCCKro5TeKM5%(9CGff6BZvQA%&Z_4@&70@vW&qWJTnu5*qqjz&3_0>cD zwyti}X@1iz^yYIt*`0^QhBST(7#J8dG&FEorlq7{ao|(RC7zo%ICK5|4T?ui&yeNk(NI07 zpo-H=kknNCyOiKF^|I9G{E6|`pDrf)wEU>FjryAKQRGFlMQ4UirVGRt?O!FZ!i^UB zGvMdBAMO-bbqf1#uic%)^O%N9(^9!E@Ty7_+TcrMoaS_0^-DBJ)2)qp z{rB^;0+Tk9FU)(5VHg*O3@~ZxjbcJtrJp|KO-?m%NEyW(JJyh?lQ~wo+mfiGrtw;k zm8D|C$?@jRXE$G-dDGR^#X@{sD|HmHiEf~*IV)sgS&w5MD0g>wF{nI?vF#wd0RMB6 ziODn)m(XBe%+1HA28I_|U0t2on0{0}JE1J0k#(+!r@ymveST$fyVe|EFDKZs)^vd4 zBNEW6gQ6@9VpH!OcJ6404A1c*apX=4@ zOjQ-}T)EFEuN2U7w9X3z(desuP12fbOe-DK&#dW-NPJ0Z-H<_>=Pmb3if z8!%Ka;zD+w{iJb;*?nkR zQCW!fg^(Z(hhIy*Ksc6)BD(ZIr3=lO0OcA4ndY`ao|(stl>N5t^v37jdbd)U)@Wc{E1Ig5Y5Je&*I8YJnf2BXS-BWZ><$0s~dfG}d#L9I|a}5q&iOdC7f+ zAZ*i%jHTCHOCqoSq)l<$4w9a4*m#Y38#XGP2=fQ@BjN5pd>=_jbQn3raGJ1jvRvC9BR@eSw{N_?=-+(s$M$-PX=RhIKf4K+ zMFGDC3;O0FWaO%rJA2p*u<@1C(=Soj4_c5V-5+qCik=)Yb`CYG_M1w)DkmqTv^i9Y zOY@QN?pb4Ss%F-pK;J*-#F@(?_BEir|-9U4ka&EP~7lpI4YOASX?94hX zo!C|WD<2#S93XaW7sSY2x%3R4coq$~#cdOFxsyAjoz!DzEI*s6h0GjQQRnyJQuOd> zH?7=lui9eusr`Ir4P#6O1<#n5l>6+g`v|9P^=e30b+zYxWBC9Byj;z@EYr%4#4BXU z2hq7J)YP1CCk|V%SYjEZbQwe%c&3TJCnut*6B=p!Rkp|?&iP-$AjTa0>I@=EMP~jY z%`FLF_w{<83GM8X-aWtTlIg~9=-H&23fBxh$aXgfke`~5SNm%kijmkVO zT+2mDs|q>wjr}$fV6(fpAFEJ_&9fD?ixai^tR6C50u@9&AnFH ztj6RgRbw=(?yaWzq??UOGne@7eV#ExBF>?AG**}8v>UOFiW3Px&2KS#x~6`U{|GWB z&Fs%px%|u^M@M;kO-tyb5R%4%kRkE+TA5}(QPB=MPb-;}mE~?Ptg@**pfAY4^>Z`Nsaf^@X2)fzEdUOaDz1$V6Cc3&eDOYB54!nifj75Qt`ZSvr{nJyU^3 zKR*jZ@jk)EhD?J3odfY5Vw?nNrE(*->)rFl28@OvY77D`2}0pAX(=%q!|uwVva$I+`jInT&%*Ea?o z^!(o-)=~7PuY0lt8pFamIFb}eq?~MO`BDhq0dVciY-VoHP^oic;r=6XDq+jE3+uo8 zwIG7V3YrHodH)v3*8li6mivIly)09cO-nImW~+X|)h~1i)1&E?E`*%wNLS4x^2W;j zL5J$>RMj|Zh;KLocTUN*!6LiP%-pJsH`kx*YDFT}FUhG0I}6;lt?``e3-ftSi1v9* z8?D_)g~8tTI(J@<|31m^!|g|qP3NBN+->S^k-GcrNaTFeS*+%WT`{NoXNvTor0Mi&qm`innQT+>)9?g#4vx$_ zwa;$!h}iaNNJvOXdae-OzIzwUDC9h)qM9UI*|%M2)5|3;UNT^v%?E{e4&R#&=|ZCeL?%~-M4JcTW($NZyneP1&rbO5yp-$FHRJ?FPc^oN4(L6yrddC5wwCS$*+)} zA?>7qhn&wbN$E9nw^KzjUMkm!)D+nbcBQE&1-g+cy@;d!)u;K4-xCOgz$=%rP;R%) zc4w|{j)&TeRCorxc+q~&q2klYD@c&{mN|G+(Z(M?I?|Gq*oTgg)#Rhb3?(rAp`VxQ zKR=}n&YtVe#Jm0OyY)&3v*cms&$~D8V9{?8F1xE?J!3F4^Z=&LmHTKe>P127#Cm5oC+>u5mW) zfkU+U;npSZm}CfY`=#4*55{lK)N{}dGW%RGaWh!==R@$|p99M48^irJua-^4&hEue z+_&0v7T8ImASW^>cHR=++_A|Jf=!p!` zfNvUV4q?P4CAoC#bQ8~Nr7F(##!cRQMMzO*HyfSZkw)yF!h9`Q)E!ATk>b4aVY5h% zbSpzoy==9mp?bd|vO|t^Cu(n(Fxi;${d+?-GZKmd3hw#$6lU=L%-&6hy546c`QdJc z^4Yc~^13dXZV}1H!eExh)OeH!O7qt^9M@)8pFYuEsBHb?gVW+RWQdC6b$Sbxh=(vb z5#eTjVVYWua>$oF!pD0lCgm#SCQ>uQ?lHg5E@}GZ@F9rfBSxtvb+ggWA1+}@B+bF1 zNdM8|(WodMfkvp%>l5d?(sy?-OO`O*jvUL&QxA&mZ}hs)U9uZ7E+r1--PoQpqa9qV zf)NWr@b!qqOmy)8(i2Y-jbrF>%hw6IbBPtrbCUB~= zlVyQO2_sBN7h6os?Dk4yUV_BEf$Pr{!!Mx$#U8b6k-rz8P-pDS<)*fm6~?SQtxv=l zKS6)~)qAmix{}C-)zx?%^IL+a>FW)QqXN9yd+ML#DGE%SmQz*hQ7)5_d^`sOq*%dr z(g2SDJC5PRN?y4gkmbGT`tTMqbQWwz+Q2kX*g9fu$JgJ}q};kUT7Zv_CZHVs5=Wef z^7gKC{&}XHp-@>(rf?bZ5(;=_kBohip4Wy!zHL5M8Ly-ohk75TNAPBezrG)vN%U@F zzg;iAk!>ai8A8oH*o`%v_a8v0#B!*|ZX{pr=t_sM7NBqueKF6U)HCIaO-`#*9Y4R~ z=**^(hERt?FvoCb=jqtV0A!|9J!N~{1kwkv$E=GZUUR&KmhIObJ$f`dRGKepKcqif z1r>jIvjz7Clxj&QxG6R6l)Ed_$!PZ_O6koG^}DVkOZpTw47dC9%ww$k^QU zK%*xFMR1UJW9r!1*e-K&rjK=(x!Q2(97_J&(LuO)o$&Hdr{5Un#TE(CJ`RyQTE@HR z+sm`>sF-~;^o$=qoob4C2hP)yw7X@_M1S^d`uq3q1M#`Jx#3)t8W*G3_h!^P)a7E_ z>OwjyJS}GFk@pmGdh`Jq^(`$u8}ozfO+u|F4Mua#~%Pa6$zB}yFscKkkpI~3|NiZs!4Y{{E})!L4}y5=z$ z49&rVmy1RKdwy57wBl(!jGG4h`PBVlW#TZ$MtE{8;_ENxsQyzwVJAmOS zqECK^S=Jbkw+>qViev(T4RGqKnR>ap`4-J=06Ac9Oi?s}H4?DlE^EJT9XN0x9T1@H zpTYcSf~mJhgRJbWg$sMR-0|M9O*az5N0KL7upm80$G^SmEO|JtG*smH<1@=VpbARr zGV2j1GRQP8PPbsMKUbQWN%d`&>0k{l0skO!oCw=p>I2UVzke8-gm28J_>;oHr)kro zk>T;yY6_j4FgSTdZe7Jfn{xC!r@=uP&+jazP@kX~T&(6Vw3~xuw95x|{wXIo2vUnY zK3RibUZfsIy3rgz2N%aZ?41jMsTGhQ!Zy0~JASLLy3)e|C>?eRK6`}>i9}O%y%R zr*kVSgqoV!>|^q;ZUQ1Lu4R!5PMiH}!Sh35i5_p)_NqyS4fSD$g56XL-3C1 zb)B2bnm!c<_QL}zTbe!hTVK4GdU1i1^J#AMnRI`Dj+3;>sc7zy&!d0a6nD0{+$rTG zgW(21o))7%b3HNCZ*xmi2O20eR-x3ZIs0=wShklq?4ELjkI z{5TjfvlZBZq{mXSprBw^@>O!z*)Htf4zU-&XCPa~W(6@N*Zu^Uvs>KQUQ_PoL=)1u zuTPV2;9|9z%sOUC$(SrB5Ov1$gL-;qzJ(|zo&$mC^Al>6192VoCDsG4l_04jARyqi zHHnXm-0zh`76qJB*S4lDB&3a@(0w6y5L0qSCP2p#AlF~@97$by=8f9G&}k9}D?IZ} z{r6leHwJ9bI_U&A#hB*9haLH?vF35AP-v!OPe7PLSPL8+IyM+BXJ*n27zmNJe__&4x~Gb234yYXnmPz+&_d|=HSEVUG>|{ADo}DEghClM~@IugWiFtUzbYkXu|t*pK(t7>|o& z$&X(GoI$(7;c;NB60+r3u~r@`3Uv3Z&+3r&tb3ypz11Lve)4^1QH(X1xah7@iB!Ng zlvH$cDN8e?6R z7?OJpad`~^t*nd!$+>HAX=!qQxi>vr*gfZ-T&+ERp-kDjH)n-h2^n)Af=#3&;$2kQ zs}6LHNVp8u0f~jZ)uc;5=eT+GHLB9@AjyZ;5XtYrasZ~sPSo{^24sxKeYwiyvvd`uds!7i zv|U_;iF=rl3!GR6S>uLgk@{~W2mu4UY@WPnm5c#hQ>$Bb5tDZH8 zb##4r%ByqDPUGJ?I#Q;XWRN9R6f7d`t<6 z62b+Ds7_nDRvHU&dvABO6*%2Dv0@H<{{H(cI}3duYx^fL?~hPZTQuN(a;y1V=cWLx zNIrh>pY2cr6W`*p5D&{FGhCqX^7Xe@&QR8AP*c=!d62fIO-#s;2OoJ>ql<^jRm*+7 z(QG`NE#?*I2Rp5!&tY)k!)a@=bB=K&|T8e(XrNdFSN8w8kt<` ze~`op=__jij&yp*Ai)d2Ee(tRQ8)z)fw@W3bzl zb*x2p=eN-C!#YMKEH;b~SSK;HuM*}`PH6R7bHw^0V^9cE)i*YBi>?Q=ZSCyPd-HvG z$7@uia^jLoTUWZ)DQ4zePzqR&)jUGc{)hXJfuh*{Y=3^*&+%~)^M;c$8-4c-!}hOu z*EHU@u&`eH)sAD!j7YQB$KB7jT6;{#B_z~oRfv&yy#RaRmDtGpYS_$@Zolw^a=^uXAx~W%17eg0f}=t zT%EE8BFR=X?)Zrlu9J~^*1$1!ZS$*bdUc<{^Ho&U$Zpd!Edwb&e@gNzI6oY(=472Jg zwRrJ2u!~Rj>!ztO7TOORmX?-o|9H+EROtXP$Hd}du9@FfN^+481P^rE0OIuy!qXhy z%&$F)*9g6!rluyaaRZXHuOq~jMt(gFjn_vQB?(}1^!=wHqrdrzfm24tL~eiAIn2(_ z&#!EyftwvGN7{Vn(HGMdF|RYGVy#d9FZ zh#a=Zv&|2e7wfBQuTC}V9jhfGohY+$buB`GNS{9>%KR0;m_u;OE-f?gA?<^ZWNf0`ulONI+6U z7^ReO8^8PCWo7Z8=n&-M_(K7>iShAwU=EqEgkOtHeAl~ImY3Vr<@N{wOP=b7q2`2Y<%<(3vFq`b`?3@z&2KCk4^HMC;3%mZKIol!P9!s~w>?XHD zmR46gm&fa9tMpbRcBEL$NW|YMRje>*t>5|{$7qif0?sxE`|1N}+MiSx6datn(JJM& z_MW|d8&)hhD@owqvu>zej)z(emlcZK`}UG$u4-??NEzq)yYJ)iP*3oxyfK-#&8xiD zXUzfbH3YZA|IpjkHtW>6b2f!Q|MeA7pSYwCjDJtk{_gbMLD;>P=!MF7KHgFD^2_JBMrKz{*-b9&HiwOPyveJ*_YmTg>8lG&%7A`qhVZNTk2f?mBU zWbzn$ba$}KZQdHackpRmuI2Nfp6Nn=hdLSCL3rf{+@hjAP)c9>Xa}R=eVxN?Q?jV!r#4+@lUZBPava=bjszjf~YgSe9g}B#S zP5N_R8Lzd=Me|iV3;F1HIki$x4x)8U;Hb#u%kTR(;6cskC@7Ddhf&xS4_lxkggS_; zh*wy+V9fX3+qXe>{;+!4eSP{in@c~w7A-UW@Ebh_8my2l=Ht>q&3O&W9Q;E)AMe!x zp2`YUI~FQ7eto%n-wyGbR+N0HJ~CKf^^S_kLmo-ohKY59Haq*!{0POZR;6}9m%bC z+F&0dt%2B(o_=8d-~oQlSxZZc7uy3sI=5a)@cjo5 z&{dx#$qc_p>sg?7^@pHxY8GQ$;qft3nJjxs9sFg7+0TSw9PTMaI!(D^Ug zj}@`$N>RoiU)(3RE(2gA-L%px&*bMvdl1%uMRq2*kIWDnkYul;NaimG|DCvJhtH*g zXHkv5a<%q3T`zzDXr==WSQ$KjZ-wV-M&UD{=V#6KxA7z1i!$K4^+M%HB@p98U>9IJ z{zLng$w zHRk86H70olGSd6`{XxcpWhIAe@+l}OXJM~)F8!!2^jfzDffDdtB&2bX*Wo*wXvj{E)qBU5vc{H(R~z~sizEN9^jI5y~@@5l~f4R zV?_T3J(WBnf)5QX@c7tOn(wldO#N70`XX=!ncU<7L3e=H(3aoZMhOiJCRN5gr@+N( z-G4!BR53vk`{`2$Z?Pi1j3oua^#svBKu&q64I%|!sTpW0P5rhet&`}E1~|Jc*an(a zJ!itXE#_3U{#WpU0I0@j8ZSkF(vRbgrw*MHRRQmSM-vQiSm_WRfL^%;St^>yKzjEp z&Y`Lu9IoqJcJVCp{^s*cWy-M&_r58{iDCgSF~6?}kg`#+QunfA_lpxL97z#g>w@!u zYuXK--v{{>73KpCif2MfFx#mv2YGJeTB|=v3k(a_67dtltCN#h>6aApItVRaY;?mht>(nTzk z0^M7y)2Eo2TAW=UG4NehH2nD4__!KYfJj+IDx3M}9Fbcc1(mLDHcc-`z*OU=N?MUI z-x8{}PoF=3K6&cYjZN~3zZ#=DsF+U{C3z_gl(auwOp3OgI`szNGIa{fG$j`knt*@wg)!lV89VvVq%Wp-gY`vQeHDDcr?A# z!&B7UVDkL(aww9p=3@0rgK3zurt&Dlcpo;OB;+UGLYZ^XBn=XB264w^*}XNb{&_b& zJXH-O=TO$AxXr70_wrf&*00awBAVj8F!DE!A&i$n{p$~}P7#72tWordo41cR)VW`2D{k8lUf@h(-{ie;7n%1&~B#8|2@t zs;XiTwtS6Nac=ViGf<_wyLuw=V1RE!9pkQC-G&UIQb8@G>+Mr&UrtU7hUh_ip;l(f zfCeZ){~osQ1-X?ASofdEP~}s~Qv0flB`9K}67zZ!(&B7SOIGqm8OlRr9muU0ezPGjM5@AzA|WJ`2I~qc19{E_0oe ztQ19ZL~No8%!AjO#X7=RJ1U7vW`0&vN3!1g?lk2G@Rv6&yOJ$2p*RF2vpJyEwF1_Y z{h1s#LoYY+$a(Rxq0dN3e0B9&J%?uPvs6ZU4%H;nWn%rmz)10DZ(2;UAgr+<^ zDa*LxT-9?K+ykr64PoKtM-#&h< zx?{F4_~rv!b8Jj~Vrvo}bW%R+zh^4EFkRB1vv`lT5j0R4A?ZUJR-L!a4=0owH$x&$){vwDDSxB*h$)5eXI2q zi|=#Z-JCCL-O>*Xk>ckoaG?sg_9D-8SHj-+LYmV6y-6_pyOe z=d`NrS#Btd^w59b3_0&vKTdh*3;^zgz+Q0B2^zcsD5`V# z9eRwW7G?lY!P-ip$sms4Pg+fy9jPn_jI|4xcg@hl2M=mMftRmzK;!Bk?;fM@#r!@j zbg_Npwd*D(X9HqEfks#4xoYwB>C;nBNx)(3_?ThMEAC1Dyv@w49S!Yr77h znTIQ$HaQSRBIzdFD7VK3MAcMj4Wgm5hIoCLn79u*-_u6SQ2tI#OlXFl19yTZOcu?t zGaw=H+US4aI{r$`8nlTt->a8@vHZ_ETkli;C#LIjyTh$f&AyiV%8282G!`IrqlfaAsgy$tZ-R+|Ew-k&HqwI zn`z&6ffJ~=e2XnE{#FHOrJ(ZN_YHabE_iWH%~kKoKXQ;XwPl4TVyy5sfB$_DN@`=L zYN&0wMMQLrifr2ptL0QlknW>dy%$7;g;pR8BaSf3X8ppE99GJqwc$8F-OkRAcNe`W zFoUSHCcWI6L~<882(mj+p~o$hx0->*B2qWDyvyVkN|604GogIs@d1?rp&83{IvE1m z=>ph$0+ZLTi^E=X**9(?fGbP_su0OG^SeJEhKv(+7|DV(%zC)Yl|kC$15j#fb#kP3 zoobo@<)w9zR$Ax>DG;+4RzHFXXt)6MBQ%j{=$U!~ojaEX(QO8B4q$zQgMWv=ex2Oa zrHw^V55L`Ig<-#)dF#&9SEx3&(5mxhoS1_y^m}mo9Y7|b0ieLg^G@z)W^OF@9VYp0 z3B~Upv%R^w?8;FQrL9lLeF6gm(E=;2tdUGZwahi#)Zc_bZlr+OLw>gHD%ZUMb>#9h z&}%MT8UKc5y%{~Z<|2e$D|JRm=?5q&{rqD;!F@DxpjZsHX7l*Z=q_4$Pt?bfcjwqG z>^D}sy3c4@k$g~k+Zm;|uNT^JSMK}W@%~IPv$JT&mq`^cjk>RMv+cFUJ(Uq}uBUJ$ zUTJhIhnQ2Pq?)Z$`4zM-sOuE+^z5LMQ|vX?s@i+d65+KW+^7)JC+zL^U~)>=%omYn zMm<+km9K@igC}`%b#;MlOvFCLcYswCE0@U377?X7F%WXbFiNq>$-#s+_^Y@SKLtyK~bV%`vR={jA`DJH|ullMt`ZPo!Pv+*EK8IF@10G#8Rj zA^uQJEjkyk^rH%#)4EEXm+jiyHS0nwBm^}x%*-Tx%YMm<#sPM_Y~9^pO!H6tF@$vW z5&C8~{kCo>saCEbuk8k#EPq^54RWA8($db6G;vwHuYjKC3P18)0rmTN_nvRf%yG{T z=xiz}ULKn6;nJzF*S{Ojx3SE+ASRg7Uz;r$SaY@S0&vd{B}S7zDGvZ1`R*w3S6N2! zDnmHq6lh0PNvJ%;;{}igM)m@YX=>QoXEC2HA{BIC5f%SuEOY2*ky(DCtV}k>g1!{X z`Ye`PH~P~>Tnx)pEgdSGm`24M_|hqcne1YkL4Q(yj4eX|5w%bH?am~Ez`Z7?o6m&r z{~ZxE1&T%SKlks7b5KfC>{AjEF#-4gqll94OvQ?g+D3(k zr<5;1V+KBoBAEe|q^_(-M)5^5h>6ZSR0)9J&=5j@iX%`SZc081bLTfN(n+bY3Xen% zQi#jtJvHb6J&M|hb47D$2<+zA`= zs@|UQk*efhkUohD`AkH{tcsPSUS;Z(Hdo$NJIhkN>-=cb4l?7OR|1WKg}? zuOAjo(QlY!ye^B2>$|QdO}kJ>nOhAMSfO0f>a2!y81pqwNjgSr+~F_-BX1 z)#4pdF_!c9&`=Wv3$bOxwYqinuQr+YOiLK%*Rp4s$y?J2O9OB8k>h zDo|Q9tw16wLyYElhAbIV$kU1WZo9~kHpfvcy0Ng@+#puyK?tg9xp@8Ad%z+L3oOHc zkS%bT>9Ec@9l)`&8ZJxH4+;{Thbk-4e+1247@!3MZC%8Lhoen}fvzBXfJ6l^YBw

(qUGzl>*%G&-{KIx{ONgr|pak&nhMCaN}XXDTV z!2f4EQj}BVYAy0lSs6v?*bYCnXo)wts|xKyXpB#pbBru05!b_OSp8|{(R1wvL>4q9 zdUhk{-$9;1c|O=P|(Fq1^KM_ieg#IjYNgeGEdBXX z*FgGUY5Xns)Le$1$F{DdnpzbnSTPW8#ck1Uqa#!^P$;sK{sHj@(i@R( zTttZuE-ob$)&ju-n-Iq+rm(KKD~6wDv%xktYxyr zC|9E#l`?-N0ZKrt-W(GKC`bbzKNh+B<)Gdk10t>R{78q~zCoYg#4)S7Be2Vp@BYH9 zIYJ#34{xvC*0R`bRPL4ydw>@UshWnhwe=@iR1^Z;lDUA~qa0G%LfITz8$O+oTct)C z?tD3DwQ6CtV9d$gsKQX`TdrMQ1vrZ%C*+|e(COHamvuLHcZnkg!vqC0pXH#fC~y#} zpj&BZYU+YKqWjHts%d(5*W=0kGlt%auuFQ6K zDD(f(YojHEHmqfb0y9>&dY^f+befGW?%07PN${cak;6XupsTd#&udx@JaGcG{0`I& z9a(M?>c}pRorI>pJGD(?N0Eg;jfw}EnP))z4lKQQow3cV@f8A=6!% ze&s*MYgK<7tx^6+P2Whm3Vz}rg*sqw^WEGmM&msA^f5>O?eMj+#}-?=o{*0mSZybF zM4MWY4J)=%U7AHKJoo)(9#J(Nyo2PRp-?-xMjqA1mAR^VcXO>y9R2slai@8Mj*xjy zoweHYLOn4!dL*M8U@1U74O*HbpV8q%7=Z!~D2BLzWDFNrX^vJ|Y=Q9QV8Nh?;qkF< z)caXa1LI02o;?g_#V)b8v& zq9Qz@0Ev(XFM4r0lb?RsqHMm;MizvZE$SMmmk=Z^ub%*lPDy~5WV{K_nK#UFmNhTX z4rZigrz?jyTmPc#3`GmWr*lpHEG~C)mR7m z_V`!{t=WDdb)3uEdsX7lJ-+oR0Ged{IXN$_9iXC@krnUDQpXDHn0W96;83ICo1EtG z-Mo`C+uaz!wud&2)G^9%*^Tn~EWo@BhJf>6RviRbI|et0-RCNlS~H^xaWeQdYtpVZ@2r z;fm}q`+fnw+!W=+hN% zTIiPWoVYTU{m015J)EZF&!?*;*a8N!gqIr|AFxrS8wq2bnvQcgT^S!2Yfb6vvy(0C znjw5!ez|pJU1X6i`6$}VrG}5Uy=J1K@}mp{7c`U}6TvSdpsSVUknnu);$%M0st{|f z3e(;s{zfr00KATgvRL4i_ODtxQxk_y1p0NpInzyjXaI4R5Mv`OnZj(X5?-%Jk2L2I z@TP(n`MH(1=fyguqN6Ni=lTR@#;%@_781&w*CUrE`{xZ3>;i|s&i3K~9w5IO!XzT& zE#)4I=rKXB;r@<)2Q%q*2mo5KA}GFw5#p$8VvJ6(yGcDw7_YbrU{*M=~|QiW^v7jcqH6gR4^wPxJ=w)sQq!&36M@QeLlfaa{kcb4LEE8)Swz5 zojeKLFMuymbJ6YZ6rh9llqVGa* zr7KgdbbW6gd&Uoao$V4x`kHdvgPSb*c_U~Xp^68&8=c;-5)3)zAGI$K53|-8N0Ff5 zAVJJx-^b6co}N#M(NW;&_a63oXeB{jCOdb*YVS#_0fbEkAeZ61M1 zMrF~|G|`hw(TcdWwYl`#0=Y_{!O`CXFUuc!;5833@faBICFu(v=YQyn`4^X@#a^3ba&*VH z{^@&JDJZM0P4U|!)IOgF(TbL(Wk{^Bz+7!DE%ThHXP#=S%=FGq@+({pl@a)MdeiSZ z@&PIF=T9(ncBquR4B_)SE5T8>c6xg&(*MarE9v?8>D7OLp1X%4%40bL=G`mRN7TgvL%R5Sys7<@@3fvEJb(;*H6V08KCPmWlpD*bU`pwJo6l%$sUWoT%(Fwu?jd=fWopLkTWIH#+;`0SbE@I+j? zP9q$x5o2DzYdh4ZLKV);(3&WM4~G`n3*nT9&&yF4q$N22$usA3nfM^V{I}`I$=SKk zd0Gwau6_XhB7M{Uk!WJ!V>j^f1%OsEN_%L)uI?QB^!OB{!g8eL(41`%L>?HEk!nfT zCi&M*Dv&mQUc!g7va`oLV&XN5y+Y4m?X|6zz$KaPd^>sOhYh?I=ZUjc(1{x7DvhBS zfujNFh#;v$NrD6Opj*S@(Bnj)>qr>~kSd^lT-cTM^=`ln--9+v9@s9JDP7R271;I* zK7H~;ywVRT(EzWNre-mKXH`%cqoomiEaBs;8eRTdP)K1cjMahd4{?OtdT*Iovl zl#}E4ymnR=7omOn>e5f-GNj=bFRpE3=>NpkkkC+SZ@}H(|W8S;rqUKQ=IEJ;be zaALyAsfm3IM?o2jVe<2vdLSP_{pjjdi*qvm@^Ekqbm*QuV|kMzEe-E;R(!P7dCKoo z^cgZ{4O10za>P+tG_Lf?xudr)1-8_4mdS9`WlhVOPpOJpMHyV<_opZdsZ?0 zΜWW}ZWax07S)_=U72{YRe2Ev%AfJOehv(Hm`_R+%~^P6?u81o1cv`63d4M+^A> zk9_d@fI^>5(E`?Ratk1iF8${C8KA)R+}zw^>I|Y%0C`8niC3>)odww!5Mc9!3=9&G z2}i;7fu;o&N4bx9DwIR(1WF_okNO$G*+HDEC4i9L0NEGX>RISIrwfqw5{BLl8A`VJ3$Y{=4&vc}0H=2=t zMhw7P%%Mj@0OE?4{$2AR!P0<3ZJ?ae`+ydkm0s&;bB5C4fPc&S6$zZn_L>!y&cbyL)>kCTdST{&aYD&BKum-XvnNxdC4eMGi5JM0yQDK`8R)I1EtDXw_RoWBC|pBxz`Aon6m53X&zCi-qX$;?CSP zqhfm_e?U{E;EndG(4U4@6x*j-K z=hrQ>^B}1}1&Sz64%#q1yw=4(U;(|W6f|Kb0tEm=jq>b1C_;~EJN$}S&J>OmagW&% zb{MI+y#?eTTJV>-FXq6Hi<`zl#={nqSOe!M1+LeHGV}Lvd%S`8e}IDGq!IJ!(^I$1 zJhmWq$mF4f#=qwYbq)CXoS~b2buhsGG7HPKW5~O`)T5AGd$87}}Zu z|3RY$ICeHHC@2l9{VGx209Wr31Q}xfIQkfEo`I$rUC@`T!Fta^^9b6-Q_{*mI*B*S zfzll-i1*`AiB86wdSOA%-T}HAVXrmI15{Ktz*XgJioi}odtg4H2P45L7zT3tyZN9$ zF}^?e^j#S^NSd4TN*(A?5ujVM87g@w={)r^r+h&Y<#XW(q)F0m7_iL#?kcg(NOGst%U&T9GGe*dXmon_SCXTr=_v0s6G`{-U4>U2fh`(YeP#5->P_H zo-j~AU%!00eEG5>47dk~kOb{Q0Z>}4|7yPu$2MhvQIv+91(3p4)F@03*);K@Qq^4XrwbfJ+Dx;dF&c zbRlYLL&@PSn{9m+^8_s1}4{yN`g5pPevW=wD?~a2@D%+mT1h8_* zX?~G!y8Nx{wDbCwDNG-#(DQ``6V!U)tTB}00U2kc*UNGk{fh(I9o)GUccF3bZP&b~ zr>8EknLz4g1-iz>#OMO|igv(Dz5Bv4@_%%4=218OQ*hB}Z?6$TU$ zA(ChjCT*#YP@&YOKYD6>A!iSKq8>|_E{Z8ZRN?$yui_OjN)68ly=jBOU z#X2N!d5rY|^ozLGF7AvFT+(3Y;zMopBID+s*POH@Tjau}D^?ld{{}%4H1m%Vo|Bw9u-TaL!qy>R)4-NXq_!);EWMdOS{tOSH2+H2OTgRGx6rxb)EpI69I`gKEImxiUU+IMTdI=m(T zNT`XvB~Y~urH_3@nIjR7vrmUNdsFynHqH ziMHim2g8`)BcEWswSrPh97cL9s9U|i*o>Fe{$JB&x1cXejmZc8sYvWpsYFaBGx*Uo zXmeQ8Y4bLD^79S6?*xiB@AhxZ^(r{D<8c2(Iuux=5J4Qk%HAq+%&0t*Xgqx2ds{tO zuiMHA!xD^%M+e(?p!aLW_ z_nu%!Q)snUf$cvh(2#I(f$Z4G$PQDGT>Hp!aRV<<$zLP3c7>vOJ=q7u>5?~;MM!8Q zN1GBMqneo>Z_H8&f$JC&2j9uan2A-4>^r5XWrMmmCB-}4axzwzUVS7L=9TSOAq%22 z3#D*JiBA^KIWWh#vE_PjsefujziDW1Z&$A> zO8FX-oE$Mcq2drB_~X`YHSqr$F^E;?0SPiLbFd&{KYTFm-ADK#twR`;BXQZXu7}c$ zsTWOoV0WwbT%Q|<2y6cB4}Poyi7Wj6VrRWas}8t=W8CBB?0NPX#C<+MOL9yQO2(d5 zz6A*%q$0xN^vMHNrl&^?Q4p-fp@CusVjqof(wg=pKgNS=1`AaXmYkNBcD->g*NmwV z-GBvJI~VitY%s2Eok+(ap~jgPhpD_zgBH_{`D2GWMGgZi871KqU3mdVwF|g93a?)z zp$#z#Ns$m~2Okz(;v0+Bhmf`*Fb-(4ou{Hm>0&O-&fW{dlD62toc$cd?4fm^-0v{p z&($Dkt2GaFU42rhY+DQv45&|@T;k4V-=v?jb{>7Wc0d9=ie3N}*n!Z_aM8|e*I&?F zA0pI9pRp*QDJDhWDlB>!A2Cu5Oi&>Cy^G@gZqLRkofuhlD5Tzeq@*gO$={MPwIEV7 z{c<&{5m%Z15!}hfzdL}F5A<}h(xa|gHfDR}o8+6?E^UM98n7`^+cxM4(6_3^)b zxaRHwj29tU^mqF%%p8z-sh-oED;8SqD~Ecl&(AP=18~*YI*29EtQ8R#v%pOAiTD1f z?xoy=ls>q^E`=+yYGiER0J0p;ZF_@^Nal2;z)oy8v<)4|_TRC>t_84r`2VJrf90Dj71C}!Z zC+yf?Vql|8p>1H`B3^i^BuMw zp0prz8<>#^2L}h|J)rR>+EQei^3{%V z2=ezc2)q|?hqJIR6vHibjfa2~(_}B-RtVh>dT13$vtl1yQL~}-ByH6u*`yw4uNK3- zS2)0lh|c~n1N8)m{UJ!lIzjYmup^N{g!+tyV%#}3HHB_^&k%Ae*N_ZU9I^O6m>ec! zx67=rgxfxdtkTd|gGJAH1?Gpy|2*EhvNl&PFUieaiac}a0~q>jsX)6xe*W08g~;MQ zhF`$^MT<_c!9C2F$N?26e>uiRau^;C3jxr_*RVwSDZjIA!v3t#K2LnZ|$SwIfb5+Y-;&C9Hk>8Vt%+u>q7G%|6a zNgei@1_wq6f_yuE&E`T36^zX^|LbBl>1EIsU6argiM3jQ-(YrW0nP5?t<*a=pFO)0 z>Ym)b>NgopsI8xGXq=u@LgRCxu5Po_ro{5wvKu#U0CP)(k-@$Pv8IYLyoL(Y5Hy3t zCW4dZGP1#w!_onD*0ZZwotQzg?!{lehkF)8M<6`#R5pxj5gNATH@-nk-IC9CkiP+z zUrA@REn}t@jB((jv&W=(909dS!+Lap66zbZ+C$s4d+`~CS~)0u*1?|3F((271I4$P zf}mm6L^nzo97n4%>sX9j7F{?WN+A3NvPONXxLEWJ#C+#a66B`(fm0p5v%Y7kkWv%O zaN#9H?CPYwXGTM$H1f;S>~87E9m7zCk~TpsO3C}W{aLt0*s-NWB<$v4)sf}F&p6!=3)@nw9O(O06Y~n+b*L8hBti3=w(Z6w$Is`dJiW>cm#%f z)wPhuA~ulDAR@x|bJ8^6uqFw!k~gP!KUyxmOmxdKmikRRa)52atnVkZf5~E3M%@z5 k>tKgY{Qv0-J2EpT;`G2grPB|u;1Ue3&#vv7U+wwg9}hn*FaQ7m literal 0 HcmV?d00001 diff --git a/tests/PinnedH2O_3DOF/aux/coords_1.00_1.00_0.0.in b/tests/PinnedH2O_3DOF/aux/coords_1.00_1.00_0.0.in new file mode 100644 index 00000000..46f5681d --- /dev/null +++ b/tests/PinnedH2O_3DOF/aux/coords_1.00_1.00_0.0.in @@ -0,0 +1,3 @@ +O1 1 0.00 0.00 0.00 0 +H1 2 1.12 1.45 0.00 1 +H2 2 1.12 -1.45 0.00 1 diff --git a/tests/PinnedH2O_3DOF/aux/coords_1.02_0.98_2.0.in b/tests/PinnedH2O_3DOF/aux/coords_1.02_0.98_2.0.in new file mode 100644 index 00000000..4c440443 --- /dev/null +++ b/tests/PinnedH2O_3DOF/aux/coords_1.02_0.98_2.0.in @@ -0,0 +1,3 @@ +O1 1 0.00 0.00 0.00 0 +H1 2 1.12 1.50 0.00 1 +H2 2 1.07 -1.44 0.00 1 diff --git a/tests/PinnedH2O_3DOF/aux/get_ROM_table.py b/tests/PinnedH2O_3DOF/aux/get_ROM_table.py new file mode 100644 index 00000000..5fea1e38 --- /dev/null +++ b/tests/PinnedH2O_3DOF/aux/get_ROM_table.py @@ -0,0 +1,31 @@ +import subprocess +import re + +bondlength_num_increments = (2, 5, 10) +bondangle_num_increments = (2, 5, 10) + +pattern = r"For energy fraction: \d+\.\d+, take first (\d+) of \d+ basis vectors" + +print("\\begin{tabular}{|c|c||c|c|c|c|c|c|c|}") +print("\\hline") +print("$N_L$ & $N_\\theta$ & $\\varepsilon = 10^{-1}$ & $\\varepsilon = 10^{-2}$ & $\\varepsilon = 10^{-3}$ & $\\varepsilon = 10^{-4}$ & $\\varepsilon = 10^{-5}$ & Snapshots \\\\") +print("\\hline") + +for _, N_L in enumerate(bondlength_num_increments): + for _, N_theta in enumerate(bondangle_num_increments): + snapshots = 2*(N_L+1)*(N_L+2)*(N_theta+1) + grep_command = f"grep 'take first' basis_{N_L}_{N_theta}_PinnedH2O_3DOF.out" + result = subprocess.run(grep_command, shell=True, capture_output=True, text=True) + matches = re.findall(pattern, result.stdout) + energy_fractions = { + "0.9": matches[0], + "0.99": matches[1], + "0.999": matches[2], + "0.9999": matches[3], + "0.99999": matches[4], + } + line = f"{N_L} & {N_theta} & {energy_fractions['0.9']} & {energy_fractions['0.99']} & {energy_fractions['0.999']} & {energy_fractions['0.9999']} & {energy_fractions['0.99999']} & {snapshots} \\\\" + print(line) + +print("\\hline") +print("\\end{tabular}") diff --git a/tests/PinnedH2O_3DOF/aux/plot_PinnedH2O_3DOF.py b/tests/PinnedH2O_3DOF/aux/plot_PinnedH2O_3DOF.py new file mode 100644 index 00000000..e679c691 --- /dev/null +++ b/tests/PinnedH2O_3DOF/aux/plot_PinnedH2O_3DOF.py @@ -0,0 +1,69 @@ +import matplotlib +import matplotlib.pyplot as plt +import numpy as np + +ref_bondlength = 1.83 +ref_bondangle = 104.5 + +# factors and increments for bond lengths and bond angle +bondlength_factor = np.linspace(0.95, 1.05, 3) +bondangle_increment = np.linspace(-5, 5, 3) + +fig, ax = plt.subplots() + +for d_bondangle in bondangle_increment: + bondangle = ref_bondangle + d_bondangle + x = ref_bondlength * np.cos(np.radians(bondangle / 2)) + y = ref_bondlength * np.sin(np.radians(bondangle / 2)) + for i, f_bondlength1 in enumerate(bondlength_factor): + H1_x = f_bondlength1*x + H1_y = f_bondlength1*y + ax.plot(H1_x, H1_y, 'ro', markersize=8, alpha=0.1) + for f_bondlength2 in bondlength_factor[:(i+1)]: + H2_x = f_bondlength2*x + H2_y = -f_bondlength2*y + ax.plot(H2_x, H2_y, 'bo', markersize=8, alpha=0.1) + +ref_bondangle = np.radians(ref_bondangle) +x = ref_bondlength * np.cos(ref_bondangle / 2) +y = ref_bondlength * np.sin(ref_bondangle / 2) + +ax.plot([0, x], [0, y], 'r-', lw=2) +ax.plot([0, x], [0, -y], 'b-', lw=2) +ax.plot(0, 0, 'ko', markersize=8) +ax.plot(x, y, 'ro', markersize=8) +ax.plot(x, -y, 'bo', markersize=8) + +arc1 = matplotlib.patches.Arc(xy=(0, 0), + width=0.5, + height=0.5, + angle=0, + theta1=0, + theta2=ref_bondangle/2, + color='red', + linewidth=2) +ax.add_patch(arc1) +ax.text(0.4, 0.25, r"$\frac{\theta}{2}$", ha='center', va='center', fontsize=12, color='red') + +arc2 = matplotlib.patches.Arc(xy=(0, 0), + width=0.5, + height=0.5, + angle=0, + theta1=-ref_bondangle/2, + theta2=0, + color='blue', + linewidth=2) +ax.add_patch(arc2) +ax.text(0.4, -0.25, r"$\frac{\theta}{2}$", ha='center', va='center', fontsize=12, color='blue') + +ax.text(x / 2 - 0.1, y / 2, r"$L_1$", ha='right', color='red') +ax.text(x / 2 - 0.1, -y / 2, r"$L_2$", ha='right', color='blue') + +ax.plot([-2, 2], [0, 0], 'k--', lw=1) +ax.set_xlim(-2.0, 2.0) +ax.set_ylim(-2.0, 2.0) +ax.set_aspect('equal') +ax.set_xlabel("x (Bohr)") +ax.set_ylabel("y (Bohr)") +ax.grid(True) +plt.savefig("PinnedH2O_3DOF.png") diff --git a/tests/PinnedH2O_3DOF/aux/rotation_test.cc b/tests/PinnedH2O_3DOF/aux/rotation_test.cc new file mode 100644 index 00000000..3f2ae1c6 --- /dev/null +++ b/tests/PinnedH2O_3DOF/aux/rotation_test.cc @@ -0,0 +1,192 @@ +#include +#include +#include +#include + +using namespace std; + +double calculate_bondlength(const double atom1[3], const double atom2[3]) +{ + return sqrt(pow(atom1[0] - atom2[0], 2) + pow(atom1[1] - atom2[1], 2) + + pow(atom1[2] - atom2[2], 2)); +} + +double calculate_bondangle(const double atom1[3], const double atom2[3], + const double atom3[3], bool radian) +{ + double vector1[3] + = { atom1[0] - atom2[0], atom1[1] - atom2[1], atom1[2] - atom2[2] }; + double vector2[3] + = { atom3[0] - atom2[0], atom3[1] - atom2[1], atom3[2] - atom2[2] }; + + double dot_product = vector1[0] * vector2[0] + vector1[1] * vector2[1] + + vector1[2] * vector2[2]; + double magnitude_product + = sqrt(pow(vector1[0], 2) + pow(vector1[1], 2) + pow(vector1[2], 2)) + * sqrt(pow(vector2[0], 2) + pow(vector2[1], 2) + pow(vector2[2], 2)); + double angle = acos(dot_product / magnitude_product); + + if (!radian) + { + angle = angle * 180.0 / M_PI; + } + return angle; +} + +void rotation_matrix(const double axis[3], double angle, double matrix[3][3]) +{ + double cos_theta = cos(angle); + double sin_theta = sin(angle); + double ux = axis[0], uy = axis[1], uz = axis[2]; + + matrix[0][0] = cos_theta + ux * ux * (1 - cos_theta); + matrix[0][1] = ux * uy * (1 - cos_theta) - uz * sin_theta; + matrix[0][2] = ux * uz * (1 - cos_theta) + uy * sin_theta; + + matrix[1][0] = uy * ux * (1 - cos_theta) + uz * sin_theta; + matrix[1][1] = cos_theta + uy * uy * (1 - cos_theta); + matrix[1][2] = uy * uz * (1 - cos_theta) - ux * sin_theta; + + matrix[2][0] = uz * ux * (1 - cos_theta) - uy * sin_theta; + matrix[2][1] = uz * uy * (1 - cos_theta) + ux * sin_theta; + matrix[2][2] = cos_theta + uz * uz * (1 - cos_theta); +} + +void normalize(double vec[3]) +{ + double norm = sqrt(vec[0] * vec[0] + vec[1] * vec[1] + vec[2] * vec[2]); + vec[0] /= norm; + vec[1] /= norm; + vec[2] /= norm; +} + +void cross(const double a[3], const double b[3], double result[3]) +{ + result[0] = a[1] * b[2] - a[2] * b[1]; + result[1] = a[2] * b[0] - a[0] * b[2]; + result[2] = a[0] * b[1] - a[1] * b[0]; +} + +void apply_rotation( + const double matrix[3][3], const double vec[3], double result[3]) +{ + result[0] + = matrix[0][0] * vec[0] + matrix[0][1] * vec[1] + matrix[0][2] * vec[2]; + result[1] + = matrix[1][0] * vec[0] + matrix[1][1] * vec[1] + matrix[1][2] * vec[2]; + result[2] + = matrix[2][0] * vec[0] + matrix[2][1] * vec[1] + matrix[2][2] * vec[2]; +} + +void apply_transpose_rotation( + const double matrix[3][3], const double vec[3], double result[3]) +{ + result[0] + = matrix[0][0] * vec[0] + matrix[1][0] * vec[1] + matrix[2][0] * vec[2]; + result[1] + = matrix[0][1] * vec[0] + matrix[1][1] * vec[1] + matrix[2][1] * vec[2]; + result[2] + = matrix[0][2] * vec[0] + matrix[1][2] * vec[1] + matrix[2][2] * vec[2]; +} + +int main() +{ + double O1[3] = { 0.00, 0.00, 0.00 }; + double H1[3] = { -0.45, -1.48, -0.97 }; + double H2[3] = { -0.45, 1.42, -1.07 }; + + double bondlength1 = calculate_bondlength(H1, O1); + double bondlength2 = calculate_bondlength(H2, O1); + double bondangle = calculate_bondangle(H1, O1, H2, false); + + cout << "Original system" << endl; + cout << "H1 = (" << H1[0] << ", " << H1[1] << ", " << H1[2] << ")" << endl; + cout << "H2 = (" << H2[0] << ", " << H2[1] << ", " << H2[2] << ")" << endl; + cout << "Bondlength of O1-H1 = " << bondlength1 << endl; + cout << "Bondlength of O1-H2 = " << bondlength2 << endl; + cout << "Angle between O1-H1 and O1-H2 = " << bondangle << endl; + + double H1_temp[3], H2_temp[3]; + double H1_rotated[3], H2_rotated[3]; + + double plane_normal[3]; + cross(H2, H1, plane_normal); + normalize(plane_normal); + double target_plane_normal[3] = { 0, 0, 1 }; + double axis_to_align[3]; + cross(plane_normal, target_plane_normal, axis_to_align); + normalize(axis_to_align); + double dot_product = plane_normal[0] * target_plane_normal[0] + + plane_normal[1] * target_plane_normal[1] + + plane_normal[2] * target_plane_normal[2]; + double angle_to_align = acos(min(max(dot_product, -1.0), 1.0)); + double out_of_plane_rotation_matrix[3][3]; + rotation_matrix( + axis_to_align, angle_to_align, out_of_plane_rotation_matrix); + apply_rotation(out_of_plane_rotation_matrix, H1, H1_temp); + apply_rotation(out_of_plane_rotation_matrix, H2, H2_temp); + + double theta1 = atan2(H1_temp[1], H1_temp[0]); + double planar_rotation_angle = -theta1 + bondangle / 360.0 * M_PI; + double planar_rotation_matrix[3][3]; + rotation_matrix( + target_plane_normal, planar_rotation_angle, planar_rotation_matrix); + apply_rotation(planar_rotation_matrix, H1_temp, H1_rotated); + apply_rotation(planar_rotation_matrix, H2_temp, H2_rotated); + + if (bondlength1 < bondlength2) + { + swap(H1_rotated, H2_rotated); + H1_rotated[1] *= -1.0; + H2_rotated[1] *= -1.0; + } + + double bondlength1_rotated = calculate_bondlength(H1_rotated, O1); + double bondlength2_rotated = calculate_bondlength(H2_rotated, O1); + double bondangle_rotated + = calculate_bondangle(H1_rotated, O1, H2_rotated, false); + + cout << "Reference system (in z=0 plane, symmetric about x=0 axis, with " + "longer bondlength in Q1)" + << endl; + cout << "H1 = (" << H1_rotated[0] << ", " << H1_rotated[1] << ", " + << H1_rotated[2] << ")" << endl; + cout << "H2 = (" << H2_rotated[0] << ", " << H2_rotated[1] << ", " + << H2_rotated[2] << ")" << endl; + cout << "Bondlength of O1-H1 = " << bondlength1_rotated << endl; + cout << "Bondlength of O1-H2 = " << bondlength2_rotated << endl; + cout << "Angle between O1-H1 and O1-H2 = " << bondangle_rotated << endl; + + double H1_restored[3], H2_restored[3]; + + if (bondlength1 < bondlength2) + { + swap(H1_rotated, H2_rotated); + H1_rotated[1] *= -1.0; + H2_rotated[1] *= -1.0; + } + + apply_transpose_rotation(planar_rotation_matrix, H1_rotated, H1_temp); + apply_transpose_rotation(planar_rotation_matrix, H2_rotated, H2_temp); + + apply_transpose_rotation( + out_of_plane_rotation_matrix, H1_temp, H1_restored); + apply_transpose_rotation( + out_of_plane_rotation_matrix, H2_temp, H2_restored); + + double bondlength1_restored = calculate_bondlength(H1_restored, O1); + double bondlength2_restored = calculate_bondlength(H2_restored, O1); + double bondangle_restored + = calculate_bondangle(H1_restored, O1, H2_restored, false); + + cout << "Restored system" << endl; + cout << "H1 = (" << H1_restored[0] << ", " << H1_restored[1] << ", " + << H1_restored[2] << ")" << endl; + cout << "H2 = (" << H2_restored[0] << ", " << H2_restored[1] << ", " + << H2_restored[2] << ")" << endl; + cout << "Bondlength of O1-H1 = " << bondlength1_restored << endl; + cout << "Bondlength of O1-H2 = " << bondlength2_restored << endl; + cout << "Angle between O1-H1 and O1-H2 = " << bondangle_restored << endl; + + return 0; +} diff --git a/tests/PinnedH2O_3DOF/aux/rotation_test.py b/tests/PinnedH2O_3DOF/aux/rotation_test.py new file mode 100644 index 00000000..41e20296 --- /dev/null +++ b/tests/PinnedH2O_3DOF/aux/rotation_test.py @@ -0,0 +1,99 @@ +import numpy as np + +O1 = np.array([0.00, 0.00, 0.00]) +H1 = np.array([-0.45, -1.48, -0.97]) +H2 = np.array([-0.45, 1.42, -1.07]) + +def calculate_bondlength(atom1, atom2): + return np.linalg.norm(atom1 - atom2) + +def calculate_bondangle(atom1, atom2, atom3, radian): + vector1 = atom1 - atom2 + vector2 = atom3 - atom2 + dot_product = np.dot(vector1, vector2) + magnitude_product = np.linalg.norm(vector1) * np.linalg.norm(vector2) + angle = np.arccos(dot_product / magnitude_product) + if not radian: + angle = np.degrees(angle) + return angle + +bondlength1 = calculate_bondlength(H1, O1) +bondlength2 = calculate_bondlength(H2, O1) +bondangle = calculate_bondangle(H1, O1, H2, False) + +print('Original system') +print(f'H1 = {H1}') +print(f'H2 = {H2}') +print(f'Bondlength of O1-H1 = {bondlength1}') +print(f'Bondlength of O1-H2 = {bondlength2}') +print(f'Angle between O1-H1 and O1-H2 = {bondangle}') + +def rotation_matrix(axis, angle): + cos_theta = np.cos(angle) + sin_theta = np.sin(angle) + ux, uy, uz = axis + return np.array([ + [cos_theta + ux**2 * (1 - cos_theta), ux * uy * (1 - cos_theta) - uz * sin_theta, ux * uz * (1 - cos_theta) + uy * sin_theta], + [uy * ux * (1 - cos_theta) + uz * sin_theta, cos_theta + uy**2 * (1 - cos_theta), uy * uz * (1 - cos_theta) - ux * sin_theta], + [uz * ux * (1 - cos_theta) - uy * sin_theta, uz * uy * (1 - cos_theta) + ux * sin_theta, cos_theta + uz**2 * (1 - cos_theta)] + ]) + +H1_rotated, H2_rotated = H1, H2 + +plane_normal = np.cross(H2, H1) +plane_normal = plane_normal / np.linalg.norm(plane_normal) + +target_plane_normal = np.array([0, 0, 1]) +axis_to_align = np.cross(plane_normal, target_plane_normal) +axis_to_align /= np.linalg.norm(axis_to_align) +angle_to_align = np.arccos(np.clip(np.dot(plane_normal, target_plane_normal), -1.0, 1.0)) +out_of_plane_rotation_matrix = rotation_matrix(axis_to_align, angle_to_align) +H1_rotated = np.dot(out_of_plane_rotation_matrix, H1) +H2_rotated = np.dot(out_of_plane_rotation_matrix, H2) + +theta1 = np.arctan2(H1_rotated[1], H1_rotated[0]) +theta2 = np.arctan2(H2_rotated[1], H2_rotated[0]) +planar_rotation_angle = -theta1 + np.radians(bondangle) / 2 +planar_rotation_matrix = rotation_matrix(target_plane_normal, planar_rotation_angle) +H1_rotated = np.dot(planar_rotation_matrix, H1_rotated) +H2_rotated = np.dot(planar_rotation_matrix, H2_rotated) + +if bondlength1 < bondlength2: + H1_rotated, H2_rotated = H2_rotated, H1_rotated + H1_rotated[1] *= -1 + H2_rotated[1] *= -1 + +bondlength1_rotated = calculate_bondlength(H1_rotated, O1) +bondlength2_rotated = calculate_bondlength(H2_rotated, O1) +bondangle_rotated = calculate_bondangle(H1_rotated, O1, H2_rotated, False) + +print('Reference system (z=0 plane, symmetric about x=0 axis, with longer bondlength in Q1)') +print(f'H1 = {H1_rotated}') +print(f'H2 = {H2_rotated}') +print(f'Bondlength of O1-H1 = {bondlength1_rotated}') +print(f'Bondlength of O1-H2 = {bondlength2_rotated}') +print(f'Angle between O1-H1 and O1-H2 = {bondangle_rotated}') + +H1_restored, H2_restored = H1_rotated, H2_rotated + +if bondlength1 < bondlength2: + H1_rotated, H2_rotated = H2_rotated, H1_rotated + H1_rotated[1] *= -1 + H2_rotated[1] *= -1 + +H1_restored = np.dot(planar_rotation_matrix.T, H1_restored) +H2_restored = np.dot(planar_rotation_matrix.T, H2_restored) + +H1_restored = np.dot(out_of_plane_rotation_matrix.T, H1_restored) +H2_restored = np.dot(out_of_plane_rotation_matrix.T, H2_restored) + +bondlength1_restored = calculate_bondlength(H1_restored, O1) +bondlength2_restored = calculate_bondlength(H2_restored, O1) +bondangle_restored = calculate_bondangle(H1_restored, O1, H2_restored, False) + +print('Restored system') +print(f'H1 = {H1_restored}') +print(f'H2 = {H2_restored}') +print(f'Bondlength of O1-H1 = {bondlength1_restored}') +print(f'Bondlength of O1-H2 = {bondlength2_restored}') +print(f'Angle between O1-H1 and O1-H2 = {bondangle_restored}') diff --git a/tests/PinnedH2O_3DOF/compare_energy_and_forces.py b/tests/PinnedH2O_3DOF/compare_energy_and_forces.py new file mode 100644 index 00000000..b11dbe91 --- /dev/null +++ b/tests/PinnedH2O_3DOF/compare_energy_and_forces.py @@ -0,0 +1,233 @@ +import numpy as np +import os +import matplotlib.pyplot as plt +import matplotlib.ticker as ticker +import argparse + +parser = argparse.ArgumentParser(description="Calculate and plot differences in energies and forces.") + +parser.add_argument("--N_l", type=int, default=2, help="Sampling frequency of bond length points") +parser.add_argument("--N_theta", type=int, default=2, help="Sampling frequency of bond angle points") +parser.add_argument("--rdim", type=int, default=18, help="Dimension of the ROM basis") + +args = parser.parse_args() + +N_l = args.N_l +N_theta = args.N_theta +rdim = args.rdim + +bondlength_min = 0.95 +bondlength_max = 1.05 +bondlength_num_increments = 10 + +bondangle_min = -5.0 +bondangle_max = 5.0 +bondangle_num_increments = 10 + +output_dir = f'results_{N_l}_{N_theta}_{rdim}' + +def process_data(s_l1, s_l2, s_theta, N_l, N_theta, rdim): + try: + data_dir = f'data/{s_l1}_{s_l2}_{s_theta}' + offline_file = os.path.join(data_dir, 'offline_PinnedH2O.out') + log_file = f'{output_dir}/{s_l1}_{s_l2}_{s_theta}.log' + + f_O1_fom = None + f_H1_fom = None + f_H2_fom = None + + if not os.path.exists(offline_file): + print(f"Error: File not found: {offline_file}") + return None + + with open(offline_file, 'r') as f: + eks_lines = [line for line in f if " SC ENERGY [Ha]" in line] + if eks_lines: + Eks_fom = float(eks_lines[-1].split()[-1]) + else: + print(f"Error: Could not find 'SC ENERGY [Ha]' line in {offline_file}") + return None + + with open(offline_file, 'r') as f: + for line in f: + if "O1" in line: + f_O1_fom = np.array(line.split()[6:9], dtype=float) + elif "H1" in line: + f_H1_fom = np.array(line.split()[5:8], dtype=float) + elif "H2" in line: + f_H2_fom = np.array(line.split()[5:8], dtype=float) + + if f_O1_fom is not None and f_H1_fom is not None and f_H2_fom is not None: + break # Exit the loop once we found all lines + + if f_O1_fom is None or f_H1_fom is None or f_H2_fom is None: + print(f"Error: Could not find O1, H1, or H2 lines in {offline_file}") + return None + + f_H2_rom = None + f_O1_rom = None + f_H1_rom = None + + if not os.path.exists(log_file): + print(f"Error: File not found: {log_file}") + return None + + with open(log_file, 'r') as f: + lines = f.readlines() + + eks_index = -1 + for i in range(len(lines) - 1, -1, -1): # Search from the end + if "Eks:" in lines[i]: + eks_index = i + break + + if eks_index == -1: + print(f"Error: Could not find 'Eks:' line in {log_file}") + return None + + Eks_rom = float(lines[eks_index].split()[-1]) # Get the last element + + force_index = -1 + for i in range(len(lines) - 1, -1, -1): # Search from the end + if "Forces:" in lines[i]: + force_index = i + break + + if force_index == -1: + print(f"Error: Could not find 'Forces:' line in {log_file}") + return None + + if force_index + 3 >= len(lines): + print(f"Error: Not enough lines after 'Forces:' in {log_file}") + return None + + f_H2_rom = np.array(lines[force_index + 1].split(), dtype=float) + f_O1_rom = np.array(lines[force_index + 2].split(), dtype=float) + f_H1_rom = np.array(lines[force_index + 3].split(), dtype=float) + + return Eks_fom, f_O1_fom, f_H1_fom, f_H2_fom, Eks_rom, f_O1_rom, f_H1_rom, f_H2_rom + + except Exception as e: + print(f"An error occurred: {e}") + return None + +Eks_diff_reproductive = [] +f_O1_diff_reproductive = [] +f_H1_diff_reproductive = [] +f_H2_diff_reproductive = [] + +Eks_diff_predictive = [] +f_O1_diff_predictive = [] +f_H1_diff_predictive = [] +f_H2_diff_predictive = [] + +for i in range(bondlength_num_increments + 1): + bondlength_one = round(bondlength_min + i * (bondlength_max - bondlength_min) / bondlength_num_increments, 2) + for j in range(i + 1): + bondlength_two = round(bondlength_min + j * (bondlength_max - bondlength_min) / bondlength_num_increments, 2) + for k in range(bondangle_num_increments + 1): + bondangle = round(bondangle_min + k * (bondangle_max - bondangle_min) / bondangle_num_increments, 1) + + s_l1 = f"{bondlength_one:.2f}" + s_l2 = f"{bondlength_two:.2f}" + s_theta = f"{bondangle:.1f}" + tag = f'{s_l1}_{s_l2}_{s_theta}' + + output_filename = f'{output_dir}/energy_and_forces_{tag}.out' + + with open(output_filename, 'w') as outfile: + print("s_l1:", s_l1, file=outfile) + print("s_l2:", s_l2, file=outfile) + print("s_theta:", s_theta, file=outfile) + print("N_l:", N_l, file=outfile) + print("N_theta:", N_theta, file=outfile) + print("rdim:", rdim, file=outfile) + + results = process_data(s_l1, s_l2, s_theta, N_l, N_theta, rdim) + + if results: + Eks_fom, f_O1_fom, f_H1_fom, f_H2_fom, Eks_rom, f_O1_rom, f_H1_rom, f_H2_rom = results + + print("Eks_fom:", Eks_fom, file=outfile) + print("Eks_rom:", Eks_rom, file=outfile) + + print("f_O1_fom:", f_O1_fom, file=outfile) + print("f_O1_rom:", f_O1_rom, file=outfile) + + print("f_H1_fom:", f_H1_fom, file=outfile) + print("f_H1_rom:", f_H1_rom, file=outfile) + + print("f_H2_fom:", f_H2_fom, file=outfile) + print("f_H2_rom:", f_H2_rom, file=outfile) + + def calculate_differences(fom, rom, name): + abs_diff = np.linalg.norm(fom - rom) + rel_diff = abs_diff / np.linalg.norm(fom) if np.linalg.norm(fom)!= 0 else float('inf') + print(f"Absolute difference in {name}:", abs_diff, file=outfile) + print(f"Relative difference in {name}:", rel_diff, file=outfile) + return abs_diff + + Eks_diff = calculate_differences(Eks_fom, Eks_rom, "Eks") + f_O1_diff = calculate_differences(f_O1_fom, f_O1_rom, "f_O1") + f_H1_diff = calculate_differences(f_H1_fom, f_H1_rom, "f_H1") + f_H2_diff = calculate_differences(f_H2_fom, f_H2_rom, "f_H2") + + if i * N_l % bondlength_num_increments == 0 and j * N_l % bondlength_num_increments == 0 and k * N_theta % bondangle_num_increments == 0: + Eks_diff_reproductive.append(Eks_diff) + f_O1_diff_reproductive.append(f_O1_diff) + f_H1_diff_reproductive.append(f_H1_diff) + f_H2_diff_reproductive.append(f_H2_diff) + else: + Eks_diff_predictive.append(Eks_diff) + f_O1_diff_predictive.append(f_O1_diff) + f_H1_diff_predictive.append(f_H1_diff) + f_H2_diff_predictive.append(f_H2_diff) + else: + print("Error occurred during data processing. Differences cannot be calculated.", file=outfile) + +def plot_histogram(data, quantity, test_case): + plt.figure(figsize=(8, 6)) + plt.hist(data, bins=20, color='skyblue', edgecolor='black') + + if quantity == "Eks": + quantity_name = "absolute difference in total energy" + elif quantity.startswith("f_"): + quantity_name = f"magnitude of difference in force on {quantity[2:]}" + else: + raise ValueError("Invalid input quantity") + + plt.title(f'Histogram of {quantity_name}') + plt.xlabel('Difference') + plt.ylabel('Frequency') + + min_val, max_val = np.min(data), np.max(data) + plt.xlim(min_val, max_val) + num_ticks = 8 + xticks = np.linspace(min_val, max_val, num_ticks) + plt.xticks(xticks) + formatter = ticker.ScalarFormatter(useMathText=True) + formatter.set_scientific(True) + formatter.set_powerlimits((0, 0)) + plt.gca().xaxis.set_major_formatter(formatter) + + total_count = len(data) + mean_val = np.mean(data) + max_val = np.max(data) + stats_text = (f"Total {test_case} cases: {total_count}\n" + f"Mean: {mean_val:.3e}") + plt.text(0.95, 0.95, stats_text, transform=plt.gca().transAxes, + fontsize=12, verticalalignment='top', horizontalalignment='right', + bbox=dict(facecolor='white', alpha=0.7, edgecolor='black')) + + plt.tight_layout() + plt.savefig(f"{output_dir}/{quantity}_difference_histogram_{test_case}.png") + +plot_histogram(Eks_diff_reproductive, "Eks", "reproductive") +plot_histogram(f_O1_diff_reproductive, "f_O1", "reproductive") +plot_histogram(f_H1_diff_reproductive, "f_H1", "reproductive") +plot_histogram(f_H2_diff_reproductive, "f_H2", "reproductive") + +plot_histogram(Eks_diff_predictive, "Eks", "predictive") +plot_histogram(f_O1_diff_predictive, "f_O1", "predictive") +plot_histogram(f_H1_diff_predictive, "f_H1", "predictive") +plot_histogram(f_H2_diff_predictive, "f_H2", "predictive") diff --git a/tests/PinnedH2O_3DOF/coords_rotate1.in b/tests/PinnedH2O_3DOF/coords_rotate1.in new file mode 100644 index 00000000..7498ff55 --- /dev/null +++ b/tests/PinnedH2O_3DOF/coords_rotate1.in @@ -0,0 +1,3 @@ +O1 1 0.00000 0.00000 0.00 0 +H1 2 -1.08248 -1.47039 0.00 1 +H2 2 -1.14768 1.43060 0.00 1 diff --git a/tests/PinnedH2O_3DOF/coords_rotate2.in b/tests/PinnedH2O_3DOF/coords_rotate2.in new file mode 100644 index 00000000..307e170c --- /dev/null +++ b/tests/PinnedH2O_3DOF/coords_rotate2.in @@ -0,0 +1,3 @@ +O1 1 0.00 0.00 0.00 0 +H1 2 -0.45 1.42 -1.07 1 +H2 2 -0.45 -1.48 -0.97 1 diff --git a/tests/PinnedH2O_3DOF/generate_coord.py b/tests/PinnedH2O_3DOF/generate_coord.py new file mode 100644 index 00000000..194c2df7 --- /dev/null +++ b/tests/PinnedH2O_3DOF/generate_coord.py @@ -0,0 +1,31 @@ +import numpy as np +import os + +O1 = np.array([0.00, 0.00, 0.00]) + +ref_bondlength = 1.83 +ref_bondangle = 104.5 + +# factors and increments for bond lengths and bond angle +bondlength_factor = np.linspace(0.95, 1.05, 11) +bondangle_increment = np.linspace(-5, 5, 11) + +# output directory +output_dir = "data" + +# generation +os.makedirs(output_dir, exist_ok=True) + +for d_bondangle in bondangle_increment: + bondangle = ref_bondangle + d_bondangle + x = ref_bondlength * np.cos(np.radians(bondangle / 2)) + y = ref_bondlength * np.sin(np.radians(bondangle / 2)) + for i, f_bondlength1 in enumerate(bondlength_factor): + for f_bondlength2 in bondlength_factor[:(i+1)]: + H1 = np.array([f_bondlength1*x, f_bondlength1*y, 0.0]) + H2 = np.array([f_bondlength2*x, -f_bondlength2*y, 0.0]) + filename = f"{output_dir}/coords_{f_bondlength1:.2f}_{f_bondlength2:.2f}_{d_bondangle}.in" + with open(filename, "w") as file: + file.write(f"O1 1 {O1[0]:.2f} {O1[1]:.2f} {O1[2]:.2f} 0\n") + file.write(f"H1 2 {H1[0]:.2f} {H1[1]:.2f} {H1[2]:.2f} 1\n") + file.write(f"H2 2 {H2[0]:.2f} {H2[1]:.2f} {H2[2]:.2f} 1\n") diff --git a/tests/PinnedH2O_3DOF/job.basis b/tests/PinnedH2O_3DOF/job.basis new file mode 100644 index 00000000..35f564a3 --- /dev/null +++ b/tests/PinnedH2O_3DOF/job.basis @@ -0,0 +1,47 @@ +#!/bin/tcsh +#SBATCH -N 1 +#SBATCH -t 0:10:00 +#SBATCH -p pdebug + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 8 + +set maindir = /p/lustre2/cheung26/mgmol + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = ${maindir}/build_quartz/libROM/build/examples/misc/combine_samples + +set snapshot_files = "" +set bondlength_min = 0.95 +set bondlength_max = 1.05 +set bondlength_num_increments = 2 +set bondangle_min = -5.0 +set bondangle_max = 5.0 +set bondangle_num_increments = 2 + +foreach i (`seq 0 $bondlength_num_increments`) + set bondlength_one = `echo "scale=2; $bondlength_min + $i * ($bondlength_max - $bondlength_min) / ($bondlength_num_increments)" | bc` + set bondlength_one = `printf "%.2f" $bondlength_one` + foreach j (`seq 0 $i`) + set bondlength_two = `echo "scale=2; $bondlength_min + $j * ($bondlength_max - $bondlength_min) / ($bondlength_num_increments)" | bc` + set bondlength_two = `printf "%.2f" $bondlength_two` + foreach k (`seq 0 $bondangle_num_increments`) + set bondangle = `echo "scale=1; $bondangle_min + $k * ($bondangle_max - $bondangle_min) / ($bondangle_num_increments)" | bc` + set bondangle = `printf "%.1f" $bondangle` + echo "bondlength1: $bondlength_one, bondlength2: $bondlength_two, bondangle: $bondangle" + set tag = ${bondlength_one}_${bondlength_two}_${bondangle} + set snapshot_files = "$snapshot_files data/${tag}/orbital_snapshot" + end + end +end +echo "Snapshot files: $snapshot_files" + +set basis_file = "data/PinnedH2O_3DOF_orbitals_basis_${bondlength_num_increments}_${bondangle_num_increments}" +srun -n $ncpus $exe -f $basis_file $snapshot_files > data/basis_${bondlength_num_increments}_${bondangle_num_increments}_PinnedH2O_3DOF.out + +date diff --git a/tests/PinnedH2O_3DOF/job.offline b/tests/PinnedH2O_3DOF/job.offline new file mode 100644 index 00000000..5b4224f0 --- /dev/null +++ b/tests/PinnedH2O_3DOF/job.offline @@ -0,0 +1,59 @@ +#!/bin/tcsh +#SBATCH -N 1 +#SBATCH -t 4:00:00 +#SBATCH -p pbatch + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 8 + +set maindir = /p/lustre2/cheung26/mgmol + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = mgmol-opt + +cp $maindir/install_quartz/bin/$exe . + +set cfg_offline = mgmol_offline.cfg + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +set bondlength_min = 0.95 +set bondlength_max = 1.05 +set bondlength_num_increments = 10 +set bondangle_min = -5.0 +set bondangle_max = 5.0 +set bondangle_num_increments = 10 + +foreach i (`seq 0 $bondlength_num_increments`) + set bondlength_one = `echo "scale=2; $bondlength_min + $i * ($bondlength_max - $bondlength_min) / ($bondlength_num_increments)" | bc` + set bondlength_one = `printf "%.2f" $bondlength_one` + foreach j (`seq 0 $i`) + set bondlength_two = `echo "scale=2; $bondlength_min + $j * ($bondlength_max - $bondlength_min) / ($bondlength_num_increments)" | bc` + set bondlength_two = `printf "%.2f" $bondlength_two` + foreach k (`seq 0 $bondangle_num_increments`) + set bondangle = `echo "scale=1; $bondangle_min + $k * ($bondangle_max - $bondangle_min) / ($bondangle_num_increments)" | bc` + set bondangle = `printf "%.1f" $bondangle` + echo "bondlength1: $bondlength_one, bondlength2: $bondlength_two, bondangle: $bondangle" + set tag = ${bondlength_one}_${bondlength_two}_${bondangle} + if (-e data/${tag}) then + continue + else + mkdir data/${tag} + endif + cp $cfg_offline data/${tag}/${cfg_offline} + mv data/coords_${tag}.in data/${tag}/coords.in + sed -i "s/CUSTOM_TAG/${tag}/g" data/${tag}/${cfg_offline} + srun -n $ncpus $exe -c data/${tag}/${cfg_offline} -i data/${tag}/coords.in > data/${tag}/offline_PinnedH2O.out + end + end +end + +date diff --git a/tests/PinnedH2O_3DOF/job.online b/tests/PinnedH2O_3DOF/job.online new file mode 100644 index 00000000..ea75e59f --- /dev/null +++ b/tests/PinnedH2O_3DOF/job.online @@ -0,0 +1,67 @@ +#!/bin/tcsh +#SBATCH -N 1 +#SBATCH -t 1:00:00 +#SBATCH -p pdebug + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 8 + +set maindir = /p/lustre2/cheung26/mgmol + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = testPinnedH2O_3DOF + +cp $maindir/build_quartz/tests/$exe . + +set cfg_online = mgmol_online.cfg + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +set sample_bondlength_num_increments = 2 +set sample_bondangle_num_increments = 2 +set num_orbital_basis = 34 + +set bondlength_min = 0.95 +set bondlength_max = 1.05 +set bondlength_num_increments = 10 +set bondangle_min = -5.0 +set bondangle_max = 5.0 +set bondangle_num_increments = 10 + +set case = ${sample_bondlength_num_increments}_${sample_bondangle_num_increments}_${num_orbital_basis} +mkdir results_${case} +cp ${cfg_online} ${case}_${cfg_online} + +@ num_empty_orbital = $num_orbital_basis - 4 +sed -i "s/CUSTOM_NE/${num_empty_orbital}/g" ${case}_${cfg_online} +sed -i "s/CUSTOM_NL/${sample_bondlength_num_increments}/g" ${case}_${cfg_online} +sed -i "s/CUSTOM_NT/${sample_bondangle_num_increments}/g" ${case}_${cfg_online} +sed -i "s/CUSTOM_NB/${num_orbital_basis}/g" ${case}_${cfg_online} + +foreach i (`seq 0 $bondlength_num_increments`) + set bondlength_one = `echo "scale=2; $bondlength_min + $i * ($bondlength_max - $bondlength_min) / ($bondlength_num_increments)" | bc` + set bondlength_one = `printf "%.2f" $bondlength_one` + foreach j (`seq 0 $i`) + set bondlength_two = `echo "scale=2; $bondlength_min + $j * ($bondlength_max - $bondlength_min) / ($bondlength_num_increments)" | bc` + set bondlength_two = `printf "%.2f" $bondlength_two` + foreach k (`seq 0 $bondangle_num_increments`) + set bondangle = `echo "scale=1; $bondangle_min + $k * ($bondangle_max - $bondangle_min) / ($bondangle_num_increments)" | bc` + set bondangle = `printf "%.1f" $bondangle` + echo "bondlength1: $bondlength_one, bondlength2: $bondlength_two, bondangle: $bondangle" + set tag = ${bondlength_one}_${bondlength_two}_${bondangle} + srun -p pdebug -n $ncpus $exe -c ${case}_${cfg_online} -i /usr/workspace/nlrom/MGmol/PinnedH2O_3DOF/data_${ncpus}/${tag}/coords.in >& results_${case}/${tag}.log + end + end +end + +mv ${case}_${cfg_online} results_${case} + +date diff --git a/tests/PinnedH2O_3DOF/job.online_rotate b/tests/PinnedH2O_3DOF/job.online_rotate new file mode 100644 index 00000000..a2065b72 --- /dev/null +++ b/tests/PinnedH2O_3DOF/job.online_rotate @@ -0,0 +1,36 @@ +#!/bin/tcsh +#SBATCH -N 1 +#SBATCH -t 0:10:00 +#SBATCH -p pdebug + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 8 + +set maindir = /p/lustre2/cheung26/mgmol + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +rm -rf snapshot0_* + +set coord_file = coords_rotate2.in + +set exe = mgmol-opt +set cfg_online = mgmol_ref_rotate.cfg +cp $maindir/install_quartz/bin/$exe . +srun -p pdebug -n $ncpus $exe -c $cfg_online -i $coord_file + +set exe = testPinnedH2O_3DOF +set cfg_online = mgmol_online_rotate.cfg +cp $maindir/build_quartz/tests/$exe . +srun -p pdebug -n $ncpus $exe -c $cfg_online -i $coord_file + +date diff --git a/tests/PinnedH2O_3DOF/mgmol_offline.cfg b/tests/PinnedH2O_3DOF/mgmol_offline.cfg new file mode 100644 index 00000000..7b2dbcfa --- /dev/null +++ b/tests/PinnedH2O_3DOF/mgmol_offline.cfg @@ -0,0 +1,30 @@ +verbosity=1 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=quench +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +[Restart] +output_level=4 +output_filename=CUSTOM_TAG +[ROM.offline] +save_librom_snapshot=true diff --git a/tests/PinnedH2O_3DOF/mgmol_online.cfg b/tests/PinnedH2O_3DOF/mgmol_online.cfg new file mode 100644 index 00000000..6f66067d --- /dev/null +++ b/tests/PinnedH2O_3DOF/mgmol_online.cfg @@ -0,0 +1,36 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=quench +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +nempty=CUSTOM_NE +[Restart] +output_level=4 +[DensityMatrix] +solver=MVP +nb_inner_it=100 +[ROM.offline] +basis_file=/usr/workspace/nlrom/MGmol/PinnedH2O_3DOF/data_8/PinnedH2O_3DOF_orbitals_basis_CUSTOM_NL_CUSTOM_NT +[ROM.basis] +compare_md=false +number_of_orbital_basis=CUSTOM_NB diff --git a/tests/PinnedH2O_3DOF/mgmol_online_rotate.cfg b/tests/PinnedH2O_3DOF/mgmol_online_rotate.cfg new file mode 100644 index 00000000..8c598248 --- /dev/null +++ b/tests/PinnedH2O_3DOF/mgmol_online_rotate.cfg @@ -0,0 +1,36 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=quench +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +nempty=30 +[Restart] +output_level=4 +[DensityMatrix] +solver=MVP +nb_inner_it=100 +[ROM.offline] +basis_file=/usr/workspace/nlrom/MGmol/PinnedH2O_3DOF/data_8/PinnedH2O_3DOF_orbitals_basis_2_2 +[ROM.basis] +compare_md=false +number_of_orbital_basis=34 diff --git a/tests/PinnedH2O_3DOF/mgmol_ref_rotate.cfg b/tests/PinnedH2O_3DOF/mgmol_ref_rotate.cfg new file mode 100644 index 00000000..96be6b8d --- /dev/null +++ b/tests/PinnedH2O_3DOF/mgmol_ref_rotate.cfg @@ -0,0 +1,27 @@ +verbosity=1 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx=64 +ny=64 +nz=64 +[Domain] +ox=-6. +oy=-6. +oz=-6. +lx=12. +ly=12. +lz=12. +[Potentials] +pseudopotential=pseudo.O_ONCV_PBE_SG15 +pseudopotential=pseudo.H_ONCV_PBE_SG15 +[Run] +type=quench +[Quench] +max_steps=100 +atol=1.e-8 +[Orbitals] +initial_type=Random +initial_width=2. +[Restart] +output_level=4 diff --git a/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc b/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc new file mode 100644 index 00000000..8d84309d --- /dev/null +++ b/tests/PinnedH2O_3DOF/testPinnedH2O_3DOF.cc @@ -0,0 +1,258 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "Control.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "PinnedH2O.h" +#include "mgmol_run.h" + +#ifdef MGMOL_HAS_LIBROM +#include "librom.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +namespace po = boost::program_options; + +int main(int argc, char** argv) +{ + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Initialization failed!!!" << std::endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + + MPI_Comm comm = MPI_COMM_WORLD; + + /* + * Initialize general things, like magma, openmp, IO, ... + */ + mgmol_init(comm); + + /* + * read runtime parameters + */ + std::string input_filename(""); + std::string lrs_filename; + std::string constraints_filename(""); + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // read from PE0 only + if (MPIdata::onpe0) + { + read_config(argc, argv, vm, input_filename, lrs_filename, + constraints_filename, total_spin, with_spin); + } + + MGmol_MPI::setup(comm, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + /* + * Setup control struct with run time parameters + */ + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_filename); + mmpi.bcastGlobal(lrs_filename); + + // Enter main scope + { + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Construct MGmol object..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + MGmolInterface* mgmol = new MGmol>( + global_comm, *MPIdata::sout, input_filename, lrs_filename, + constraints_filename); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "MGmol setup..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + mgmol->setup(); + + if (MPIdata::onpe0) + { + std::cout << "-------------------------" << std::endl; + std::cout << "Setup done..." << std::endl; + std::cout << "-------------------------" << std::endl; + } + + // here we just use the atomic positions read in and used + // to initialize MGmol + std::vector positions; + mgmol->getAtomicPositions(positions); + std::vector anumbers; + mgmol->getAtomicNumbers(anumbers); + if (MPIdata::onpe0) + { + std::cout << "Positions:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + } + + // rotate the molecule to the reference coordinate system + PinnedH2O H2O_molecule; + H2O_molecule.rotate(positions, anumbers); + + Mesh* mymesh = Mesh::instance(); + const pb::Grid& mygrid = mymesh->grid(); + const pb::PEenv& myPEenv = mymesh->peenv(); + + // compute energy and forces again with projected problem onto ROM + // subspace + const int rdim = ct.getROMOptions().num_orbbasis; + if (rdim != ct.numst) + { + std::cerr << "The number of functions in the ROM basis file, " + << rdim << " is not equal to ct.numst, " << ct.numst + << std::endl; + MPI_Abort(mmpi.commSameSpin(), 0); + } + + std::shared_ptr projmatrices + = mgmol->getProjectedMatrices(); + + ExtendedGridOrbitals orbitals("new_orbitals", mygrid, + mymesh->subdivx(), ct.numst, ct.bcWF, projmatrices.get(), nullptr, + nullptr, nullptr, nullptr); + + orbitals.set(ct.getROMOptions().basis_file, ct.numst); + orbitals.orthonormalizeLoewdin(); + orbitals.setDataWithGhosts(true); + + // set the iterative index to 1 to differentiate it from first instance + // in MGmol initial() function. This is not very clean and could be + // better designed, but works for now + orbitals.setIterativeIndex(10); + + // set initial DM with uniform occupations + projmatrices->setDMuniform(ct.getNelSpin()); + projmatrices->printDM(std::cout); + + // + // evaluate energy and forces with ROM bases just read + // + std::vector forces; + double eks = mgmol->evaluateDMandEnergyAndForces( + &orbitals, positions, anumbers, forces); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Eks: " << eks << std::endl; + std::cout << "Positions in the reference domain:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + + std::cout << "Forces in the reference domain:" << std::endl; + ita = anumbers.begin(); + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + } + + // rotate the forces to the original coordinate system + H2O_molecule.transpose_rotate(positions, anumbers, forces); + + // print out results + if (MPIdata::onpe0) + { + std::cout << "Positions:" << std::endl; + std::vector::iterator ita = anumbers.begin(); + for (std::vector::iterator it = positions.begin(); + it != positions.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + std::cout << "Forces:" << std::endl; + ita = anumbers.begin(); + for (std::vector::iterator it = forces.begin(); + it != forces.end(); it += 3) + { + std::cout << *ita; + for (int i = 0; i < 3; i++) + std::cout << " " << *(it + i); + std::cout << std::endl; + ita++; + } + } + + delete mgmol; + + } // close main scope + + mgmol_finalize(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + } + + time_t tt; + time(&tt); + if (onpe0) std::cout << " Run ended at " << ctime(&tt) << std::endl; + + return 0; +} +#endif // MGMOL_HAS_LIBROM diff --git a/tests/PinnedH2O_3DOF/transfer_nlrom.sh b/tests/PinnedH2O_3DOF/transfer_nlrom.sh new file mode 100644 index 00000000..62824de2 --- /dev/null +++ b/tests/PinnedH2O_3DOF/transfer_nlrom.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +ncpus=8 +target_dir="/usr/workspace/nlrom/MGmol/PinnedH2O_3DOF/data_${ncpus}" + +if [ ! -d $target_dir ]; then + mkdir -p $target_dir +fi + +for dir in data/*/; do + dirname=$(basename "$dir") + new_dir="$target_dir/${dirname}" + if [ ! -d $new_dir ]; then + mkdir -p $new_dir + fi + mv -f $dir/* $new_dir/ + rm -rf $dir +done + +rm -rf data diff --git a/tests/ROM/test_rom_poisson/carbyne.in b/tests/ROM/test_rom_poisson/carbyne.in new file mode 100644 index 00000000..9f4e975a --- /dev/null +++ b/tests/ROM/test_rom_poisson/carbyne.in @@ -0,0 +1,14 @@ +H00 1 -0.0000 -0.0000 15.2674 +C01 2 -0.0000 0.0000 13.2519 +C02 2 -0.0000 0.0000 10.9495 +C03 2 -0.0000 -0.0000 8.4221 +C04 2 0.0000 0.0000 6.0897 +C05 2 -0.0000 0.0000 3.5892 +C06 2 -0.0000 -0.0000 1.2470 +C07 2 0.0000 -0.0000 -1.2469 +C08 2 0.0000 -0.0000 -3.5891 +C09 2 -0.0000 -0.0000 -6.0897 +C10 2 -0.0000 0.0000 -8.4221 +C11 2 0.0000 -0.0000 -10.9495 +C12 2 0.0000 0.0000 -13.2520 +H13 1 0.0000 0.0000 -15.2675 diff --git a/tests/ROM/test_rom_poisson/carbyne.ion.cfg b/tests/ROM/test_rom_poisson/carbyne.ion.cfg new file mode 100644 index 00000000..d98232ad --- /dev/null +++ b/tests/ROM/test_rom_poisson/carbyne.ion.cfg @@ -0,0 +1,63 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx= 96 +ny= 96 +nz= 192 +[Domain] +ox= -10. +oy= -10. +oz= -20. +lx= 20. +ly= 20. +lz= 40. +[Poisson] +FDtype=4th +#max_steps_initial=99 +#max_steps=99 +[Potentials] +pseudopotential=pseudo.H_ONCV_PBE_SG15 +pseudopotential=pseudo.C_ONCV_PBE_SG15 +[Run] +#type=QUENCH +type=MD +[Quench] +#solver=PR +max_steps=300 +atol=1.e-8 +[MD] +num_steps=3000 +dt=40. +print_interval=1 +[Orbitals] +initial_type=Fourier +nempty=10 +temperature=300. +[ProjectedMatrices] +solver=exact +[DensityMatrix] +solver=MVP +nb_inner_it=1 + +[Thermostat] +type=Langevin +temperature=300. +relax_time=1000. + +[Restart] +#input_filename=init_cond_144proc +#input_level=4 +output_level=4 +interval=1 + +[ROM] +stage=test_ion + +[ROM.offline] +restart_filefmt=case-300K/snapshot%05d +restart_min_idx=800 +restart_max_idx=1999 +basis_file=basis_300K_2/test_300K +variable=potential + diff --git a/tests/ROM/test_rom_poisson/carbyne.poisson.cfg b/tests/ROM/test_rom_poisson/carbyne.poisson.cfg new file mode 100644 index 00000000..6aac926c --- /dev/null +++ b/tests/ROM/test_rom_poisson/carbyne.poisson.cfg @@ -0,0 +1,63 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx= 96 +ny= 96 +nz= 192 +[Domain] +ox= -10. +oy= -10. +oz= -20. +lx= 20. +ly= 20. +lz= 40. +[Poisson] +FDtype=4th +#max_steps_initial=99 +#max_steps=99 +[Potentials] +pseudopotential=pseudo.H_ONCV_PBE_SG15 +pseudopotential=pseudo.C_ONCV_PBE_SG15 +[Run] +#type=QUENCH +type=MD +[Quench] +#solver=PR +max_steps=300 +atol=1.e-8 +[MD] +num_steps=3000 +dt=40. +print_interval=1 +[Orbitals] +initial_type=Fourier +nempty=10 +temperature=300. +[ProjectedMatrices] +solver=exact +[DensityMatrix] +solver=MVP +nb_inner_it=1 + +[Thermostat] +type=Langevin +temperature=300. +relax_time=1000. + +[Restart] +#input_filename=init_cond_144proc +#input_level=4 +output_level=4 +interval=1 + +[ROM] +stage=test_poisson + +[ROM.offline] +restart_filefmt=case-300K/snapshot%05d +restart_min_idx=800 +restart_max_idx=1999 +basis_file=basis_300K_2/test_300K +variable=potential + diff --git a/tests/RestartEnergyAndForces/job.test b/tests/RestartEnergyAndForces/job.test new file mode 100644 index 00000000..89b4865f --- /dev/null +++ b/tests/RestartEnergyAndForces/job.test @@ -0,0 +1,28 @@ +#!/bin/tcsh +#SBATCH -N 2 +#SBATCH -t 1:00:00 +#SBATCH -p pbatch + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 2 + +set maindir = /p/lustre2/cheung26/mgmol-20241220 + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = testRestartEnergyAndForces + +cp $maindir/build_quartz/tests/$exe . + +ln -s -f $maindir/potentials/pseudo.O_ONCV_PBE_SG15 . +ln -s -f $maindir/potentials/pseudo.H_ONCV_PBE_SG15 . + +source $maindir/scripts/modules.quartz + +python3 test.py srun -n $ncpus ${maindir}/install_quartz/bin/mgmol-opt ${exe} mgmol.cfg restart.cfg h2o.xyz . + +date diff --git a/tests/WFEnergyAndForces/job.test b/tests/WFEnergyAndForces/job.test new file mode 100644 index 00000000..7a9e49ee --- /dev/null +++ b/tests/WFEnergyAndForces/job.test @@ -0,0 +1,27 @@ +#!/bin/tcsh +#SBATCH -N 2 +#SBATCH -t 1:00:00 +#SBATCH -p pbatch + +date + +setenv OMP_NUM_THREADS 1 +#setenv KMP_DETERMINISTIC_REDUCTION 1 + +set ncpus = 2 + +set maindir = /p/lustre2/cheung26/mgmol-20241220 + +setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH + +set exe = testWFEnergyAndForces + +cp $maindir/build_quartz/tests/$exe . + +set cfg_online = mgmol.cfg + +source $maindir/scripts/modules.quartz + +python3 test.py srun -p pdebug -n $ncpus $exe $cfg_online sih4.xyz $maindir/potentials + +date diff --git a/tests/testDensityMatrix.cc b/tests/testDensityMatrix.cc index 5941e4e5..6a65c995 100644 --- a/tests/testDensityMatrix.cc +++ b/tests/testDensityMatrix.cc @@ -9,7 +9,7 @@ #include "DensityMatrix.h" #include "GramMatrix.h" -#ifdef MGMOL_USE_SCALAPACK +#ifndef HAVE_MAGMA #include "BlacsContext.h" #include "DistMatrix.h" #else @@ -23,10 +23,10 @@ TEST_CASE( "Check functionalities of class DensityMatrix", "[functions_DensityMatrix") { -#ifdef MGMOL_USE_SCALAPACK - typedef dist_matrix::DistMatrix MatrixType; -#else +#ifdef HAVE_MAGMA typedef ReplicatedMatrix MatrixType; +#else + typedef dist_matrix::DistMatrix MatrixType; #endif int myrank; MPI_Comm_rank(MPI_COMM_WORLD, &myrank); @@ -36,7 +36,7 @@ TEST_CASE( MGmol_MPI::setup(MPI_COMM_WORLD, std::cout); -#ifdef MGMOL_USE_SCALAPACK +#ifndef HAVE_MAGMA INFO("This example to set up to use only 4 processes"); REQUIRE(npes == 4); diff --git a/tests/testGramMatrix.cc b/tests/testGramMatrix.cc index 6a356b0e..a25c39ef 100644 --- a/tests/testGramMatrix.cc +++ b/tests/testGramMatrix.cc @@ -9,7 +9,7 @@ #include "GramMatrix.h" #include "ReplicatedMatrix.h" -#ifdef MGMOL_USE_SCALAPACK +#ifndef HAVE_MAGMA #include "BlacsContext.h" #include "DistMatrix.h" #endif @@ -20,10 +20,10 @@ TEST_CASE("Check functionalities of class GramMatrix", "[functions_GramMatrix") { -#ifdef MGMOL_USE_SCALAPACK - typedef dist_matrix::DistMatrix MatrixType; -#else +#ifdef HAVE_MAGMA typedef ReplicatedMatrix MatrixType; +#else + typedef dist_matrix::DistMatrix MatrixType; #endif int myrank; MPI_Comm_rank(MPI_COMM_WORLD, &myrank); @@ -33,7 +33,7 @@ TEST_CASE("Check functionalities of class GramMatrix", "[functions_GramMatrix") MGmol_MPI::setup(MPI_COMM_WORLD, std::cout); -#ifdef MGMOL_USE_SCALAPACK +#ifndef HAVE_MAGMA INFO("This example to set up to use only 4 processes"); REQUIRE(npes == 4); diff --git a/tests/testIons.cc b/tests/testIons.cc index 2bfda28a..33c3cd24 100644 --- a/tests/testIons.cc +++ b/tests/testIons.cc @@ -6,11 +6,9 @@ #include -#include "catch.hpp" - // check that all forces components have integer values larger than 0 // and differ from each other -void checkForces(std::vector& forces) +int checkForces(std::vector& forces) { const double tol = 1.e-14; @@ -20,15 +18,23 @@ void checkForces(std::vector& forces) for (auto f1 = f0 + 1; f1 != forces.end(); f1++) { // make sure each force component is different - CHECK(std::abs(*f0 - *f1) > tol); - CHECK(*f1 > tol); - CHECK(*f0 > tol); + if (std::abs(*f0 - *f1) < tol || *f1 < tol || *f0 < tol) + { + std::cerr << "f0 = " << *f0 << ", f1 = " << *f1 << std::endl; + return 1; + } } } + + return 0; } -TEST_CASE("Ions", "[ions]") +int main(int argc, char** argv) { + int status = 0; + + int mpirc = MPI_Init(&argc, &argv); + MPI_Comm comm = MPI_COMM_WORLD; int myrank; @@ -37,8 +43,6 @@ TEST_CASE("Ions", "[ions]") MGmol_MPI::setup(comm, std::cout); Control::setup(comm, false, 0.); - MGmol_MPI& mmpi = *(MGmol_MPI::instance()); - // create a domain [0.10.]^3 const double origin[3] = { 0., 0., 0. }; const double ll = 10.; @@ -60,7 +64,8 @@ TEST_CASE("Ions", "[ions]") Species sp(MPI_COMM_WORLD); // read species info from pseudopotential file - std::string filename("pseudo.C_ONCV_PBE_SG15"); + std::string file_path = argv[1]; + std::string filename(file_path + "/pseudo.C_ONCV_PBE_SG15"); if (myrank == 0) std::cout << "Potential = " << filename << std::endl; sp.read_1species(filename); @@ -109,9 +114,13 @@ TEST_CASE("Ions", "[ions]") int ntotal = 0; MPI_Allreduce(&nlocal, &ntotal, 1, MPI_INT, MPI_SUM, comm); - CHECK(ntotal == na); + if (ntotal != na) + { + std::cout << "ntotal = " << ntotal << std::endl; + status = 1; + } } - mmpi.barrier(); + MPI_Barrier(MPI_COMM_WORLD); // verify some functionalities of class Ions { @@ -133,7 +142,7 @@ TEST_CASE("Ions", "[ions]") i++; } } - mmpi.barrier(); + MPI_Barrier(MPI_COMM_WORLD); // swap x and z for (size_t i = 0; i < positions.size() - 2; i += 3) @@ -147,7 +156,7 @@ TEST_CASE("Ions", "[ions]") ions.setPositions(positions, anumbers); } - mmpi.barrier(); + MPI_Barrier(MPI_COMM_WORLD); { std::vector& new_local_ions(ions.local_ions()); @@ -156,7 +165,11 @@ TEST_CASE("Ions", "[ions]") int ntotal = 0; MPI_Allreduce(&nlocal, &ntotal, 1, MPI_INT, MPI_SUM, comm); - CHECK(ntotal == na); + if (ntotal != na) + { + std::cerr << "ntotal = " << ntotal << std::endl; + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } } // get the names of all the ions @@ -165,9 +178,12 @@ TEST_CASE("Ions", "[ions]") if (myrank == 0) for (auto& name : names) std::cout << "name = " << name << std::endl; - CHECK(names.size() == na); - - mmpi.barrier(); + if (names.size() != na) + { + std::cerr << "Incorrect count of names..." << std::endl; + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } + MPI_Barrier(MPI_COMM_WORLD); std::vector forces(3 * na); // set forces to a different arbitrary value for each component @@ -187,13 +203,18 @@ TEST_CASE("Ions", "[ions]") ions.getLocalForces(lforces); for (auto& f : lforces) { - CHECK(std::fmod(f, 1.) < 1.e-14); + if (std::fmod(f, 1.) > 1.e-14) + { + std::cerr << "f = " << f << std::endl; + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } } ions.getForces(forces); if (myrank == 0) { - checkForces(forces); + int status = checkForces(forces); + if (status > 0) MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } // test Ions::setLocalForces based on coordinates matching @@ -209,7 +230,17 @@ TEST_CASE("Ions", "[ions]") ions.getForces(forces); if (myrank == 0) { - checkForces(forces); + int status = checkForces(forces); + if (status > 0) MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } } + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + std::cerr << "MPI Finalize failed!!!" << std::endl; + status = 1; + } + + return status; } diff --git a/util/compareTimers.py b/util/compareTimers.py index 04622810..002ccf0b 100644 --- a/util/compareTimers.py +++ b/util/compareTimers.py @@ -27,67 +27,34 @@ timers1={} timers2={} -count1=0 -count2=0 - -for line in lines1: - if line.count('comp_res_from_Hphi'): - words=line.split() - count1=eval(words[8]) - -print("count 1 = {}".format(count1)) - -for line in lines2: - if line.count('comp_res_from_Hphi'): - words=line.split() - count2=eval(words[8]) - -print("count 2 = {}".format(count2)) - for line in lines1: if line.count('Timer:'): words=line.split() - key=words[1] - val=eval(words[6]) - if 'gemm' in key: - key='gemm' - timers1[key]=val/count1 + timers1[words[1]]=words[6] for line in lines2: if line.count('Timer:'): words=line.split() - key=words[1] - val=eval(words[6]) - if 'gemm' in key: - key='gemm' - timers2[key]=val/count2 + timers2[words[1]]=words[6] #analyse timers results={} -for key in timers1.keys(): - if key in timers2.keys(): +for timer in timers1.keys(): + if timer in timers2.keys(): #compute relative difference - diff=(timers2[key]-timers1[key])/timers1[key] - if abs(diff)>rel_threshold and abs(timers1[key])>abs_threshold/count1: - results[key]=diff - else: - if 'MGmol::total' in key: - results[key]=diff - if 'quench' in key: - results[key]=diff - if 'gemm' in key: - results[key]=diff + diff=(eval(timers2[timer])-eval(timers1[timer]))/eval(timers1[timer]) + if abs(diff)>rel_threshold and abs(eval(timers1[timer]))>abs_threshold: + results[timer]=diff sorted_timers=sorted(results.items(), key=operator.itemgetter(1)) #print results -print('---------------------------------------------------------------------------------------') -print('Timer time1 time2 relative diff.(%)') -print('---------------------------------------------------------------------------------------') -ndec=2 +print('-----------------------------------------------------------------------------') +print('Timer time1 time2 relative diff.(%)') +print('-----------------------------------------------------------------------------') for timer in reversed(sorted_timers): key=timer[0] print(key.ljust(50), end="") - print(str(round(timers1[key]*count1,ndec))[:8].ljust(10), end="") - print(str(round(timers2[key]*count1,ndec))[:8].ljust(10), end="") - print(str(round(100.*timer[1],ndec))[:8].ljust(10)) + print(str(eval(timers1[key])).ljust(10), end="") + print(str(eval(timers2[key])).ljust(10), end="") + print(str(100.*timer[1]).ljust(20))