Merge branch 'publicGithubDevelop' into release1.2

Author: dsambit

CMakeLists.txt

@@ -5,6 +5,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.17)
 set(CMAKE_CXX_STANDARD 17
     CACHE STRING "Minimum required C++ standard (default is 17)")
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 # Set the name of the project and target:
 SET(TARGET "dftfe_exe")
@@ -30,6 +31,7 @@ IF (WITH_GPU)
       enable_language(CUDA)
     ELSEIF ("${GPU_LANG}" STREQUAL "hip")
       enable_language(HIP)
+    ELSEIF ("${GPU_LANG}" STREQUAL "sycl")
     ELSE ()
       MESSAGE(FATAL_ERROR "\n"
     "*** Could not find valid GPU_LANG=cuda/hip ***\n\n"
@@ -101,6 +103,7 @@ SET(TARGET_SRC
   ./src/excManager/SlaterBasis.cpp
   ./src/excManager/GaussianBasis.cpp
   ./src/excManager/hubbardClass.cpp
+  ./src/excManager/exchangeCorrelationFunctionalEvaluatorCPU.cc
   ./src/poisson/poissonSolverProblem.cc
   ./src/poisson/MultiVectorPoissonLinearSolverProblem.cpp
   ./src/helmholtz/kerkerSolverProblem.cc
@@ -180,7 +183,6 @@ SET(TARGET_SRC
   ./src/dft/moveAtoms.cc
   ./src/dft/moveMeshToAtoms.cc
   ./src/dft/kerker.cc
-  ./src/dft/kineticEnergyDensityCalculator.cc
   ./src/dft/pRefinedDoFHandler.cc
   ./src/dft/psiInitialGuess.cc
   ./src/dft/publicMethods.cc
@@ -242,7 +244,6 @@ SET(DEVICE_SRC
   ./src/dft/densityCalculatorDeviceKernels.cc
   ./src/dft/densityFirstOrderResponseCalculatorDeviceKernels.cc
   ./src/dft/solveVselfInBinsDeviceKernels.cc
-  ./src/dft/kineticEnergyDensityCalculatorDeviceKernels.cc
   ./src/linAlg/linearAlgebraOperationsDeviceKernels.cc
   ./src/force/forceWfcContractionsDeviceKernels.cc
   ./utils/constraintMatrixInfoDeviceKernels.cc
@@ -253,6 +254,9 @@ SET(DEVICE_SRC
   ./utils/matrixFreeDeviceKernels.cc
   ./utils/FEBasisOperationsKernelsInternalDevice.cc
   ./src/atom/AtomicCenteredNonLocalOperatorKernelsDevice.cc
+  ./src/dftOperator/KohnShamHamiltonianOperatorDeviceKernels.cc
+  ./src/excManager/excManagerDeviceKernels.cc
+  ./src/excManager/exchangeCorrelationFunctionalEvaluatorDevice.cc  
   )
 ELSEIF ("${GPU_LANG}" STREQUAL "hip")
 SET(DEVICE_SRC
@@ -262,7 +266,6 @@ SET(DEVICE_SRC
   ./src/dft/densityCalculatorDeviceKernels.cc
   ./src/dft/densityFirstOrderResponseCalculatorDeviceKernels.cc
   ./src/dft/solveVselfInBinsDeviceKernels.cc
-  ./src/dft/kineticEnergyDensityCalculatorDeviceKernels.cc
   ./src/linAlg/linearAlgebraOperationsDeviceKernels.cc
   ./src/force/forceWfcContractionsDeviceKernels.cc
   ./utils/constraintMatrixInfoDeviceKernels.cc
@@ -273,6 +276,31 @@ SET(DEVICE_SRC
   ./utils/matrixFreeDeviceKernels.cc
   ./utils/FEBasisOperationsKernelsInternalDevice.cc
   ./src/atom/AtomicCenteredNonLocalOperatorKernelsDevice.cc
+  ./src/dftOperator/KohnShamHamiltonianOperatorDeviceKernels.cc
+  ./src/excManager/excManagerDeviceKernels.cc
+  ./src/excManager/exchangeCorrelationFunctionalEvaluatorDevice.cc
+  )
+ELSEIF ("${GPU_LANG}" STREQUAL "sycl")
+SET(DEVICE_SRC
+  ./utils/MemoryTransferKernelsDevice.cc
+  ./utils/DeviceKernelsGeneric.cc
+  ./utils/DeviceDirectCCLWrapper.cc
+  ./src/dft/densityCalculatorDeviceKernels.cc
+  ./src/dft/densityFirstOrderResponseCalculatorDeviceKernels.cc
+  ./src/dft/solveVselfInBinsDeviceKernels.cc
+  ./src/linAlg/linearAlgebraOperationsDeviceKernels.cc
+  ./src/force/forceWfcContractionsDeviceKernels.cc
+  ./utils/constraintMatrixInfoDeviceKernels.cc
+  ./utils/DeviceAPICalls.sycl.cc
+  ./utils/BLASWrapperDevice.cc
+  ./utils/MPICommunicatorP2PKernelsDevice.cc
+  ./src/solvers/linearSolverCGDeviceKernels.cc
+  ./utils/matrixFreeDeviceKernels.cc
+  ./utils/FEBasisOperationsKernelsInternalDevice.cc
+  ./src/atom/AtomicCenteredNonLocalOperatorKernelsDevice.cc
+  ./src/dftOperator/KohnShamHamiltonianOperatorDeviceKernels.cc
+  ./src/excManager/excManagerDeviceKernels.cc
+  ./src/excManager/exchangeCorrelationFunctionalEvaluatorDevice.cc  
   )
 
 ENDIF()
@@ -288,6 +316,8 @@ IF (WITH_GPU)
        ELSEIF ("${GPU_VENDOR}" STREQUAL "nvidia")
           ADD_DEFINITIONS(-D__HIP_PLATFORM_NVIDIA__)
        ENDIF()
+    ELSEIF ("${GPU_LANG}" STREQUAL "sycl")
+      set_source_files_properties(${DEVICE_SRC} PROPERTIES COMPILE_FLAGS ${CMAKE_SYCL_FLAGS})
     ENDIF()
 ENDIF()
 IF (WITH_GPU)
@@ -353,6 +383,9 @@ ENDIF()
 
 IF (USE_64BIT_INT)
   ADD_DEFINITIONS(-DDFTFE_WITH_64BIT_INT)
+  IF ("${GPU_VENDOR}" STREQUAL "intel")
+    add_compile_options("-fno-sycl-id-queries-fit-in-int")
+  ENDIF()
 ENDIF()
 
 #
@@ -379,6 +412,9 @@ IF (WITH_GPU)
     ADD_DEFINITIONS(-DDFTFE_WITH_DEVICE_NVIDIA)
   ELSEIF ("${GPU_VENDOR}" STREQUAL "amd")
     ADD_DEFINITIONS(-DDFTFE_WITH_DEVICE_AMD)
+  ELSEIF ("${GPU_VENDOR}" STREQUAL "intel")
+    ADD_DEFINITIONS(-DDFTFE_WITH_DEVICE_INTEL)
+    ADD_DEFINITIONS(-DDFTFE_WITH_DEVICE_MKL)
   ENDIF()
 
   IF ("${GPU_LANG}" STREQUAL "cuda")
@@ -393,6 +429,34 @@ IF (WITH_GPU)
     ADD_DEFINITIONS(-DDFTFE_WITH_DEVICE_LANG_HIP)
     find_package(hipblas)
     target_link_libraries(${TARGETLIB} PUBLIC roc::hipblas)
+  ELSEIF ("${GPU_LANG}" STREQUAL "sycl")
+    MESSAGE(STATUS "DFTFE with GPU LANG SYCL")
+    ADD_DEFINITIONS(-DDFTFE_WITH_DEVICE_LANG_SYCL)
+    find_program(MPI_BASE_CXX_COMPILER "icpx")
+    get_filename_component(_REALPATH_SYCL_COMPILER "${MPI_BASE_CXX_COMPILER}" REALPATH)
+    get_filename_component(SYCL_BIN_DIR "${_REALPATH_SYCL_COMPILER}" DIRECTORY)
+    get_filename_component(SYCL_PACKAGE_DIR "${SYCL_BIN_DIR}" DIRECTORY CACHE)
+    find_file(SYCL_INCLUDE_DIR
+    NAMES
+    include
+    HINTS
+    ${SYCL_PACKAGE_DIR}
+    NO_DEFAULT_PATH)
+    target_include_directories(${TARGETLIB} PUBLIC ${SYCL_INCLUDE_DIR})
+    target_include_directories(${TARGETLIB} PUBLIC ${SYCL_INCLUDE_DIR}/sycl)
+    set(MKL_INTERFACE lp64)
+    set(ENABLE_SCALAPACK ON)
+    set(MKL_MPI mpich)
+    find_PACKAGE(IntelSYCL REQUIRED)
+    find_package(MKL REQUIRED)
+    target_link_libraries(${TARGETLIB} PUBLIC MKL::MKL)
+    get_target_property(MKL_SYCL_INCLUDE_DIRS MKL::MKL_SYCL INTERFACE_INCLUDE_DIRECTORIES)
+    target_include_directories(${TARGETLIB} PUBLIC ${MKL_SYCL_INCLUDE_DIRS})
+    get_target_property(mkl_sycl_includes MKL::MKL_SYCL INTERFACE_INCLUDE_DIRECTORIES)
+    target_include_directories(${TARGETLIB} PUBLIC ${mkl_sycl_includes})
+    get_target_property(mkl_sycl_libs MKL::MKL_SYCL INTERFACE_LINK_LIBRARIES)
+    target_link_libraries(${TARGETLIB} PUBLIC ${mkl_sycl_libs})
+    set_target_properties(${TARGETLIB} PROPERTIES LINK_FLAGS ${CMAKE_SYCL_FLAGS})
   ENDIF()
   IF (WITH_GPU_AWARE_MPI)
     ADD_DEFINITIONS(-DDFTFE_WITH_DEVICE_AWARE_MPI)

README.md

@@ -12,19 +12,19 @@ Installation instructions
 -------------------------
 
 DFT-FE code builds on top of the deal.II library for everything that has to do with finite elements, geometries, meshes, etc., and, through deal.II on p4est for parallel adaptive mesh handling.
-The steps to install the necessary dependencies and DFT-FE itself are described in the *Installation* section of the DFT-FE manual (compile doc/manual/manual.tex or download the release version manual [here](https://github.com/dftfeDevelopers/dftfe/blob/manual/manual-release1.2.pdf)). 
+The steps to install the necessary dependencies and DFT-FE itself are described in the *Installation* section of the DFT-FE manual (download the release version manual [here](https://github.com/dftfeDevelopers/dftfe/blob/manual/manual-release.pdf)). 
 
 We have created several shell based installation scripts for the development version of DFT-FE (`publicGithubDevelop` branch) on various machines:
-  - [OLCF Frontier](https://github.com/dftfeDevelopers/install_DFTFE/tree/frontierCurrentRelease)
-  - [NERSC Perlmutter](https://github.com/dftfeDevelopers/install_DFTFE/tree/perlmutterCurrentRelease)
-  - [UMICH Greatlakes](https://github.com/dftfeDevelopers/install_DFTFE/tree/greatlakesCurrentRelease)
+  - [OLCF Frontier](https://github.com/dftfeDevelopers/install_DFTFE/tree/frontierRelease)
+  - [NERSC Perlmutter](https://github.com/dftfeDevelopers/install_DFTFE/tree/perlmutterRelease)
+  - [UMICH Greatlakes](https://github.com/dftfeDevelopers/install_DFTFE/tree/greatlakesRelease)
 
 
 
 Running DFT-FE
 --------------
 
-Instructions on how to run DFT-FE including demo examples can also be found in the *Running DFT-FE* section of the manual (compile doc/manual/manual.tex or download the release version manual [here](https://github.com/dftfeDevelopers/dftfe/blob/manual/manual-release1.2.pdf)). Beyond the demo examples in the manual, we also refer to our [benchmarks repository](https://github.com/dftfeDevelopers/dftfe-benchmarks) which contains several accuracy and performance benchmarks on a range of system sizes.
+Instructions on how to run DFT-FE including demo examples can also be found in the *Running DFT-FE* section of the manual (download the release version manual [here](https://github.com/dftfeDevelopers/dftfe/blob/manual/manual-release.pdf)). Beyond the demo examples in the manual, we also refer to our [benchmarks repository](https://github.com/dftfeDevelopers/dftfe-benchmarks) which contains several accuracy and performance benchmarks on a range of system sizes.
 
 
 Contributing to DFT-FE

authors

@@ -10,9 +10,7 @@ at the Indian Institute of Science.
 'DFT-FE authors' is an all encompassing list of contributors to the DFT-FE code. 
 
 DFT-FE code is currently maintained by a group of development leads/mentors and  principal developers, 
-who manage the architecture of the code and the core functionalities. Developers with significant contributions
-to core functionalities and code architecture in the past who are no longer active principal developers, 
-are listed under principal developers emeriti. Finally, all contributors who have contributed to major parts
+who manage the architecture of the code and the core functionalities. Finally, all contributors who have contributed to major parts
 of the DFT-FE code or sent important fixes and enhancements are listed under contributors. All the underlying 
 lists are in alphabetical order based on the last name.