Merge release 20251210

.github/workflows/ci.yml

@@ -0,0 +1,82 @@
+name: CI
+on:
+ workflow_dispatch: {}
+ pull_request:
+ types: [opened, labeled, synchronize]
+ branches:
+ - 'ROMFPMD'
+ # push:
+ # branches:
+ # - release
+
+jobs:
+ docker-image:
+ uses: ./.github/workflows/docker_image.yml
+ build:
+ runs-on: ubuntu-latest
+ needs: [docker-image]
+ container:
+ image: ghcr.io/llnl/mgmol/mgmol_env:latest
+ options: --user 1001 --privileged
+ volumes:
+ - /mnt:/mnt
+ steps:
+ - name: Cancel previous runs
+ uses: styfle/cancel-workflow-action@0.11.0
+ with:
+ access_token: ${{ github.token }}
+ # - name: Set Swap Space
+ # uses: pierotofy/set-swap-space@master
+ # with:
+ # swap-size-gb: 10
+ - name: Check out mgmol
+ uses: actions/checkout@v1
+ with:
+ submodules: 'true'
+ - name: cmake
+ run: |
+ mkdir ${GITHUB_WORKSPACE}/build
+ cd ${GITHUB_WORKSPACE}/build
+ cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMPIEXEC_PREFLAGS="--oversubscribe" -DUSE_LIBROM=On -DLIBROM_PATH=/env/dependencies/libROM
+ - name: make
+ run: |
+ cd ${GITHUB_WORKSPACE}/build && make -j 4
+ - name: test
+ run: |
+ cd ${GITHUB_WORKSPACE}/build && ctest --no-compress-output -V -T Test -I 1,20,1
+ - name: test ROM Poisson operator
+ run: |
+ cd ${GITHUB_WORKSPACE}/tests/ROM/test_rom_poisson
+ ln -s ${GITHUB_WORKSPACE}/build/src/mgmol-rom .
+ ln -s ${GITHUB_WORKSPACE}/potentials/* .
+ mpirun -n 3 --oversubscribe ./mgmol-rom -c carbyne.poisson.cfg -i carbyne.in
+ - name: test ROM ion density evaluation
+ run: |
+ cd ${GITHUB_WORKSPACE}/tests/ROM/test_rom_poisson
+ mpirun -n 3 --oversubscribe ./mgmol-rom -c carbyne.ion.cfg -i carbyne.in
+ # code-style:
+ # runs-on: ubuntu-latest
+ # needs: [docker-image]
+ # container:
+ # image: ghcr.io/llnl/mgmol/mgmol_env:latest
+ # options: --user 1001 --privileged
+ # volumes:
+ # - /mnt:/mnt
+ # steps:
+ # - name: Cancel previous runs
+ # uses: styfle/cancel-workflow-action@0.11.0
+ # with:
+ # access_token: ${{ github.token }}
+ # - name: Check out mgmol
+ # uses: actions/checkout@v1
+ # with:
+ # submodules: 'true'
+ # - name: cmake
+ # run: |
+ # mkdir build
+ # cd build
+ # cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMGMOL_WITH_CLANG_FORMAT=ON
+ # - name: make
+ # run: |
+ # cd build && make format
+

.github/workflows/docker_image.yml

@@ -0,0 +1,59 @@
+name: docker-image
+on:
+ workflow_call:
+
+env:
+ REGISTRY: ghcr.io
+ # github.repository as <account>/<repo>
+ IMAGE_NAME: llnl/mgmol/mgmol_env
+ DOCKERPATH: docker
+
+jobs:
+ docker-ci:
+ runs-on: ubuntu-latest
+ name: "docker env"
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+ - uses: Ana06/get-changed-files@v2.2.0 
+ id: files
+ - name: DockerPATH configuration
+ run: echo "DOCKERPATH=$DOCKERPATH"
+ - name: DockerPATH - check if files in docker path changed
+ if: contains(steps.files.outputs.all,env.DOCKERPATH) || contains(steps.files.outputs.all,'docker_image.yml')
+ run: |
+ echo "CI container needs rebuilding..."
+ echo "CI_NEEDS_REBUILD=true" >> $GITHUB_ENV
+ - name: Log into registry ${{ env.REGISTRY }}
+ if: env.CI_NEEDS_REBUILD
+ uses: docker/login-action@v2
+ with:
+ registry: ${{ env.REGISTRY }}
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+ - name: Extract metadata (tags, labels) for Docker
+ id: meta
+ if: env.CI_NEEDS_REBUILD
+ uses: docker/metadata-action@v4
+ with:
+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+ tags: type=sha
+ flavor: latest=true
+ - name: Build Container motd
+ if: env.CI_NEEDS_REBUILD
+ run: |
+ echo "#!/bin/bash" > ${{env.DOCKERPATH}}/motd.sh
+ echo "echo --------------------------" >> ${{env.DOCKERPATH}}/motd.sh
+ echo "echo mgmol_env/CI Development Container" >> ${{env.DOCKERPATH}}/motd.sh
+ echo "echo \"Revision: `echo ${GITHUB_SHA} | cut -c1-8`\"" >> ${{env.DOCKERPATH}}/motd.sh
+ echo "echo --------------------------" >> ${{env.DOCKERPATH}}/motd.sh
+ chmod 755 ${{env.DOCKERPATH}}/motd.sh
+ cat ${{env.DOCKERPATH}}/motd.sh
+ - name: Docker Image - Build and push
+ if: env.CI_NEEDS_REBUILD
+ uses: docker/build-push-action@v3
+ with:
+ push: true
+ context: ${{ env.DOCKERPATH }}
+ tags: ${{ steps.meta.outputs.tags }}

.gitignore

@@ -0,0 +1 @@
+src/mgmol_config.h

CMakeLists.txt

@@ -139,6 +139,22 @@ if (${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT)
  endif(${SCALAPACK_FOUND}) 
 endif(${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT)
 
+# libROM (optional)
+set(USE_LIBROM False CACHE BOOL "Build with libROM")
+set(LIBROM_PATH "" CACHE STRING "Path of libROM")
+if(USE_LIBROM)
+ message(STATUS "LIBROM_PATH: ${LIBROM_PATH}")
+ if(NOT LIBROM_PATH)
+ message(FATAL_ERROR "Cmake is asked to use libROM, but LIBROM_PATH not specified.")
+ endif(NOT LIBROM_PATH)
+
+ find_package(libROM REQUIRED)
+
+ if(libROM_FOUND)
+ add_definitions(-DMGMOL_HAS_LIBROM)
+ endif(libROM_FOUND)
+endif(USE_LIBROM)
+
 # ARPACK (optional)
 set(MGMOL_WITH_ARPACK FALSE CACHE BOOL "Compile with ARPACK package")
 if(${MGMOL_WITH_ARPACK} OR DEFINED ARPACK_ROOT)
@@ -263,6 +279,9 @@ include_directories("${PROJECT_SOURCE_DIR}/src/sparse_linear_algebra")
 include_directories("${PROJECT_SOURCE_DIR}/src/tools")
 include_directories("${PROJECT_SOURCE_DIR}/src")
 
+include_directories("${LIBROM_PATH}/lib")
+link_libraries(${LIBROM_LIB})
+
 # add subdirectories for source files, tests
 add_subdirectory(src)
 

cmake_modules/FindlibROM.cmake

@@ -0,0 +1,11 @@
+if(NOT LIBROM_PATH)
+ message(FATAL_ERROR "LIBROM_PATH not specified.")
+endif(NOT LIBROM_PATH)
+
+find_library(LIBROM_LIB libROM.so HINTS "${LIBROM_PATH}/build/lib")
+find_path(LIBROM_INCLUDES librom.h HINTS "${LIBROM_PATH}/lib")
+
+mark_as_advanced(LIBROM_LIB LIBROM_INCLUDES)
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(libROM REQUIRED_VARS LIBROM_LIB LIBROM_INCLUDES)

cmake_toolchains/quartz.default.cmake

@@ -0,0 +1,6 @@
+set(CMAKE_C_COMPILER mpicc)
+set(CMAKE_CXX_COMPILER mpicxx)
+set(CMAKE_Fortran_COMPILER mpif90)
+
+set(SCALAPACK_ROOT $ENV{MKLROOT})
+set(SCALAPACK_BLACS_LIBRARY $ENV{MKLROOT}/lib/intel64/libmkl_blacs_intelmpi_lp64.so)

docker/Dockerfile

@@ -0,0 +1,50 @@
+FROM ubuntu:22.04
+
+ENV ENVDIR=env
+
+# install sudo
+RUN apt-get -yq update && apt-get -yq install sudo
+
+WORKDIR /$ENVDIR
+
+# install packages
+RUN sudo apt-get install -yq git
+RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake
+RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpi-dev
+RUN sudo apt-get install -yq libboost-all-dev
+RUN sudo apt-get install -yq vim
+RUN sudo apt-get install -yq git-lfs
+RUN sudo apt-get install -yq valgrind hdf5-tools
+RUN sudo apt-get install -yq wget
+### clang-format seems to be updated to 14.0. Not using it for now.
+# RUN sudo apt-get install -yq clang-format
+
+# install lldb and gdb for debugging
+RUN sudo apt-get install -yq lldb gdb
+
+RUN sudo apt-get clean -q
+
+ENV LIB_DIR=/$ENVDIR/dependencies
+WORKDIR $LIB_DIR
+
+# cmake toolchain file for librom
+RUN echo 'set(CMAKE_C_COMPILER mpicc)\n\
+set(CMAKE_CXX_COMPILER mpicxx)\n\
+set(CMAKE_Fortran_COMPILER mpif90)' > ./librom_env.cmake
+ENV TOOLCHAIN_FILE=$LIB_DIR/librom_env.cmake
+
+# install libROM for scaleupROM
+RUN sudo git clone https://github.com/LLNL/libROM.git
+WORKDIR ./libROM/build
+# libROM without MFEM.
+RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=Optimized -DUSE_MFEM=OFF
+RUN sudo make -j 16
+
+# create and switch to a user
+ENV USERNAME=test
+RUN echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+RUN useradd --no-log-init -u 1001 --create-home --shell /bin/bash $USERNAME
+RUN adduser $USERNAME sudo
+USER $USERNAME
+WORKDIR /home/$USERNAME
+

examples/Carbyne/carbyne.rom.cfg

@@ -0,0 +1,34 @@
+verbosity=2
+xcFunctional=PBE
+FDtype=4th
+[Mesh]
+nx= 96
+ny= 96
+nz= 192
+[Domain]
+ox= -10.
+oy= -10.
+oz= -20.
+lx= 20.
+ly= 20.
+lz= 40.
+[Potentials]
+pseudopotential=pseudo.H_ONCV_PBE_SG15
+pseudopotential=pseudo.C_ONCV_PBE_SG15
+[Run]
+type=QUENCH
+[Quench]
+max_steps=5
+atol=1.e-8
+[Orbitals]
+initial_type=Fourier
+[Restart]
+output_level=4
+input_level=4
+input_filename=snapshot0_000
+
+[ROM.offline]
+restart_filefmt=snapshot0_%03d
+restart_min_idx=0
+restart_max_idx=1
+basis_file=carom

examples/PinnedH2O/coords_test2.in

@@ -0,0 +1,3 @@
+O1 1 0.00 0.00 0.00 0
+H1 2 -0.45 1.57 -1.07 1
+H2 2 -0.45 -1.48 -0.97 1

examples/PinnedH2O/get_ROM_table.py

@@ -0,0 +1,28 @@
+import subprocess
+import re
+
+pattern = r"For energy fraction: \d+\.\d+, take first (\d+) of \d+ basis vectors"
+
+print("\\begin{tabular}{|c||c|c|c|c|c|c|c|}")
+print("\\hline")
+print("$k$ & $\\varepsilon = 10^{-1}$ & $\\varepsilon = 10^{-2}$ & $\\varepsilon = 10^{-3}$ & $\\varepsilon = 10^{-4}$ & $\\varepsilon = 10^{-5}$ & Snapshots \\\\")
+print("\\hline")
+
+for t in range(10):
+ k = 50*(t+1)
+ snapshots = 4*k
+ grep_command = f"grep 'take first' basis_1_{k}_Pinned_H2O.out"
+ result = subprocess.run(grep_command, shell=True, capture_output=True, text=True)
+ matches = re.findall(pattern, result.stdout)
+ energy_fractions = {
+ "0.9": matches[0],
+ "0.99": matches[1],
+ "0.999": matches[2],
+ "0.9999": matches[3],
+ "0.99999": matches[4],
+ }
+ line = f"{k} & {energy_fractions['0.9']} & {energy_fractions['0.99']} & {energy_fractions['0.999']} & {energy_fractions['0.9999']} & {energy_fractions['0.99999']} & {snapshots} \\\\"
+ print(line)
+
+print("\\hline")
+print("\\end{tabular}")

examples/PinnedH2O/get_result.sh

@@ -0,0 +1,37 @@
+#filename="offline_PinnedH2O" # FOM
+#filename="rom39_PinnedH2O" # ROM compare MD
+#filename="39_force_PinnedH2O" # ROM compare force
+
+#filename="PinnedH2O_test2_ref" # FOM
+filename="PinnedH2O_rom_3DOF_test2_2_2_34" # ROM PinnedH2O 3DOF MD
+
+# Extracting kinetic energy, total energy, temperature from MGmgol output log
+awk '/Kinetic/ {print $3}' $filename.out > ke_$filename.txt
+awk '/Kinetic/ {print $5}' $filename.out >temp_$filename.txt
+awk '/Total/ {print $3}' $filename.out > te_$filename.txt
+
+# Extracting H1, H2, F1, F2 from MGmgol output log
+# if FOM, these files contain the FOM results
+# if compare MD, these files contain the results with projected orbitals
+awk '/O1 / {print $4, $5, $6}' $filename.out > O1_$filename.txt
+awk '/H1 / {print $3, $4, $5}' $filename.out > H1_$filename.txt
+awk '/H2 / {print $3, $4, $5}' $filename.out > H2_$filename.txt
+awk '/O1 / {print $7, $8, $9}' $filename.out > f_O1_$filename.txt
+awk '/H1 / {print $6, $7, $8}' $filename.out > f_H1_$filename.txt
+awk '/H2 / {print $6, $7, $8}' $filename.out > f_H2_$filename.txt
+
+# if compare force, files with "_fom" contain the FOM results
+# files with "_rom" contain the results with projected orbitals
+if [[ "$filename" == *"force_"* ]]; then
+ sed -n '1~2p' H1_$filename.out > H1_rom$filename.txt
+ sed -n '1~2p' H2_$filename.out > H2_rom$filename.txt
+ sed -n '1~2p' f_H1_$filename.out > f_H1_rom$filename.txt
+ sed -n '1~2p' f_H2_$filename.out > f_H2_rom$filename.txt
+
+ sed -n '2~2p' H1_$filename.out > H1_fom$filename.txt
+ sed -n '2~2p' H2_$filename.out > H2_fom$filename.txt
+ sed -n '2~2p' f_H1_$filename.out > f_H1_fom$filename.txt
+ sed -n '2~2p' f_H2_$filename.out > f_H2_fom$filename.txt
+fi
+
+rm -rf snapshot_*

examples/PinnedH2O/job.basis_1_50

@@ -0,0 +1,32 @@
+#!/bin/tcsh
+#SBATCH -N 1
+#SBATCH -t 0:10:00
+#SBATCH -p pdebug
+
+date
+
+setenv OMP_NUM_THREADS 1
+#setenv KMP_DETERMINISTIC_REDUCTION 1
+
+set ncpus = 8
+
+set maindir = /p/lustre2/cheung26/mgmol
+
+setenv LD_LIBRARY_PATH ${maindir}/build_quartz/libROM/build/lib:$LD_LIBRARY_PATH
+
+set exe = ${maindir}/build_quartz/libROM/build/examples/misc/combine_samples
+
+set snapshot_files = ""
+set increment_md_steps = 1
+set num_md_steps = 50
+
+foreach k (`seq $increment_md_steps $increment_md_steps $num_md_steps`)
+ set snapshot_files = "$snapshot_files MD_mdstep${k}_snapshot"
+end
+echo "Snapshot files: $snapshot_files"
+
+set basis_file = "PinnedH2O_orbitals_basis_${increment_md_steps}_${num_md_steps}"
+
+srun -n $ncpus $exe -f $basis_file $snapshot_files > basis_${increment_md_steps}_${num_md_steps}_PinnedH2O.out
+
+date