Using bwUniCluster¶

Login¶

There are 2 login nodes and 2 gateways that redirect to any of the login nodes in a load-balanced way:

Hostname	Node type
`bwunicluster.scc.kit.edu`	login to one of the two login nodes
`uc3.scc.kit.edu`	login to one of the two login nodes

The login nodes can also be reached directly:

Hostname	Node type
`uc3-login1.scc.kit.edu`	bwUniCluster 3.0 first login node
`uc3-login2.scc.kit.edu`	bwUniCluster 3.0 second login node

Host key fingerprint:

Algorithm	Fingerprint (SHA256)
RSA	`SHA256:RaE0/tqQMMBmJuDCIo3WZ38YJsz0godVyt6aUOk/E0M`
ECDSA	`SHA256:LjBYL/x86ZAlL0JdlXrCmPYXvS3DaSiMuvycojBMdwQ`
ED25519	`SHA256:5mZYEpKigwK5ibBMHRrh3WIkOtCqomJW6H7OMbPk3ec`

More details can be found in bwUniCluster3.0/Login.

Building dependencies¶

Boost¶

# last update: September 2025
source /opt/bwhpc/common/etc/easybuild/enable_eb_modules
module load OpenMPI/5.0.7-GCC-14.2.0-CUDA-12.8.0
cd "${HOME}"
mkdir boost-build
cd boost-build
BOOST_VERSION=1.89.0
BOOST_DOMAIN="https://archives.boost.io"
BOOST_ROOT="${HOME}/bin/boost_gompi-2025a_${BOOST_VERSION//./_}"
mkdir -p "${BOOST_ROOT}"
curl -sL "${BOOST_DOMAIN}/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION//./_}.tar.bz2" | tar xj
cd "boost_${BOOST_VERSION//./_}"
echo 'using mpi ;' > tools/build/src/user-config.jam
./bootstrap.sh --with-libraries=filesystem,system,mpi,serialization,test
./b2 -j 4 install --prefix="${BOOST_ROOT}" --without-python
cd "${HOME}"
rm -rf boost-build

# last update: September 2025
source /opt/bwhpc/common/etc/easybuild/enable_eb_modules
module load OpenMPI/5.0.7-GCC-14.2.0
cd "${HOME}"
mkdir boost-build
cd boost-build
BOOST_VERSION=1.89.0
BOOST_DOMAIN="https://archives.boost.io"
BOOST_ROOT="${HOME}/bin/boost_gompi-2025b_${BOOST_VERSION//./_}"
mkdir -p "${BOOST_ROOT}"
curl -sL "${BOOST_DOMAIN}/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION//./_}.tar.bz2" | tar xj
cd "boost_${BOOST_VERSION//./_}"
echo 'using mpi ;' > tools/build/src/user-config.jam
./bootstrap.sh --with-libraries=mpi,serialization,test
./b2 -j 4 install --prefix="${BOOST_ROOT}" --without-python
cd "${HOME}"
rm -rf boost-build

# last update: September 2025
module load compiler/gnu/14.2 mpi/openmpi/5.0-gnu-14.2
cd "${HOME}"
mkdir boost-build
cd boost-build
BOOST_VERSION=1.89.0
BOOST_DOMAIN="https://archives.boost.io"
BOOST_ROOT="${HOME}/bin/boost_default_${BOOST_VERSION//./_}"
mkdir -p "${BOOST_ROOT}"
curl -sL "${BOOST_DOMAIN}/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION//./_}.tar.bz2" | tar xj
cd "boost_${BOOST_VERSION//./_}"
echo 'using mpi ;' > tools/build/src/user-config.jam
./bootstrap.sh --with-libraries=mpi,serialization,test
./b2 -j 4 install --prefix="${BOOST_ROOT}" --without-python
cd "${HOME}"
rm -rf boost-build

Building software¶

ESPResSo¶

# last update: September 2025
source /opt/bwhpc/common/etc/easybuild/enable_eb_modules
module load OpenMPI/5.0.7-GCC-14.2.0-CUDA-12.8.0 \
            FFTW.MPI/3.3.10-gompi-2025a Python/3.13.1-GCCcore-14.2.0
export BOOST_VERSION=1.89.0
export BOOST_ROOT="${HOME}/bin/boost_gompi-2025a_${BOOST_VERSION//./_}"
export LD_LIBRARY_PATH="${BOOST_ROOT}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
git clone --branch python --origin upstream \
    https://github.com/espressomd/espresso.git espresso-5.0
python -m venv venv-gompi-2025a
source venv-gompi-2025a/bin/activate
python -m pip install --upgrade pip
python -m pip install -c "espresso-5.0/requirements.txt" "cmake>=4.1" cython setuptools numpy scipy vtk
cd espresso-5.0
mkdir build-gompi-2025a-maxset
cd build-gompi-2025a-maxset
cp ../maintainer/configs/maxset.hpp myconfig.hpp
sed -i "/ADDITIONAL_CHECKS/d" myconfig.hpp
CC="${GNU_BIN_DIR}/gcc" CXX="${GNU_BIN_DIR}/g++" Boost_DIR="${BOOST_ROOT}" \
    CUDAARCHS="80;90" cmake .. -D CUDAToolkit_ROOT="${CUDA_PATH}" \
    -D CMAKE_BUILD_TYPE=Release -D ESPRESSO_BUILD_WITH_CCACHE=OFF \
    -D ESPRESSO_BUILD_WITH_CUDA=ON \
    -D ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM=OFF \
    -D ESPRESSO_BUILD_WITH_HDF5=OFF -D ESPRESSO_BUILD_WITH_SCAFACOS=OFF \
    -D ESPRESSO_BUILD_WITH_WALBERLA=ON -D ESPRESSO_BUILD_WITH_WALBERLA_AVX=ON

# last update: September 2025
source /opt/bwhpc/common/etc/easybuild/enable_eb_modules
module load OpenMPI/5.0.7-GCC-14.2.0 HDF5/1.14.6-gompi-2025b \
            FFTW.MPI/3.3.10-gompi-2025b Python/3.13.5-GCCcore-14.3.0
export BOOST_VERSION=1.89.0
export BOOST_ROOT="${HOME}/bin/boost_gompi-2025b_${BOOST_VERSION//./_}"
export LD_LIBRARY_PATH="${BOOST_ROOT}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
export LD_LIBRARY_PATH="${CUDA_ROOT}/stubs/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
git clone --branch python --origin upstream \
    https://github.com/espressomd/espresso.git espresso-5.0
python -m venv venv-gompi-2025b
source venv-gompi-2025b/bin/activate
python -m pip install --upgrade pip
python -m pip install -c "espresso-5.0/requirements.txt" "cmake>=4.1" cython setuptools numpy scipy vtk h5py
cd espresso-5.0
mkdir build-gompi-2025b-maxset
cd build-gompi-2025b-maxset
cp ../maintainer/configs/maxset.hpp myconfig.hpp
sed -i "/ADDITIONAL_CHECKS/d" myconfig.hpp
CC="${GNU_BIN_DIR}/gcc" CXX="${GNU_BIN_DIR}/g++" Boost_DIR="${BOOST_ROOT}" \
    cmake .. \
    -D CMAKE_BUILD_TYPE=Release -D ESPRESSO_BUILD_WITH_CCACHE=OFF \
    -D ESPRESSO_BUILD_WITH_CUDA=OFF \
    -D ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM=OFF \
    -D ESPRESSO_BUILD_WITH_HDF5=ON -D ESPRESSO_BUILD_WITH_SCAFACOS=OFF \
    -D ESPRESSO_BUILD_WITH_WALBERLA=ON -D ESPRESSO_BUILD_WITH_WALBERLA_AVX=ON

# last update: September 2025
module load compiler/gnu/14.2 mpi/openmpi/5.0-gnu-14.2 devel/cuda/12.8 \
            numlib/fftw/3.3.10_gnu_14.2_openmpi_5.0 \
            lib/hdf5/1.14-gnu-14.2-openmpi-5.0 \
            devel/python/3.13.3-gnu-14.2
export BOOST_VERSION=1.89.0
export BOOST_ROOT="${HOME}/bin/boost_default_${BOOST_VERSION//./_}"
export LD_LIBRARY_PATH="${BOOST_ROOT}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
git clone --branch python --origin upstream \
    https://github.com/espressomd/espresso.git espresso-5.0
python -m venv venv-default
source venv-default/bin/activate
python -m pip install --upgrade pip
python -m pip install -c "espresso-5.0/requirements.txt" "cmake>=4.1" cython setuptools numpy scipy vtk h5py
cd espresso-5.0
mkdir build-maxset-default
cd build-maxset-default
cp ../maintainer/configs/maxset.hpp myconfig.hpp
sed -i "/ADDITIONAL_CHECKS/d" myconfig.hpp
CC="${GNU_BIN_DIR}/gcc" CXX="${GNU_BIN_DIR}/g++" Boost_DIR="${BOOST_ROOT}" \
    CUDAARCHS="80;90" cmake .. -D CUDAToolkit_ROOT="${CUDA_PATH}" \
    -D CMAKE_BUILD_TYPE=Release -D ESPRESSO_BUILD_WITH_CCACHE=OFF \
    -D ESPRESSO_BUILD_WITH_CUDA=ON \
    -D ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM=ON \
    -D ESPRESSO_BUILD_WITH_HDF5=ON -D ESPRESSO_BUILD_WITH_SCAFACOS=OFF \
    -D ESPRESSO_BUILD_WITH_WALBERLA=ON -D ESPRESSO_BUILD_WITH_WALBERLA_AVX=ON

Submitting jobs¶

To show which nodes are idle:

sinfo_t_idle

To show partition properties:

scontrol show partition

Batch command:

srun --partition=cpu_il --time=0:30:00 --job-name=lb --hint=nomultithread \
     --ntasks=8 --ntasks-per-node=8 --mem-per-cpu=400MB job.sh

When running the testsuite, export a UCX-specific environment variable

export UCX_POSIX_USE_PROC_LINK=n

Otherwise, the following fatal error is triggered (openucx/ucx#4224):

[1756469860.903967] [uc3n991:409846:0]        mm_posix.c:238  UCX  ERROR   open(file_name=/proc/409848/fd/66 flags=0x0) failed: Permission denied
[1756469860.904072] [uc3n991:409846:0]        mm_posix.c:238  UCX  ERROR   open(file_name=/proc/409848/fd/66 flags=0x0) failed: Permission denied

The default and gompi-2025a recipes can fail during MPI reductions:

MPI_Reduce: MPI_ERR_OTHER: known error not in list

This is a known issue for which no solution exists as of September 2025 (Ticket 782206).

Refer to bwUniCluster3.0/Running Jobs for more details on submitting job scripts on bwUniCluster.