See also
You need to have all dependencies installed to complete this chapter.
picongpu.profile¶
Use a picongpu.profile
file to set up your software environment without colliding with other software.
Ideally, store that file directly in your $HOME/
and source it after connecting to the machine:
. $HOME/picongpu.profile
We listed some example picongpu.profile files below which can be used to set up PIConGPU’s dependencies on various HPC systems.
Hypnos (HZDR)¶
# Modules #####################################################################
#
if [ -f /etc/profile.modules ]
then
. /etc/profile.modules
module purge
# export MODULES_NO_OUTPUT=1
# Core Dependencies
module load gcc/4.9.2
module load cmake/3.7.2
module load boost/1.62.0
module load cuda/8.0
module load openmpi/2.1.2.cuda80
# Plugins (optional)
module load pngwriter/0.6.0
module load hdf5-parallel/1.8.20 libsplash/1.6.0
# either use libSplash or ADIOS for file I/O
#module load adios/1.10.0
# Debug Tools
#module load gdb
#module load valgrind/3.8.1
# unset MODULES_NO_OUTPUT
fi
# Environment #################################################################
#
alias getk20='qsub -I -q k20 -lwalltime=00:30:00 -lnodes=1:ppn=8'
alias getlaser='qsub -I -q laser -lwalltime=00:30:00 -lnodes=1:ppn=16'
export PICSRC=/home/`whoami`/src/picongpu
export PIC_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE)
# send me mails on job (b)egin, (e)nd, (a)bortion or (n)o mail
export MY_MAILNOTIFY="n"
export MY_MAIL="someone@example.com"
export MY_NAME="$(whoami) <$MY_MAIL>"
export PATH=$PATH:$PICSRC
export PATH=$PATH:$PICSRC/src/splash2txt/build
export PATH=$PATH:$PICSRC/src/tools/bin
# Development #################################################################
#
#function make
#{
# real_make=`which make`
# $real_make $* 2>&1 | $HOME/grcat/usr/bin/grcat conf.gcc
#}
# "tbg" default options #######################################################
# - PBS/Torque (qsub)
# - "k20" queue
export TBG_SUBMIT="qsub"
export TBG_TPLFILE="submit/hypnos-hzdr/k20_profile.tpl"
Titan (ORNL)¶
export proj=<yourProject>
# send me mails on job (b)egin, (e)nd, (a)bortion or (n)o mail
export MY_MAILNOTIFY="n"
export MY_MAIL="someone@example.com"
export MY_NAME="$(whoami) <$MY_MAIL>"
# basic environment #################################################
source /opt/modules/3.2.6.7/init/bash
module load craype-accel-nvidia35
module swap PrgEnv-pgi PrgEnv-gnu
# module swap gcc gcc/4.8.2 # default
# Compile for CLE nodes
# (CMake likes to unwrap the Cray wrappers)
export CC=`which cc`
export CXX=`which CC`
export FC=`which ftn`
#export LD="/sw/xk6/altd/bin/ld"
# symbol bug work around (should not be required)
#MY_CRAY_LIBS=/opt/gcc/4.8.2/snos/lib64
#export LD_PRELOAD=$MY_CRAY_LIBS/libstdc++.so.6:$LD_PRELOAD
#export LD_PRELOAD=$MY_CRAY_LIBS/libgomp.so.1:$LD_PRELOAD
#export LD_PRELOAD=$MY_CRAY_LIBS/libgfortran.so.3:$LD_PRELOAD
# required tools and libs
module load git
module load cmake/3.5.2
module load cudatoolkit
module load boost/1.57.0
export BOOST_ROOT=$BOOST_DIR
export MPI_ROOT=$MPICH_DIR
# vampirtrace (optional) ############################################
# pic-configure with -c "-DVAMPIR_ENABLE=ON"
# e.g.:
# pic-configure -c "-DVAMPIR_ENABLE=ON" ~/paramSets/case001
#module load vampirtrace/5.14.4
#export VT_ROOT=$VAMPIRTRACE_DIR
# scorep (optional) #################################################
# pic-configure with -c "-DCMAKE_CXX_COMPILER=`which scorep-CC` \
# -DCUDA_NVCC_EXECUTABLE=`which scorep-nvcc`"
# e.g.:
# SCOREP_WRAPPER=OFF pic-configure -a 35 \
# -c "-DCMAKE_CXX_COMPILER=`which scorep-CC` \
# -DCUDA_NVCC_EXECUTABLE=`which scorep-nvcc`" \
# ~/paramSets/case001
# export SCOREP_WRAPPER_INSTRUMENTER_FLAGS="--cuda --mpp=mpi"
# make -j
# make install
module load scorep/2.0
# plugins (optional) ################################################
module load cray-hdf5-parallel/1.8.14
#module load adios/1.10.0 dataspaces/1.4.0
export HDF5_ROOT=$HDF5_DIR
#export ADIOS_ROOT=$ADIOS_DIR
#export DATASPACES_ROOT=$DATASPACES_DIR
# download libSplash and compile it yourself from
# https://github.com/ComputationalRadiationPhysics/libSplash/
export SPLASH_ROOT=$PROJWORK/$proj/lib/splash
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SPLASH_ROOT/lib
#export T3PIO_ROOT=$PROJWORK/$proj/lib/t3pio
#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$T3PIO_ROOT/lib
# download libpng.h and compile yourself with
# http://www.libpng.org/pub/png/libpng.html
# tar -xvf libpng-1.6.9.tar.gz
# ./configure --host=x86 --prefix=$PROJWORK/$proj/lib/libpng
# afterwards install pngwriter yourself:
# https://github.com/ax3l/pngwriter#install
export LIBPNG_ROOT=$PROJWORK/$proj/lib/libpng
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIBPNG_ROOT/lib
export PNGWRITER_ROOT=$PROJWORK/$proj/lib/pngwriter
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PNGWRITER_ROOT/lib
# helper variables and tools ########################################
export PICSRC=$HOME/src/picongpu
export PIC_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE)
export PATH=$PATH:$PICSRC
export PATH=$PATH:$PICSRC/src/tools/bin
export PATH=$PATH:$SPLASH_ROOT/bin
export PYTHONPATH=$PYTHONPATH:$SPLASH_ROOT/bin
alias getInteractive="qsub -I -A $proj -q debug -l nodes=1,walltime=30:00"
# "tbg" default options #######################################################
export TBG_SUBMIT="qsub"
export TBG_TPLFILE="submit/titan-ornl/batch_profile.tpl"
Piz Daint (CSCS)¶
# this file is loaded from all PIConGPU template files `*.tpl` therefore please
# copy this file to `$SCRATCH/picongpu.profile`
#
# General modules #############################################################
#
# if the wrong environment is loaded we switch to the gnu environment
module li 2>&1 | grep "PrgEnv-cray" > /dev/null
if [ $? -eq 0 ] ; then
module swap PrgEnv-cray PrgEnv-gnu/6.0.3
else
module load PrgEnv-gnu/6.0.3
fi
module load CMake/3.6.2
module load cudatoolkit/8.0.54_2.2.8_ga620558-2.1
# Libraries ###################################################################
module load cray-mpich/7.5.0
module load cray-hdf5-parallel/1.10.0
# Other Software ##############################################################
#
# Environment #################################################################
#
# needs to be compiled by the user
export BOOST_ROOT=$SCRATCH/lib/boost-1.62.0
export PNGWRITER_ROOT=$SCRATCH/lib/pngwriter
export ADIOS_ROOT=$SCRATCH/lib/adios-1.11.1
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PNGWRITER_ROOT/lib/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$BOOST_ROOT/lib/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ADIOS_ROOT/lib/
export MPI_ROOT=$MPICH_DIR
export HDF5_ROOT=$HDF5_DIR
# define cray compiler target architecture
# if not defined the linker crashed because wrong from `*/lib` instead
# of `*/lib64` are used
export CRAY_CPU_TARGET=x86-64
# Compile for cluster nodes
# (CMake likes to unwrap the Cray wrappers)
export CC=`which cc`
export CXX=`which CC`
export PICSRC=$HOME/src/picongpu
export PIC_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE)
export PATH=$PATH:$PICSRC
export PATH=$PATH:$PICSRC/src/tools/bin
# send me a mail on BEGIN, END, FAIL, REQUEUE, ALL,
# TIME_LIMIT, TIME_LIMIT_90, TIME_LIMIT_80 and/or TIME_LIMIT_50
export MY_MAILNOTIFY="FAIL"
export MY_MAIL="someone@example.com"
export MY_NAME="$(whoami) <$MY_MAIL>"
# "tbg" default options #######################################################
# - SLURM (sbatch)
# - "normal" queue
export TBG_SUBMIT="sbatch"
export TBG_TPLFILE="submit/pizdaint-cscs/normal_profile.tpl"
# helper tools ################################################################
# allocate an interactive shell for one hour
# `getInteractive 2` # allocates to interactive nodes (default: 1)
getInteractive() {
if [ -z "$1" ] ; then
numNodes=1
else
numNodes=$1
fi
# `--ntasks-per-core=2` activates intel hyper threading
salloc --time=1:00:00 --nodes="$numNodes" --ntasks-per-node=12 --ntasks-per-core=2 --partition normal --gres=gpu:1 --constraint=gpu
}
Taurus (TU Dresden)¶
module purge
# General modules #############################################################
#
module load oscar-modules
module load cmake/3.3.1 git
module load cuda/8.0.44 # gcc <= 5, intel 15-16
module load bullxmpi
module load gnuplot/4.6.1
# Compilers ###################################################################
### GCC
module load gcc/5.3.0 boost/1.60.0-gnu5.3
### ICC
#module load intel/2015.3.187 boost/1.59.0-intel2015.3.187
### PGI
#export BOOST_ROOT=$HOME/lib/boost_1_57_pgi_14_9
#export BOOST_INC=$BOOST_ROOT/include
#export BOOST_LIB=$BOOST_ROOT/lib
# must be set in `which <pgiDir>/bin/localrc`:
# set NOSWITCHERROR=YES;
#module load pgi/14.9 boost/<noneBuildYet>
# Other Software ##############################################################
#
module load hdf5/1.8.18-gcc-5.3.0-xmpi
module load zlib/1.2.8
# Environment #################################################################
#
#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$BOOST_LIB
export PNGWRITER_ROOT=$HOME/lib/pngwriter
export SPLASH_ROOT=$HOME/lib/splash
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/lib/pngwriter/lib/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/lib/splash/lib/
export PICSRC=$HOME/src/picongpu
export PIC_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE)
export PATH=$PATH:$PICSRC
export PATH=$PATH:$PICSRC/src/tools/bin
# send me a mail on BEGIN, END, FAIL, REQUEUE, ALL,
# TIME_LIMIT, TIME_LIMIT_90, TIME_LIMIT_80 and/or TIME_LIMIT_50
export MY_MAILNOTIFY="ALL"
export MY_MAIL="someone@example.com"
export MY_NAME="$(whoami) <$MY_MAIL>"
# "tbg" default options #######################################################
# - SLURM (sbatch)
# - "gpu" queue
export TBG_SUBMIT="sbatch"
export TBG_TPLFILE="submit/taurus-tud/k80_profile.tpl"
Lawrencium (LBNL)¶
if [ -f /etc/profile.d/modules.sh ]
then
. /etc/profile.d/modules.sh
module purge
# Core Dependencies
module load gcc/4.4.7
module load cuda/5.5
# not yet available, build boost as in `INSTALL.md`
#module load boost/1.57.0-gcc
module load openmpi/1.6.5-gcc
# Core tools
module load git
module load cmake
module load python/2.6.6
module load ipython/0.12 matplotlib/1.1.0 numpy/1.6.1 scipy/0.10.0
# Plugins (optional)
module load hdf5/1.8.11-gcc-p
export CMAKE_PREFIX_PATH=$HOME/lib/pngwriter:$CMAKE_PREFIX_PATH
export CMAKE_PREFIX_PATH=$HOME/lib/libSplash:$CMAKE_PREFIX_PATH
export LD_LIBRARY_PATH=$HOME/lib/pngwriter/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=$HOME/lib/libSplash/lib:$LD_LIBRARY_PATH
# Debug Tools
#module load valgrind/3.10.1
#module load totalview/8.10.0-0
fi
# Environment #################################################################
#
alias allocK20='salloc --time=0:30:00 --nodes=1 --ntasks-per-node=1 --cpus-per-task=8 --partition lr_manycore'
alias allocFermi='salloc --time=0:30:00 --nodes=1 --ntasks-per-node=2 --cpus-per-task=6 --partition mako_manycore'
export PICSRC=$HOME/src/picongpu
export PIC_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE)
# fix pic-create: re-enable rsync
# ssh lrc-xfer.scs00
# -> cp /usr/bin/rsync $HOME/bin/
export PATH=$HOME/bin:$PATH
export PATH=$PATH:$PICSRC
export PATH=$PATH:$PICSRC/src/splash2txt/build
export PATH=$PATH:$PICSRC/src/tools/bin
# send me a mail on BEGIN, END, FAIL, REQUEUE, ALL,
# TIME_LIMIT, TIME_LIMIT_90, TIME_LIMIT_80 and/or TIME_LIMIT_50
export MY_MAILNOTIFY="ALL"
export MY_MAIL="someone@example.com"
export MY_NAME="$(whoami) <$MY_MAIL>"
# "tbg" default options #######################################################
# - SLURM (sbatch)
# - fermi queue (also available: 2 K20 via k20_profile.tpl)
export TBG_SUBMIT="sbatch"
export TBG_TPLFILE="submit/lawrencium-lbnl/fermi_profile.tpl"
Judge (FZJ)¶
(example missing)