See also

You need to have all dependencies installed to complete this chapter.

picongpu.profile

Section author: Axel Huebl

Use a picongpu.profile file to set up your software environment without colliding with other software. Ideally, store that file directly in your $HOME/ and source it after connecting to the machine:

. $HOME/picongpu.profile

We listed some example picongpu.profile files below which can be used to set up PIConGPU’s dependencies on various HPC systems.

Hypnos (HZDR)

For this profile to work, you need to download the PIConGPU source code manually.

# Modules #####################################################################
#

if [ -f /etc/profile.modules ]
then
        . /etc/profile.modules
        module purge
#       export MODULES_NO_OUTPUT=1

        # Core Dependencies
        module load gcc/4.9.2
        module load cmake/3.7.2
        module load boost/1.62.0
        module load cuda/8.0
        module load openmpi/1.8.6.kepler.cuda80

        # Plugins (optional)
        module load pngwriter/0.6.0
        module load hdf5-parallel/1.8.15 libsplash/1.6.0

        # either use libSplash or ADIOS for file I/O
        #module load adios/1.10.0

        # Debug Tools
        #module load gdb
        #module load valgrind/3.8.1

#       unset MODULES_NO_OUTPUT
fi

# Environment #################################################################
#
alias getk20='qsub -I -q k20 -lwalltime=00:30:00 -lnodes=1:ppn=8'
alias getlaser='qsub -I -q laser -lwalltime=00:30:00 -lnodes=1:ppn=16'

export PICSRC=/home/$(whoami)/src/picongpu
export PIC_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE)

# send me mails on job (b)egin, (e)nd, (a)bortion or (n)o mail
export MY_MAILNOTIFY="n"
export MY_MAIL="someone@example.com"
export MY_NAME="$(whoami) <$MY_MAIL>"

export PATH=$PATH:$PICSRC
export PATH=$PATH:$PICSRC/src/splash2txt/build
export PATH=$PATH:$PICSRC/src/tools/bin

export PYTHONPATH=$PICSRC/lib/python:$PYTHONPATH

# Development #################################################################
#
#function make
#{
#  real_make=$(which make)
#  $real_make $* 2>&1 | $HOME/grcat/usr/bin/grcat conf.gcc
#}

# "tbg" default options #######################################################
#   - PBS/Torque (qsub)
#   - "k20" queue
export TBG_SUBMIT="qsub"
export TBG_TPLFILE="etc/picongpu/hypnos-hzdr/k20_profile.tpl"

Titan (ORNL)

For this profile to work, you need to download the PIConGPU source code and install libSplash, libpng and PNGwriter manually.

export proj=<yourProject>

# send me mails on job (b)egin, (e)nd, (a)bortion or (n)o mail
export MY_MAILNOTIFY="n"
export MY_MAIL="someone@example.com"
export MY_NAME="$(whoami) <$MY_MAIL>"

# basic environment #################################################
source /opt/modules/3.2.6.7/init/bash
module load craype-accel-nvidia35
module swap PrgEnv-pgi PrgEnv-gnu
module swap gcc gcc/4.9.3

# Compile for CLE nodes
#   (CMake likes to unwrap the Cray wrappers)
export CC=$(which cc)
export CXX=$(which CC)
export FC=$(which ftn)
#export LD="/sw/xk6/altd/bin/ld"

# symbol bug work around (should not be required)
#MY_CRAY_LIBS=/opt/gcc/4.9.3/snos/lib64
#export LD_PRELOAD=$MY_CRAY_LIBS/libstdc++.so.6:$LD_PRELOAD
#export LD_PRELOAD=$MY_CRAY_LIBS/libgomp.so.1:$LD_PRELOAD
#export LD_PRELOAD=$MY_CRAY_LIBS/libgfortran.so.3:$LD_PRELOAD

# required tools and libs
module load git
module load cmake3/3.9.0
module load cudatoolkit  # 7.5.18
# might fail to link with missing symbols:
#   C++11 module rebuild pending [CCS #366279]
module load boost/1.62.0
export BOOST_ROOT=$BOOST_DIR
export MPI_ROOT=$MPICH_DIR

# vampirtrace (optional) ############################################
#   pic-configure with -c "-DVAMPIR_ENABLE=ON"
#   e.g.:
#     pic-configure -c "-DVAMPIR_ENABLE=ON" ~/picInputs/case001
#module load vampirtrace/5.14.4
#export VT_ROOT=$VAMPIRTRACE_DIR

# scorep (optional) #################################################
#   pic-configure with -c "-DCMAKE_CXX_COMPILER=$(which scorep-CC) \
#                          -DCUDA_NVCC_EXECUTABLE=$(which scorep-nvcc)"
#   e.g.:
#     SCOREP_WRAPPER=OFF pic-configure -a "cuda:35" \
#         -c "-DCMAKE_CXX_COMPILER=$(which scorep-CC) \
#         -DCUDA_NVCC_EXECUTABLE=$(which scorep-nvcc)" \
#         ~/picInputs/case001
#     export SCOREP_WRAPPER_INSTRUMENTER_FLAGS="--cuda --mpp=mpi"
#     make -j
#     make install
#module load scorep

# plugins (optional) ################################################
module load cray-hdf5-parallel/1.8.14
module load adios/1.10.0
export HDF5_ROOT=$HDF5_DIR
#export ADIOS_ROOT=$ADIOS_DIR
#export DATASPACES_ROOT=$DATASPACES_DIR

# download libSplash and compile it yourself from
#   https://github.com/ComputationalRadiationPhysics/libSplash/
export SPLASH_ROOT=$PROJWORK/$proj/lib/splash
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SPLASH_ROOT/lib

#export T3PIO_ROOT=$PROJWORK/$proj/lib/t3pio
#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$T3PIO_ROOT/lib

# download libpng.h and compile yourself with
#   http://www.libpng.org/pub/png/libpng.html
#   tar -xvf libpng-1.6.9.tar.gz
#   ./configure --host=x86 --prefix=$PROJWORK/$proj/lib/libpng
# afterwards install pngwriter yourself:
#   https://github.com/ax3l/pngwriter#install
export LIBPNG_ROOT=$PROJWORK/$proj/lib/libpng
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIBPNG_ROOT/lib
export PNGWRITER_ROOT=$PROJWORK/$proj/lib/pngwriter
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PNGWRITER_ROOT/lib

# helper variables and tools ########################################
export PICSRC=$HOME/src/picongpu
export PIC_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE)

export PATH=$PATH:$PICSRC
export PATH=$PATH:$PICSRC/src/tools/bin

export PYTHONPATH=$PICSRC/lib/python:$PYTHONPATH

alias getInteractive="qsub -I -A $proj -q debug -l nodes=1,walltime=30:00"

# "tbg" default options #######################################################
export TBG_SUBMIT="qsub"
export TBG_TPLFILE="etc/picongpu/titan-ornl/batch_profile.tpl"

Piz Daint (CSCS)

For this profile to work, you need to download the PIConGPU source code and install boost, PNGwriter and ADIOS manually.

# this file is loaded from all PIConGPU template *.tpl files, therefore please
# copy this file to $SCRATCH/picongpu.profile
#

# General modules #############################################################
#
# if the wrong environment is loaded we switch to the gnu environment
module li 2>&1 | grep "PrgEnv-cray" > /dev/null
if [ $? -eq 0 ] ; then
    module swap PrgEnv-cray PrgEnv-gnu/6.0.3
else
    module load PrgEnv-gnu/6.0.3
fi

module load CMake/3.8.1
module load cudatoolkit/8.0.54_2.2.8_ga620558-2.1

# Libraries ###################################################################
module load cray-mpich/7.5.0
module load cray-hdf5-parallel/1.10.0

# Other Software ##############################################################
#

# Environment #################################################################
#
# needs to be compiled by the user
export BOOST_ROOT=$SCRATCH/lib/boost-1.62.0
export PNGWRITER_ROOT=$SCRATCH/lib/pngwriter
export ADIOS_ROOT=$SCRATCH/lib/adios-1.11.1

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PNGWRITER_ROOT/lib/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$BOOST_ROOT/lib/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ADIOS_ROOT/lib/

export MPI_ROOT=$MPICH_DIR
export HDF5_ROOT=$HDF5_DIR

# define cray compiler target architecture
# if not defined the linker crashed because wrong from */lib instead
# of */lib64 are used
export CRAY_CPU_TARGET=x86-64

# Compile for cluster nodes
#   (CMake likes to unwrap the Cray wrappers)
export CC=$(which cc)
export CXX=$(which CC)

export PICSRC=$HOME/src/picongpu
export PIC_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE)

export PATH=$PATH:$PICSRC
export PATH=$PATH:$PICSRC/src/tools/bin

export PYTHONPATH=$PICSRC/lib/python:$PYTHONPATH

# send me a mail on BEGIN, END, FAIL, REQUEUE, ALL,
# TIME_LIMIT, TIME_LIMIT_90, TIME_LIMIT_80 and/or TIME_LIMIT_50
export MY_MAILNOTIFY="FAIL"
export MY_MAIL="someone@example.com"
export MY_NAME="$(whoami) <$MY_MAIL>"

# "tbg" default options #######################################################
#   - SLURM (sbatch)
#   - "normal" queue
export TBG_SUBMIT="sbatch"
export TBG_TPLFILE="etc/picongpu/pizdaint-cscs/normal_profile.tpl"

# helper tools ################################################################

# allocate an interactive shell for one hour
#   getInteractive 2  # allocates to interactive nodes (default: 1)
getInteractive() {
    if [ -z "$1" ] ; then
        numNodes=1
    else
        numNodes=$1
    fi
    # --ntasks-per-core=2  # activates intel hyper threading
    salloc --time=1:00:00 --nodes="$numNodes" --ntasks-per-node=12 --ntasks-per-core=2 --partition normal --gres=gpu:1 --constraint=gpu
}

Taurus (TU Dresden)

For this profile to work, you need to download the PIConGPU source code and install PNGwriter and libSplash manually.

module purge

# General modules #############################################################
#
module load oscar-modules
module load cmake/3.9.0
module load git
module load cuda/8.0.44 # gcc <= 5, intel 15-16
module load bullxmpi
module load gnuplot/4.6.1

# Compilers ###################################################################
### GCC
module load gcc/5.3.0
module load boost/1.64.0-gnu5.3
### ICC
#module load intel/2015.3.187 boost/1.59.0-intel2015.3.187
### PGI
#export BOOST_ROOT=$HOME/lib/boost_1_57_pgi_14_9
#export BOOST_INC=$BOOST_ROOT/include
#export BOOST_LIB=$BOOST_ROOT/lib
# must be set in $(which <pgiDir>/bin/localrc):
#   set NOSWITCHERROR=YES;
#module load pgi/14.9 boost/<noneBuildYet>

# Other Software ##############################################################
#
module load hdf5/1.8.18-gcc-5.3.0-xmpi
module load zlib/1.2.8

# Environment #################################################################
#
#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$BOOST_LIB

export PNGWRITER_ROOT=$HOME/lib/pngwriter
export SPLASH_ROOT=$HOME/lib/splash

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/lib/pngwriter/lib/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/lib/splash/lib/

export PICSRC=$HOME/src/picongpu
export PIC_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE)

export PATH=$PATH:$PICSRC
export PATH=$PATH:$PICSRC/src/tools/bin

export PYTHONPATH=$PICSRC/lib/python:$PYTHONPATH

# send me a mail on BEGIN, END, FAIL, REQUEUE, ALL,
# TIME_LIMIT, TIME_LIMIT_90, TIME_LIMIT_80 and/or TIME_LIMIT_50
export MY_MAILNOTIFY="ALL"
export MY_MAIL="someone@example.com"
export MY_NAME="$(whoami) <$MY_MAIL>"

# "tbg" default options #######################################################
#   - SLURM (sbatch)
#   - "gpu" queue
export TBG_SUBMIT="sbatch"
export TBG_TPLFILE="etc/picongpu/taurus-tud/k80_profile.tpl"

Lawrencium (LBNL)

For this profile to work, you need to download the PIConGPU source code and install boost, PNGwriter and libSplash manually. Additionally, you need to make the rsync command available as written below.

if [ -f /etc/profile.d/modules.sh ]
then
        . /etc/profile.d/modules.sh
        module purge

        # Core Dependencies
        module load gcc
        module load cuda
        echo "WARNING: Boost version is too old! (Need: 1.62.0+)" >&2
        # module load boost/1.62.0-gcc
        module load openmpi/1.6.5-gcc

        # Core tools
        module load git
        module load cmake
        module load python/2.6.6
        module load ipython/0.12 matplotlib/1.1.0 numpy/1.6.1 scipy/0.10.0

        # Plugins (optional)
        module load hdf5/1.8.11-gcc-p
        export CMAKE_PREFIX_PATH=$HOME/lib/pngwriter:$CMAKE_PREFIX_PATH
        export CMAKE_PREFIX_PATH=$HOME/lib/libSplash:$CMAKE_PREFIX_PATH
        export LD_LIBRARY_PATH=$HOME/lib/pngwriter/lib:$LD_LIBRARY_PATH
        export LD_LIBRARY_PATH=$HOME/lib/libSplash/lib:$LD_LIBRARY_PATH

        # Debug Tools
        #module load valgrind/3.10.1
        #module load totalview/8.10.0-0

fi

# Environment #################################################################
#
alias allocK20='salloc --time=0:30:00 --nodes=1 --ntasks-per-node=1 --cpus-per-task=8 --partition lr_manycore'
alias allocFermi='salloc --time=0:30:00 --nodes=1 --ntasks-per-node=2 --cpus-per-task=6 --partition mako_manycore'

export PICSRC=$HOME/src/picongpu
export PIC_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE)

# fix pic-create: re-enable rsync
#   ssh lrc-xfer.scs00
#   -> cp /usr/bin/rsync $HOME/bin/
export PATH=$HOME/bin:$PATH

export PATH=$PATH:$PICSRC
export PATH=$PATH:$PICSRC/src/splash2txt/build
export PATH=$PATH:$PICSRC/src/tools/bin

export PYTHONPATH=$PICSRC/lib/python:$PYTHONPATH

# send me a mail on BEGIN, END, FAIL, REQUEUE, ALL,
# TIME_LIMIT, TIME_LIMIT_90, TIME_LIMIT_80 and/or TIME_LIMIT_50
export MY_MAILNOTIFY="ALL"
export MY_MAIL="someone@example.com"
export MY_NAME="$(whoami) <$MY_MAIL>"

# "tbg" default options #######################################################
#   - SLURM (sbatch)
#   - fermi queue (also available: 2 K20 via k20_profile.tpl)
export TBG_SUBMIT="sbatch"
export TBG_TPLFILE="etc/picongpu/lawrencium-lbnl/fermi_profile.tpl"

Judge (FZJ)

(example missing)