Example libEnsemble Submission Scripts
Below are example submission scripts used to configure and launch libEnsemble on a variety of high-powered systems. See here for more information about the respective systems and configuration.
Slurm - Basic
/examples/libE_submission_scripts/submit_slurm_simple.sh
#!/bin/bash
#SBATCH -J libE_simple
#SBATCH -A <myproject>
#SBATCH -p <partition_name>
#SBATCH -C <constraint_name>
#SBATCH --time 10
#SBATCH --nodes 2
# Usually either -p or -C above is used.
# On some SLURM configurations, these ensure runs can share nodes
export SLURM_EXACT=1
export SLURM_MEM_PER_NODE=0
python libe_calling_script.py --comms local --nworkers 8
Bridges - Central Mode
/examples/libE_submission_scripts/bridges_submit_slurm_central.sh
#!/bin/bash
#SBATCH -J libE_test_central
#SBATCH -N 5
#SBATCH -p RM
#SBATCH -A <my_project>
#SBATCH -o tlib.%j.%N.out
#SBATCH -e tlib.%j.%N.error
#SBATCH -t 00:30:00
# Launch script for running in central mode with mpi4py.
# libEnsemble will run on a dedicated node (or nodes).
# The remaining nodes in the allocation will be dedicated to worker launched apps.
# Initialize Executor with auto-resources=True and central_mode=True.
# User to edit these variables
export EXE=libE_calling_script.py
export NUM_WORKERS=4
mpirun -np $(($NUM_WORKERS+1)) -ppn $(($NUM_WORKERS+1)) python $EXE
# To use local mode instead of mpi4py (with parse_args())
# python $EXE --comms local --nworkers $NUM_WORKERS
Bebop - Central Mode
/examples/libE_submission_scripts/bebop_submit_slurm_central.sh
#!/bin/bash
#SBATCH -J libE_test_central
#SBATCH -N 5
#SBATCH -p knlall
#SBATCH -A <my_project>
#SBATCH -o tlib.%j.%N.out
#SBATCH -e tlib.%j.%N.error
#SBATCH -t 01:00:00
# Launch script for running in central mode with mpi4py.
# libEnsemble will run on a dedicated node (or nodes).
# The remaining nodes in the allocation will be dedicated to worker launched apps.
# Use executor with auto-resources=True and central_mode=True.
# User to edit these variables
export EXE=libE_calling_script.py
export NUM_WORKERS=4
export I_MPI_FABRICS=shm:tmi
# Overcommit will allow ntasks up to the no. of contexts on one node (eg. 320 on Bebop)
srun --overcommit --ntasks=$(($NUM_WORKERS+1)) --nodes=1 python $EXE
# To use local mode instead of mpi4py (with parse_args())
# python calling_script.py --comms local --nworkers $NUM_WORKERS
Bebop - Distributed Mode
/examples/libE_submission_scripts/bebop_submit_slurm_distrib.sh
#!/bin/bash
#SBATCH -J libE_test
#SBATCH -N 4
#SBATCH -p knlall
#SBATCH -A <my_project>
#SBATCH -o tlib.%j.%N.out
#SBATCH -e tlib.%j.%N.error
#SBATCH -t 01:00:00
# Launch script that runs in distributed mode with mpi4py.
# Workers are evenly spread over nodes and manager added to the first node.
# Requires even distribution - either multiple workers per node or nodes per worker
# Option for manager to have a dedicated node.
# Use of MPI Executor will ensure workers co-locate tasks with workers
# If node_list file is kept, this informs libe of resources. Else, libe auto-detects.
# User to edit these variables
export EXE=libE_calling_script.py
export NUM_WORKERS=4
export MANAGER_NODE=false # true = Manager has a dedicated node (assign one extra)
export USE_NODE_LIST=true # If false, allow libE to determine node_list from environment.
# As libE shares nodes with user applications allow fallback if contexts overrun.
unset I_MPI_FABRICS
export I_MPI_FABRICS_LIST=tmi,tcp
export I_MPI_FALLBACK=1
# If using in calling script (After N mins manager kills workers and exits cleanly)
export LIBE_WALLCLOCK=55
#-----------------------------------------------------------------------------
# Work out distribution
if [[ $MANAGER_NODE = "true" ]]; then
WORKER_NODES=$(($SLURM_NNODES-1))
else
WORKER_NODES=$SLURM_NNODES
fi
if [[ $NUM_WORKERS -ge $WORKER_NODES ]]; then
SUB_NODE_WORKERS=true
WORKERS_PER_NODE=$(($NUM_WORKERS/$WORKER_NODES))
else
SUB_NODE_WORKERS=false
NODES_PER_WORKER=$(($WORKER_NODES/$NUM_WORKERS))
fi;
#-----------------------------------------------------------------------------
# A little useful information
echo -e "Manager process running on: $HOSTNAME"
echo -e "Directory is: $PWD"
# Generate a node list with 1 node per line:
srun hostname | sort -u > node_list
# Add manager node to machinefile
head -n 1 node_list > machinefile.$SLURM_JOBID
# Add worker nodes to machinefile
if [[ $SUB_NODE_WORKERS = "true" ]]; then
awk -v repeat=$WORKERS_PER_NODE '{for(i=0; i<repeat; i++)print}' node_list \
>>machinefile.$SLURM_JOBID
else
awk -v patt="$NODES_PER_WORKER" 'NR % patt == 1' node_list \
>> machinefile.$SLURM_JOBID
fi;
if [[ $USE_NODE_LIST = "false" ]]; then
rm node_list
wait
fi;
# Put in a timestamp
echo Starting execution at: `date`
# To use srun
export SLURM_HOSTFILE=machinefile.$SLURM_JOBID
# The "arbitrary" flag should ensure SLURM_HOSTFILE is picked up
# cmd="srun --ntasks $(($NUM_WORKERS+1)) -m arbitrary python $EXE"
cmd="srun --ntasks $(($NUM_WORKERS+1)) -m arbitrary python $EXE $LIBE_WALLCLOCK"
echo The command is: $cmd
echo End PBS script information.
echo All further output is from the process being run and not the script.\n\n $cmd
$cmd
# Print the date again -- when finished
echo Finished at: `date`
Cori - Central Mode
/examples/libE_submission_scripts/cori_submit.sh
#!/bin/bash
#SBATCH -J libE_test_central
#SBATCH -N 5
#SBATCH -q debug
#SBATCH -A <my_project>
#SBATCH -o tlib.%j.%N.out
#SBATCH -e tlib.%j.%N.error
#SBATCH -t 01:00:00
#SBATCH -C knl
# Launch script for running in central mode with mpi4py.
# libEnsemble will run on a dedicated node (or nodes).
# The remaining nodes in the allocation will be dedicated to worker launched apps.
# Use executor with auto-resources=True and central_mode=True.
# User to edit these variables
export EXE=libE_calling_script.py
export NUM_WORKERS=4
export I_MPI_FABRICS=shm:ofi # Recommend OFI
# Ensure anaconda Python module is loaded
module load python/3.7-anaconda-2019.07
# If libensemble is installed under common (set to your install location and python version)
export PYTHONPATH=/global/common/software/<my_project>/<user_name>/packages/lib/python3.7/site-packages:$PYTHONPATH
# Overcommit will allow ntasks up to the no. of contexts on one node (eg. 320 on Bebop)
srun --overcommit --ntasks=$(($NUM_WORKERS+1)) --nodes=1 python $EXE
# To use local mode instead of mpi4py (with parse_args())
# python calling_script.py --comms local --nworkers $NUM_WORKERS
Blues (Blue Gene Q) - Distributed Mode
/examples/libE_submission_scripts/blues_script.pbs
#!/bin/sh
# Job name to be reported by qstat
#PBS -N libE_run
# Declare Job, non-rerunable
#PBS -r n
# Specify name for output log file
#PBS -o log_libE_run
# Join standard output and error so we only get one logfile
#PBS -j oe
# Mail to user on a=abort, b=begin, e=end
#PBS -m aeb
# set the email address where job-related notifications will be sent
#PBS -M name@email.com
# Number of nodes (request 4)
#PBS -l nodes=4:ppn=16
# Specify CPU time needed
#PBS -l walltime=0:01:00
# Select queue
##PBS -q haswell
##PBS -q biggpu
##PBS -q ivy
#PBS -q shared
cd $PBS_O_WORKDIR
# A little useful information for the log file...
echo Manager process running on: $HOSTNAME
echo Directory is: $PWD
echo PBS has allocated the following nodes:
cat $PBS_NODEFILE
NPROCS="$(wc -l < $PBS_NODEFILE)"
echo This job has allocated $NPROCS cores
rm libE_machinefile
# Parse the PBS_NODEFILE to a machinefile (to be used below) in order to put
# ranks 1-4 on each of the 4 nodes, and put rank 0 (the manager) on the first
# node as well.
head -n 1 $PBS_NODEFILE > libE_machinefile
cat $PBS_NODEFILE | sort | uniq >> libE_machinefile
# Put in a timestamp
echo Starting execution at: `date`
pwd
cmd="mpiexec -np 5 -machinefile libE_machinefile python libE_calling_script.py libE_machinefile"
# This note that this command passes the libE_machinefile to both MPI and the
# libE_calling_script, in the latter script, it can be parsed and given to the
# alloc_func
echo The command is: $cmd
echo End PBS script information.
echo All further output is from the process being run and not the pbs script.\n\n
$cmd
# Print the date again -- when finished
echo Finished at: `date`
Theta - On MOM Node with Multiprocessing
/examples/libE_submission_scripts/theta_submit_mproc.sh
#!/bin/bash -x
#COBALT -t 00:30:00
#COBALT -O libE_mproc_MOM
#COBALT -n 4
#COBALT -q debug-flat-quad # Up to 8 nodes only
##COBALT -q default # For large jobs >=128 nodes
##COBALT -A <project code>
# Script to run libEnsemble using multiprocessing on launch nodes.
# Assumes Conda environment is set up.
# To be run with central job management
# - Manager and workers run on launch node.
# - Workers submit tasks to the compute nodes in the allocation.
# Name of calling script
export EXE=libE_calling_script.py
# Communication Method
export COMMS="--comms local"
# Number of workers.
export NWORKERS="--nworkers 4"
# Wallclock for libE (allow clean shutdown)
export LIBE_WALLCLOCK=25 # Optional if pass to script
# Name of Conda environment
export CONDA_ENV_NAME=<conda_env_name>
# Conda location - theta specific
export PATH=/opt/intel/python/2017.0.035/intelpython35/bin:$PATH
export LD_LIBRARY_PATH=~/.conda/envs/$CONDA_ENV_NAME/lib:$LD_LIBRARY_PATH
export PMI_NO_FORK=1 # Required for python kills on Theta
# Unload Theta modules that may interfere with job monitoring/kills
module unload trackdeps
module unload darshan
module unload xalt
# Activate conda environment
export PYTHONNOUSERSITE=1
. activate $CONDA_ENV_NAME
# Launch libE
# python $EXE $NUM_WORKERS > out.txt 2>&1 # No args. All defined in calling script
# python $EXE $COMMS $NWORKERS > out.txt 2>&1 # If calling script is using parse_args()
python $EXE $LIBE_WALLCLOCK $COMMS $NWORKERS > out.txt 2>&1 # If calling script takes wall-clock as positional arg.
Theta - Central Mode with Balsam
/examples/libE_submission_scripts/theta_submit_balsam.sh
#!/bin/bash -x
#COBALT -t 30
#COBALT -O libE_MPI_balsam
#COBALT -n 5
#COBALT -q debug-flat-quad # Up to 8 nodes only # Use default for >=128 nodes
#COBALT -A <project code>
# Script to launch libEnsemble using Balsam.
# Assumes Conda environment is set up.
# Requires Balsam is installed and a database initialized.
# To be run with central job management
# - Manager and workers run on one node (or a dedicated set of nodes).
# - Workers submit tasks to the rest of the nodes in the pool.
# Name of calling script
export EXE=libE_calling_script.py
# Number of workers.
export NUM_WORKERS=4
# Number of nodes to run libE
export LIBE_NODES=1
# Balsam wall-clock in minutes - make few mins smaller than batch wallclock
export BALSAM_WALLCLOCK=25
# Name of working directory within database where Balsam places running jobs/output
export WORKFLOW_NAME=libe_workflow
# Wall-clock in mins for libE (allow clean shutdown).
# Script must be set up to receive as argument.
export LIBE_WALLCLOCK=$(($BALSAM_WALLCLOCK-3))
# libEnsemble calling script arguments (some alternatives shown)
# No args. All defined in calling script
export SCRIPT_ARGS=''
# If calling script takes wall-clock as positional argument.
# export SCRIPT_ARGS="$LIBE_WALLCLOCK"
# Name of Conda environment
export CONDA_ENV_NAME=<conda_env_name>
# Name of database
export BALSAM_DB_NAME=<dbase_name> # default - to use default database.
# Required for killing tasks from workers on Theta
export PMI_NO_FORK=1
# Unload Theta modules that may interfere with job monitoring/kills
module unload trackdeps
module unload darshan
module unload xalt
# Obtain Conda PATH from miniconda-3/latest module
CONDA_DIR=/soft/datascience/conda/miniconda3/latest/bin
# Ensure environment isolated
export PYTHONNOUSERSITE=1
# Activate conda environment
source $CONDA_DIR/activate $CONDA_ENV_NAME
# Activate Balsam database
source balsamactivate $BALSAM_DB_NAME
# Currently need at least one DB connection per worker (for postgres).
if [[ $NUM_WORKERS -gt 100 ]]
then
#Add a margin
echo -e "max_connections=$(($NUM_WORKERS+20)) #Appended by submission script" \
>> $BALSAM_DB_PATH/balsamdb/postgresql.conf
fi
wait
# Make sure no existing apps/tasks registered to database
balsam rm apps --all --force
balsam rm jobs --all --force
wait
sleep 3
# Add calling script to Balsam database as app and job.
export THIS_DIR=$PWD
export SCRIPT_BASENAME=${EXE%.*}
# Multiple nodes
export LIBE_PROCS=$((NUM_WORKERS+1)) # Manager and workers
export PROCS_PER_NODE=$((LIBE_PROCS/LIBE_NODES)) # Must divide evenly
balsam app --name $SCRIPT_BASENAME.app --exec $EXE --desc "Run $SCRIPT_BASENAME"
balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME \
--application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS \
--wall-time-minutes $LIBE_WALLCLOCK \
--num-nodes $LIBE_NODES --ranks-per-node $PROCS_PER_NODE \
--url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" \
--url-in="local:/$THIS_DIR/*" --yes
# Hyper-thread libE (note this will not affect HT status of user calcs - only libE itself)
# E.g. Running 255 workers and one manager on one libE node.
# balsam job --name job_$SCRIPT_BASENAME --workflow $WORKFLOW_NAME \
# --application $SCRIPT_BASENAME.app --args $SCRIPT_ARGS \
# --wall-time-minutes $LIBE_WALLCLOCK \
# --num-nodes 1 --ranks-per-node 256 --threads-per-core 4 \
# --url-out="local:/$THIS_DIR" --stage-out-files="*.out *.txt *.log" \
# --url-in="local:/$THIS_DIR/*" --yes
# Run job
balsam launcher --consume-all --job-mode=mpi --num-transition-threads=1
wait
source balsamdeactivate
Summit - On Launch Nodes with Multiprocessing
/examples/libE_submission_scripts/summit_submit_mproc.sh
#!/bin/bash -x
#BSUB -P <project code>
#BSUB -J libe_mproc
#BSUB -W 30
#BSUB -nnodes 4
#BSUB -alloc_flags "smt1"
# Script to run libEnsemble using multiprocessing on launch nodes.
# Assumes Conda environment is set up.
# To be run with central job management
# - Manager and workers run on launch node.
# - Workers submit tasks to the compute nodes in the allocation.
# Name of calling script-
export EXE=libE_calling_script.py
# Communication Method
export COMMS="--comms local"
# Number of workers.
export NWORKERS="--nworkers 4"
# Wallclock for libE. (allow clean shutdown)
export LIBE_WALLCLOCK=25 # Optional if pass to script
# Name of Conda environment
export CONDA_ENV_NAME=<conda_env_name>
# Need these if not already loaded
# module load python
# module load gcc/4.8.5
# Activate conda environment
export PYTHONNOUSERSITE=1
. activate $CONDA_ENV_NAME
# hash -d python # Check pick up python in conda env
hash -r # Check no commands hashed (pip/python...)
# Launch libE
# python $EXE $NUM_WORKERS > out.txt 2>&1 # No args. All defined in calling script
# python $EXE $COMMS $NWORKERS > out.txt 2>&1 # If calling script is using parse_args()
python $EXE $LIBE_WALLCLOCK $COMMS $NWORKERS > out.txt 2>&1 # If calling script takes wall-clock as positional arg.