HPC GridRunner

#-------------------------------------------------------------------------------------------
[GRID]
# grid type:
gridtype=LSF

# template for a grid submission
cmd=bsub -q regevlab -R "rusage[mem=10]"
# note -e error.file -o out.file are set internally, so dont set them in the above cmd.

# uses the LSF feature to pre-exec and check that the file system is mounted before executing.
# this helps when you have some misbehaving grid nodes that lost certain file mounts.
mount_test=T

##########################################################################################
# settings below configure the Trinity job submission system, not tied to the grid itself.
##########################################################################################

# number of grid submissions to be maintained at steady state by the Trinity submission system
max_nodes=500

# number of commands that are batched into a single grid submission job.
cmds_per_node=100

#--------------------------------------------------------------------------------------------

./hpc_cmds_GridRunner.pl

################################################################
# Required:
#
#  -c <string>        file containing list of commands
#  --grid_conf|G <string>   grid config file
#
# Optional:
#
#  --parafly          if any grid commands fail on the grid, try rerunning
#                     them locally using ParaFly (second chance to succeed).
#                     This requires that ParaFly be installed and in your PATH
#                     Get ParaFly here: http://parafly.sourceforge.net/
#
####################################################################

echo hello1
echo hello2
echo hello3
echoXX hello4

hpc_cmds_GridRunner.pl -c myCommands.txt -G /path/to/grid.conf

bhaas@tin $ ~/GITHUB/HpcGridRunner/hpc_cmds_GridRunner.pl  -c ./myCommands.txt -G ~/GITHUB/HpcGridRunner/hpc_conf/BroadInst_LSF.hour.1.conf
SERVER: tin.broadinstitute.org, PID: 24527
 $VAR1 = {
       'mount_test' => 'T',
       'grid' => 'LSF',
       'cmd' => 'bsub -q hour -W 4:0 -R "rusage[mem=10]"',
       'max_nodes' => '500',
       'cmds_per_node' => '1'
     };
CMDS: 4 / 4  [4/500 nodes in use]
* All cmds submitted to grid.  Now waiting for them to finish.

Failures encountered:
num_success: 3  num_fail: 1     num_unknown: 0
Finished.

1 commands failed during grid computing.
-failed commands written to: ./myCommands.txt.hpc-cache_success.__failures

Error, not all commands could complete successfully... cannot continue.

bhaas@tin $ cat ./myCommands.txt.hpc-cache_success.__failures
    echoXX hello4

echo a | wc -c > a.len
echo ab | wc -c > ab.len
echo abc | wc -c > abc.len
echo hello world | wc > h.whatever

 bhaas@tin $ ~/GITHUB/HpcGridRunner/hpc_cmds_GridRunner.pl  -c ./commands_ex2.txt -G ~/GITHUB/HpcGridRunner/hpc_conf/BroadInst_LSF.hour.1.conf
 SERVER: tin.broadinstitute.org, PID: 19267
 $VAR1 = {
        'mount_test' => 'T',
        'grid' => 'LSF',
        'cmd' => 'bsub -q hour -W 4:0 -R "rusage[mem=10]"',
        'max_nodes' => '500',
        'cmds_per_node' => '1'
      };
CMDS: 4 / 4  [4/500 nodes in use]
* All cmds submitted to grid.  Now waiting for them to finish.
CMDS: 4 / 4  [0/500 nodes in use]
* All nodes completed.  Now auditing job completion status values
All commands completed successfully.
Finished.

All commands completed successfully on the computing grid.

-rw-rw-rw- 1 bhaas broad    2 Jan 26 10:37 ab.len
-rw-rw-rw- 1 bhaas broad    2 Jan 26 10:37 abc.len
-rw-rw-rw- 1 bhaas broad   24 Jan 26 10:37 h.whatever
-rw-rw-rw- 1 bhaas broad    2 Jan 26 10:37 a.len

bhaas@tin $ cat *len
2
3
4

bhaas@tin $ cat h.whatever
   1       2      12

./BioIfx/hpc_FASTA_GridRunner.pl

#######################################################################################################################
#
# Required:
#
#   --query_fasta|Q <string>       query multiFastaFile (full or relative path)
#
#   --cmd_template|T <string>      program command line template:   eg. "/path/to/prog [opts] __QUERY_FILE__ [other opts]"
#
#  --grid_conf|G <string>          grid config file (see hpc_conf/ for examples)
#
#   --seqs_per_bin|N <int>         number of sequences per partition.
#
#   --out_dir|O <string>           output directory
#
# Optional:
#
#   --prep_only|X                  partion data and create cmds list, but don't launch on the grid.
#
#   --parafly                      use parafly to re-exec previously failed grid commands
#
########################################################################################################################