[Sashimi-commit] SF.net SVN: sashimi:[5120] trunk/trans_proteomic_pipeline/extern/hpctools/ grid/bin

[<sla...@us...>]

Revision: 5120
          http://sashimi.svn.sourceforge.net/sashimi/?rev=5120&view=rev
Author:   slagelwa
Date:     2010-08-25 00:08:14 +0000 (Wed, 25 Aug 2010)

Log Message:
-----------
Major refinements.  Now writes "qjob" files and removes job files when done.

Modified Paths:
--------------
    trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qconvert
    trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qinspect
    trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qomssa
    trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qspectrast
    trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qtandem
    trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qtpp
    trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/runtpp

Added Paths:
-----------
    trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qgrid_functions

Removed Paths:
-------------
    trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qgrid_functions

Modified: trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qconvert
===================================================================
--- trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qconvert	2010-08-24 22:43:24 UTC (rev 5119)
+++ trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qconvert	2010-08-25 00:08:14 UTC (rev 5120)
@@ -3,7 +3,7 @@
 # Program: TPP HPC Tools
 # Author:  Joe Slagel
 #
-# Copyright (C) 2009 by Joseph Slagel
+# Copyright (C) 2009-2010 by Joseph Slagel
 # 
 # This library is free software; you can redistribute it and/or             
 # modify it under the terms of the GNU Lesser General Public                
@@ -27,31 +27,25 @@
 # $Id: $
 #
 
-. qgrid_functions                               # Load grid functions
-
 # -- Local settings ------------------------------------------------------------
 
-   # What command tool are we
-   QJOBNAME=${QJOBNAME-TPP-convert}
-   
    # Default queue to submit jobs to
-   QQUEUE=${QQUEUE-convert}
+   QQUEUE=${QQUEUE:-convert}
    
-   # Additional commands to run at the end
-#   QEND=${QEND-}
-
-   # Default remote convert flags
-   REMOTECONVERTFLAGS=${REMOTECONVERTFLAGS-} 
+   # Default name for jobs in queue
+   QNAME=${QNAME:-TPP-convert}
    
    # Programs
-   QSUB=${QSUB-`which qsub 2>/dev/null`}
    REMOTECONVERT=`which remoteconvert.pl 2>/dev/null`
+   
+   # Default remote convert options
+   REMOTECONVERTFLAGS="${REMOTECONVERTFLAGS-} --log2screen"
 
 # -----------------------------------------------------------------------------
 
 # 
 # Output a message to STDERR on usage and exit with a error status
-# @param  mesg(s) to print
+# @param  text to print
 #
 function usage() {
    [ "$1" ] && echo "$(basename $0): $*" 1>&2
@@ -60,43 +54,35 @@
    echo
    echo "Options:"
    echo "   -h, --help          help"
-   echo "   -m, --manual        manual"
+   echo "   -m, --man           manual"
    echo
-   echo "[remoteconvert opts]   See remoteconvert.pl for options"
+   echo "[remoteconvert opts]   See 'remoteconvert.pl -h' for valid options"
    echo "[qsub options]         All flags following '--' are passed to qsub"
    exit 1
 }
 
 #
-# Write search commands to qsub script
+# List of commands to run in the qsub script.  Commands are echo'ed out and
+# are intended to be read in by the qsubmit() function.
 #
-function write_qsub {
-   rm -f $root.convert.qsub $root.convert.log
-   cat <<EOF > $1
-$(qsub_header $root.convert.log)
+function cmds() {
+   cat <<EOF
+$REMOTECONVERT $REMOTECONVERTFLAGS "$input"
 
-log "CONVERT $input"
-$REMOTECONVERT $REMOTECONVERTFLAGS --log2screen "$input"
-[ -f convert.log ] && cat convert.log
-rm -f convert.log
-
 # Did it produce a file?
 if [ ! -f "${input%.*}".mz*ML ]; then
    echo "Error: missing mzML/mzXML file for $input"
    exit 1
 fi
+EOF
+}
 
-# Extra commands to run
-${QEND}
+# -- Main ----------------------------------------------------------------------
 
-log "DONE $input"
+source qgrid_functions                          # Load "Q" grid functions
 
-EOF
-}
-
 # Check for necessary programs
-[ ! -x ${QSUB} ]          && usage "Error no qsub in your path"
-[ ! -x ${REMOTECONVERT} ] && usage "Error no remoteconvert in your path"
+[ -x ${REMOTECONVERT} ] || die "Error: no remoteconvert.pl in your path"
 
 # Check options
 INPUTS=
@@ -110,12 +96,12 @@
    -h|--help)
       usage
       ;;
-   -m|--manual)
+   -m|--man)
       pod2man -n 'HPC TOOLS' -c 'Trans-Proteomic Pipeline (TPP)' $0 | nroff -man | less
       exit
       ;;
    *.d|*.wiff|*.RAW|*.baf|*.raw)
-      [ -e "$1" ] || usage "Error $1 file not found";
+      [ -e "$1" ] || usage "Error: $1 file not found";
       INPUTS="$INPUTS $1"
       shift
       ;;
@@ -125,6 +111,7 @@
       ;;
    esac
 done
+[ -z "$INPUTS" ] && usage "Error missing one or more input files"
 
 # Submit jobs
 for input in $INPUTS; do
@@ -132,36 +119,48 @@
    dir=`dirname "$abs"`	                        # base directory of input
    root=$(basename "${input%.*}")               # filename w/o extension
    
-   write_qsub "$root.convert.qsub"
+   rm -f $root.{mzML,mzXML}                     # remove previous results
    
-   log "SUBMIT $input"
-   $QSUB "$dir/$root.convert.qsub"
-   [ $? -ne 0 ] && exit 1
+   cmds | submit "$dir/$root.convert" "$dir"    # submit job
 done
 
 
 # -- POD DOCUMENTATION ------------------------------------------------------
 #
-# Documentation for wrappers.  Uses pod with a shell trick.  Simply run
+# Documentation for command.  Uses perl's POD with a shell trick.  Simply run
 # pod2man on this file to get the documentation.
 #
 : <<POD
 
 =head1 NAME
 
-qconvert - Use msconvert(via remoteconvert.pl) to convert a MS file
+qconvert - converts MS run files into mzML/MzXML formatted files
 
 =head1 SYNOPSIS            
 
 qconvert [options] [remoteconvert opts] file(s) [-- [qsub options]]
 
  Options:
-   -h, --help           display this help and exit
+   -h, --help           Displays a short help about the command
+   -m, --man            Displays the manual page for the command
    
 =head1 DESCRIPTION
 
-Simple script which submits a job to remotely convert a mass spec run.
+A "Q" script which for every given file submits a new cluster job that converts
+the file from their original mass spectrometry proprietary format to the open
+standards mzML (or mzXML) format.
 
+This script uses a ISB in-house tool called  B<remoteconvert.pl> to manage the 
+actual conversions.  It copies the files to convert to a Windows OS based
+computer that has the various converter tools already installed on it and then
+runs appropriate converter based on the type of file to convert. The resulting
+output files are then copied back from the Windows OS based computer to the
+local directory.
+
+By default the files are converted using the program "msconvert" from
+ProtoWizard and the output format is mzML.  To override these options, this
+script accepts the same options as the B<remoteconvert.pl> program.
+
 =head1 OPTIONS
 
 =over 5
@@ -174,6 +173,16 @@
 
 Print a longer version of the help.
 
+=item B<remoteconvert.pl options>
+
+Run remoteconvert.pl -h to see what all options are accepted.
+
+=item B<qsub options>
+
+It is possible to pass options to the qsub command directly (the command which
+submits the batch job script) by placing these options after a "--" indicator.
+Please man qsub to see what all options are accepted.
+
 =back
 
 =head1 OUTPUT
@@ -202,6 +211,26 @@
 
 =head1 EXAMPLE 2
 
+To convert all of the .d runs in the current directory:
+
+=over 5
+
+S<qconvert *.d>
+ 
+=back
+ 
+=head1 EXAMPLE 3
+
+To convert a single .d directory into a mzXML file:
+
+=over 5
+
+S<qconvert --mzXML mydata.d>
+ 
+=back
+ 
+=head1 EXAMPLE 4
+
 To convert a single .d directory using a different queue:
 
 =over 5
@@ -210,16 +239,23 @@
 
 =back
 
+This command will submit the convert job to the queue named "serial" in the
+cluster.
+
 =head1 ADDITIONAL RESOURCES
 
 =item L<http://proteowizard.sourceforge.net/>
 
 ProteoWizard toolset which includes msconvert, the HUPO-PSI standard compliant
 conversion tool.
+ 
+=item L<http://tools.proteomecenter.org/wiki/index.php?title=Formats:mzXML>
 
+Details on the various mass spectroscopy data file formats and their converters.
+
 =item remoteconvert.pl -h
 
-Remote conversion tool.
+ISB remote conversion tool.
 
 =head1 AUTHORS
 

Deleted: trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qgrid_functions
===================================================================
--- trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qgrid_functions	2010-08-24 22:43:24 UTC (rev 5119)
+++ trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qgrid_functions	2010-08-25 00:08:14 UTC (rev 5120)
@@ -1,433 +0,0 @@
-#!/bin/bash
-#
-# Program: TPP HPC Tools
-# Author:  Joe Slagel
-#
-# Copyright (C) 2009 by Joseph Slagel
-# 
-# This library is free software; you can redistribute it and/or             
-# modify it under the terms of the GNU Lesser General Public                
-# License as published by the Free Software Foundation; either              
-# version 2.1 of the License, or (at your option) any later version.        
-#                                                                           
-# This library is distributed in the hope that it will be useful,           
-# but WITHOUT ANY WARRANTY; without even the implied warranty of            
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU         
-# General Public License for more details.                                  
-#                                                                           
-# You should have received a copy of the GNU Lesser General Public          
-# License along with this library; if not, write to the Free Software       
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
-# 
-# Institute for Systems Biology
-# 1441 North 34th St.
-# Seattle, WA  98103  USA
-# js...@sy...
-#
-# $Id: $
-#
-
-#
-# Common BASH Shell Functions used by the set of "q*" search/TPP job
-# submission commands
-#
-
-# -- Local settings -----------------------------------------------------------
-
-   # Default grid queue to submit searches to
-   TPP_SEARCHQUEUE=${TPP_SEARCHQUEUE-}
-   
-   # Default grid queue to submit searches to
-   TPP_ANALYSISQUEUE=${TPP_ANALYSISQUEUE-}
-   
-   # File to log messages to
-   QLOG=${QLOG-/proteomics/sw/log/qgrid.log}
-   
-   # Comma separated list of users to email results to
-   QMAIL=${QMAIL-}
-   
-   # Path to  qsub command
-   QSUB=${QSUB-`which qsub 2>/dev/null`}        
-   
-   # Default qsub flags to use
-   QSUBFLAGS=
-
-   # Extra commands to invoke when exiting
-   QEXIT=${QEXIT-}
-
-
-# -- Functions -----------------------------------------------------------------
-
-# 
-# Output a message to STDERR on usage and exit with a error status
-# @param  mesg(s) to print
-#
-function usage() {
-   [ "$1" ] && echo "$(basename $0): $*" 1>&2
-   echo "Usage:"
-   echo "$(basename $0) [options] *.mzXML|*.mzML [-- [qsub options]]"
-   echo
-   echo "Options:"
-   echo "   -h, --help          help"
-   echo "   -m, --manual        manual"
-   echo "   --params <file>     parameter file to use (defaults to <search>.params)"
-   echo "   --dest <dir>        destination directory for search files"
-   echo
-   echo "[qsub options]         all flags following '--' are passed to qsub"
-   echo
-   exit 1
-}
-
-#
-# Log a message to the QLOG file
-#
-log () { echo "[`date --rfc-3339=sec`] $USER ($QJOBNAME) $*" >> ${QLOG:-/dev/null}; }
-
-#
-# Check search script input:
-#   Sets INPUTS to list of input files
-#   Sets PARAMS to parameter file
-#   Sets QSUBFLAGS to any qsub flags
-#
-function search_input () {
-   INPUTS=
-   while [ "$1" ]; do
-      case "$1" in 
-      --)
-         shift
-         QSUBFLAGS=$*
-         break
-         ;;
-      -h|--help)
-         usage
-         ;;
-      -m|--manual)
-         POD=`dirname $0`/qgrid_functions
-         pod2man -n 'HPC TOOLS' -c 'Trans-Proteomic Pipeline (TPP)' $POD | nroff -man | less
-         exit
-         ;;
-      --params)
-         PARAMS="$2"
-         shift 2
-         ;;
-      --dest)
-         DEST="$2"
-         shift 2
-         ;;
-      *.mzML|*.mzXML|*.mzXML.gz|*.mgf)
-         [ -f "$1" ] || usage "Error $1 file not found";
-         INPUTS="$INPUTS $1"
-         shift
-         ;;
-      *)
-         usage "invalid option $1"; 
-         exit 1
-         ;;
-      esac
-   done
-   [ -z "$PARAMS" ] && usage "Missing required <search>.params file"
-   [ -f "$PARAMS" ] || usage "Error '$PARAMS' parameters file not found"
-   [ -z "$INPUTS" ] && usage "Error missing one or more mzXML/mzML files"
-   [ ! -z "$DEST" ] && make_destination $DEST $INPUTS
-}
-
-# Absolute to relative path
-function abs2rel () {
-   perl -MFile::Spec -e "print File::Spec->abs2rel( '$1', '$2' )"
-}
-
-# Relative to absolute path
-function rel2abs() {
-   perl -MFile::Spec -e "print File::Spec->rel2abs( '$1', '$2' )"
-}
-
-#
-# Create a destination directory and link files
-#
-function make_destination () {
-   DEST=`rel2abs $1`            # abs path for destination directory
-   shift
-   mkdir -p $DEST
-   
-   # for parameters file(s)...
-   P=${PARAMS-tandem.params omssa.params inspect.params sequest.params}
-   for p in $P; do
-      echo p=$p
-      [ ! -f $p ] && continue
-      cp -f $p $DEST
-      break
-   done
-   
-   # for each input file...
-   for f in $*; do                      
-      f=`rel2abs $f`;           # 1st make sure its absolute
-      r=`abs2rel $f $DEST`      # calculate relative path to it
-      ln -s $r $DEST
-   done
-   cd '$DEST'
-}
-
-#
-# Email notification 
-#
-function email_notify() {
-   SUBJ="$1"
-   shift
-   if [ "$1" != "" ]; then
-      TO=`echo "$1" | tr "," "\n" | sort | uniq | tr "\n" ","`
-      QJOB=`echo ${PBS_JOBID} | cut -d'.' -f1`
-      BODY=(/PBS/spool/${QJOB}.*.OU)
-      if [ ! -f "$BODY" ]; then
-         BODY=/dev/null
-      fi
-      cat "$BODY" | mail -v -s "$SUBJ" "$TO" > /dev/null
-      echo mailed "$BODY" to "$TO"
-   fi
-}
-
-#
-# Outputs a "qsub" style shell script header that can used in a qsub script
-# to submit to a grid cluster.
-#
-function qsub_header()
-{
-   local QSUB_LOG=${1}
-   local QSUB_DIR=${2:-$PWD} 
-   
-   # Write out qsub script, be wary of escaping environment variables
-   cat <<QSUB_SCRIPT
-#!/bin/bash
-
-### Grid Directives
-#PBS ${QJOBNAME:+-N $QJOBNAME} ${QQUEUE:+-q $QQUEUE}
-#PBS -S /bin/bash
-#PBS -v PATH 
-#PBS -d '$QSUB_DIR'
-#PBS -j oe ${QSUB_LOG:+-o $QSUB_LOG}
-#PBS ${QSUBFLAGS}
-
-#$ -S /bin/bash
-#$ ${QJOBNAME:+-N $QJOBNAME} ${QPROJECT:+-P $QPROJECT}
-#$ -v PATH -cwd
-#$ -terse -j y ${QSUB_LOG:+-o $QSUB_LOG}
-#$ ${QSUBFLAGS}
-### Grid Directives
-
-# Make absolutely sure we are in the working directory
-cd '$QSUB_DIR'
-QJOBNAME=$QJOBNAME
-QLOG=$QLOG
-
-echo
-echo "==== Job Environment ===="
-echo "Date     = "\`date\`
-echo "Hostname = "\`hostname\`
-echo "Username = "\`whoami\`
-echo "PID      = \$PID"
-echo "PWD      = \$PWD"
-echo "PATH     = \$PATH"
-if [ "\$PBS_O_HOST" != "" ]; then
-   echo
-   echo "PBS_O_HOST    = \$PBS_O_HOST"
-   echo "PBS_O_QUEUE   = \$PBS_O_QUEUE"
-   echo "PBS_O_WORKDIR = \$PBS_O_WORKDIR"
-   echo "PBS_JOBID     = \$PBS_JOBID"
-   echo "PBS_JOBNAME   = \$PBS_JOBNAME"
-   echo "PBS_QUEUE     = \$PBS_QUEUE"
-fi
-if [ "\$SGE_ROOT" != "" ]; then
-   echo
-   echo "SGE_ROOT      = \$SGE_ROOT"
-   echo "SGE_O_HOST    = \$SGE_O_HOST"
-   echo "SGE_O_WORKDIR = \$SGE_O_WORKDIR"
-   echo "JOB_ID        = \$JOB_ID"
-   echo "JOB_NAME      = \$JOB_NAME"
-   echo "QUEUE         = \$QUEUE"
-fi
-echo "==== Job Environment ===="
-echo ""
-
-. qgrid_functions				# Load grid functions
-
-#
-# Function to catch errors/signals and report them
-#
-function on_exit() {
-   local exited=\${1:-\$?}
-   trap - HUP INT QUIT TERM ERR EXIT
-   
-   # Signal or error?
-   if [ \$exited -gt 128 -a \$exited -lt 166 ]; then
-      exited=\$(( exited - 128 ))
-      QSTATUS="ABORT(\$exited) \$JOB_ID\$PBS_JOBID"
-   elif [ \$exited -ne 0 ]; then
-      QSTATUS="ERROR(\$exited) \$JOB_ID\$PBS_JOBID"
-   else
-      QSTATUS="COMPLETED \$JOB_ID\$PBS_JOBID"
-   fi
-   log \$QSTATUS
- 
-   # Extra commands to invoke when exiting
-   $QEXIT
-
-   # Depending on cluster either exit with special abort code or just status
-   [ ! -z "\$SGE_ROOT" ] && exit 100    # SGE exit w/special 100 "abort" code 
-   exit \$exited                        # PBS exit status
-}
-trap on_exit HUP INT QUIT TERM ERR EXIT
-
-QSUB_SCRIPT
-
-}
-
-
-# -- POD DOCUMENTATION ------------------------------------------------------
-#
-# Documentation for wrappers.  Uses pod with a shell trick.  Simply run
-# pod2man on this file to get the documentation.
-#
-: <<POD
-
-=head1 NAME
-
-qtandem  - Submit a Tandem MS/MS peptide search to a cluster
-
-qsomssa  - Submit a OMSSA MS/MS peptide search to a cluster 
-
-qinspect - Submit a InsPect MS/MS peptide search to a cluster
-
-=head1 SYNOPSIS            
-
-q<search program> [options] *.mzXML|*.mzML [-- [qsub options]]
-
- Options:
-   -h, --help           display this help and exit
-   --params <file>      file containing MS search algorithm parameters 
-   --dest <dir>         directory as the destination for output files  
-   
- [qsub options]         all options following '--' are passed to qsub
- 
-=head1 DESCRIPTION
-
-Collection of wrapper programs which can be used to submit MS/MS peptide
-spectra searches to a OpenPBS or Sun Grid Engine cluster.
-
-=head1 OPTIONS
-
-=over 5
-
-=item B<-h, --help>
-
-Print a brief help message and exit.
-
-=item B<--params FILE>
-
-Specify the parameters file to use in the search.  If not provided then the
-program will default to a file named "I<search>.params" in the local current
-directory, where I<search> is the name of the search program to invoke.
-E.g. qtandem will defaults to "tandem.params".
-
-=item B<--dest DIR>
-
-Specifies the destination directory of the search results.  If the destination
-directory doesn't exist it will be created first then links to all input files
-will be placed in this destination along with all output files from the search.
-No care is taken to prevent overwriting any existing files.
-
-=back
-
-Finally any options following "--" will be used as additional parameters to
-the qsub command.  So for example to change the queue that the job is submitted
-to you could run the command "qtandem *.mzXML -- -q serial".
-
-=head1 PREREQUSITES
-
-First off you can only invoke these commands on the head node of the cluster
-(or any system that has permissions to submit jobs to the cluster). Secondly,
-these programs require the appropriate search programs be installed on the
-cluster and in an accessible location for all nodes in the cluster.  Lastly each
-program assumes that local working directory is the same on all nodes and that
-any additional databases or auxiliary files are also available on all nodes.
-
-=over 5
-
-=item I<Sun Grid Engine>
-
-In order to use SGE you need to ensure that your shell environment is setup
-correctly.  In most cases your system administrator should already have provided
-local system settings in your login files.  You can check to see if your
-settings are correct by examining your environment for any variables beginning
-with the name "SGE_".
-
-=item I<PBS> 
-
-Nothing special.  Just need to have the qsub command in your path. 
-
-
-=item I<InsPect>
-
-Uses the InsPect program developed at the University of California, San Diego.
-You'll have to install this program locally on your cluster and ensure that
-the program is in your path. The location of the resource files  used by 
-I<inspect> will be inferred by the location of the program itself as this
-is the typical setup for a installation.
-
-=back
-
-=head1 OUTPUT
-
-All of these programs will print a job identifier for each search job submitted.
-See either OpenPBS qsub or Sun Grid Engine's qsub command manual for more 
-information about the format of the job id and how to use them.
-
-Each search program's output (both STDOUT and STDERR) will be captured in a 
-job specific "<input file>.<search>.log" file.  This file is written by the
-grid software when the job finishes.
-
-=head1 EXAMPLE 
-
-To run a search on the cluster you'll need to provide two things, a search 
-parameters file and one or more mzXML formatted files containing mass 
-spectroscopy data. 
-
-So for example:
-
-=over 5
-
-S<qtandem --params tandem.params foo.mzXML>
-
-=back
-
-This will submit a tandem search job to the cluster for foo.mzXML in the
-current directory.  The results will be written to the file foo.pep.xml and
-any output for the job will be captured in  foo.tandem.log.
-
-=head1 ADDITIONAL RESOURCES
-
-=item L<http://gridengine.sunsource.net/>
-
-Sun Grid Engine project.
-
-=item L<http://www.clusterresources.com/products/torque-resource-manager.php>
-
-TORQUE resource manager (OpenPBS)
-
-=item L<http://www.thegpm.org/tandem/index.html>
-
-X!Tandem  open source software.
-
-=item L<http://pubchem.ncbi.nlm.nih.gov/omssa/>
-
-The Open Mass Spectrometry Search Algorithm [OMSSA]
-
-=item L<http://proteomics.ucsd.edu/>
-
-InsPecT: A Proteomics Search Toolkit
-
-=head1 AUTHORS
-
-Joe Slagel E<lt>js...@sy...<gt>
-
-=cut
-
-POD

Added: trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qgrid_functions
===================================================================
--- trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qgrid_functions	                        (rev 0)
+++ trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qgrid_functions	2010-08-25 00:08:14 UTC (rev 5120)
@@ -0,0 +1,468 @@
+#!/bin/bash
+#
+#
+# Program: TPP HPC Tools
+# Author:  Joe Slagel
+#
+# Copyright (C) 2009 by Joseph Slagel
+# 
+# This library is free software; you can redistribute it and/or             
+# modify it under the terms of the GNU Lesser General Public                
+# License as published by the Free Software Foundation; either              
+# version 2.1 of the License, or (at your option) any later version.        
+#                                                                           
+# This library is distributed in the hope that it will be useful,           
+# but WITHOUT ANY WARRANTY; without even the implied warranty of            
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU         
+# General Public License for more details.                                  
+#                                                                           
+# You should have received a copy of the GNU Lesser General Public          
+# License along with this library; if not, write to the Free Software       
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
+# 
+# Institute for Systems Biology
+# 1441 North 34th St.
+# Seattle, WA  98103  USA
+# js...@sy...
+#
+# $Id: $
+#
+
+#
+# Common BASH Shell Functions used by the set of "q" scripts used to submit
+# jobs to run some of the various TPP programs
+#
+# Note there are a number of different environment variables used by these
+# functions that should be set in the "Q" scripts. All begin with the letter
+# "Q" and the default values are set in the local settings below.
+#
+
+# -- Local settings -----------------------------------------------------------
+#
+# Feel free to modify these variables as needed for your site.
+#
+   # Name of the "q" command used for job name and logging
+   QNAME=${QNAME:-unknown}
+   
+   # Queue to use for job submission
+   QQUEUE=${QQUEUE-}
+   
+   # File to log event messages to
+   QLOG=${QLOG:-/proteomics/sw/log/qgrid.qlog}
+   
+   # Path to the qsub command
+   QSUB=${QSUB-`which qsub 2>/dev/null`}        
+   
+   # Default qsub flags to use
+   QSUBFLAGS=
+
+   # Extra commands to invoke when finished with the commands
+   QEXTRA=${QEXTRA-}
+
+   # Comma separated list of users to email results to
+   QMAIL=${QMAIL-}
+   
+
+# -- Functions -----------------------------------------------------------------
+
+# Log a event message to the QLOG file
+function qlog () { 
+   echo "[`date --rfc-3339=sec`] $USER ($QNAME) $*" >> ${QLOG:-/dev/null}
+}
+
+# Simple "die" function
+function die () {
+   echo "$(basename $0): ${1:-"Unknown error"}" 1>&2
+   exit 1
+}
+
+# Absolute to relative path
+function abs2rel () {
+   perl -MFile::Spec -e "print File::Spec->abs2rel( '$1', '$2' )"
+}
+
+# Relative to absolute path
+function rel2abs() {
+   perl -MFile::Spec -e "print File::Spec->rel2abs( '$1', '$2' )"
+}
+
+# 
+# All of the MS-MS search commands (qtandem, qinspect, ...) should have a
+# this common command line interface
+#
+function search_usage() {
+   [ "$1" ] && echo "$(basename $0): $*" 1>&2
+   echo "Usage:"
+   echo "$(basename $0) [options] $EXTS [-- [qsub options]]"
+   echo
+   echo "Options:"
+   echo "   -h, --help          help"
+   echo "   -m, --manual        manual"
+   echo "   -p,--params <file>  parameter file to use (defaults to <search>.params)"
+   echo "   -d,--dest <dir>     destination directory for search files"
+   echo
+   echo "   [qsub options]      all flags following '--' are passed to qsub"
+   echo
+   exit 1
+}
+
+#
+# All of the MS-MS search commands (qtandem, qinspect, ...) should have a
+# this common command line interface
+#
+#   Sets PARAMS to be the search parameter file
+#   Sets INPUTS to list of input files to search
+#   Appends any qsub flags to QSUBFLAGS 
+#
+function search_options() {
+   PARAMS=$1; shift
+   EXTS=$1; shift
+   
+   INPUTS=
+   while [ "$1" ]; do
+      case "$1" in 
+      --)
+         shift
+         QSUBFLAGS+=" $*"
+         break
+         ;;
+      -h|--help)
+         search_usage
+         ;;
+      -m|--man)
+         POD=`dirname $0`/qgrid_functions
+         pod2man -n 'HPC TOOLS' -c 'Trans-Proteomic Pipeline (TPP)' $POD | nroff -man | less
+         exit
+         ;;
+      -p|--params)
+         PARAMS="$2"
+         shift 2
+         ;;
+      -d|--dest)
+         DEST="$2"
+         shift 2
+         ;;
+      -*)
+         search_usage "invalid option $1"
+         ;;
+      *)
+         eval "case "$1" in $EXTS) ;; *) search_usage "invalid file $1";; esac"
+         [ -f "$1" ] || search_usage "$1 file not found"
+         INPUTS+=" $1"
+         shift
+         ;;
+      esac
+   done
+   [ -z "$PARAMS" ] && search_usage "Missing required <search>.params file"
+   [ -f "$PARAMS" ] || search_usage "Error '$PARAMS' parameters file not found"
+   [ -z "$INPUTS" ] && search_usage "Error missing one or more input files"
+   [ ! -z "$DEST" ] && make_destination $DEST $INPUTS
+}
+
+#
+# Create a destination directory and link files
+#
+function make_destination () {
+   DEST=`rel2abs $1`            # absolute path for destination directory
+   shift
+   mkdir -p $DEST
+   
+   # for parameters file(s)...
+   P=${PARAMS-tandem.params omssa.params inspect.params sequest.params}
+   for p in $P; do
+      echo p=$p
+      [ ! -f $p ] && continue
+      cp -f $p $DEST
+      break
+   done
+   
+   # for each input file...
+   for f in $*; do                      
+      f=`rel2abs $f`;           # 1st make sure its absolute
+      r=`abs2rel $f $DEST`      # calculate relative path to it
+      ln -s $r $DEST
+   done
+   cd '$DEST'
+}
+
+#
+# Email notification 
+#
+function email_notify() {
+   SUBJ="$1"
+   shift
+   if [ "$1" != "" ]; then
+      TO=`echo "$1" | tr "," "\n" | sort | uniq | tr "\n" ","`
+      QJOB=`echo ${PBS_JOBID} | cut -d'.' -f1`
+      BODY=(/PBS/spool/${QJOB}.*.OU)
+      if [ ! -f "$BODY" ]; then
+         BODY=/dev/null
+      fi
+      cat "$BODY" | mail -v -s "$SUBJ" "$TO" > /dev/null
+      echo mailed "$BODY" to "$TO"
+   fi
+}
+
+#
+# Writes a "qsub" file and submits it to the cluster.
+#
+function submit()
+{
+   local QROOT=${1}
+   local QDIR=${2:-$PWD} 
+   
+   [ -x ${QSUB} ] || die "Error: no qsub in your path"
+   
+   # Write out qsub script, BE WARY of escaping environment variables
+   cat <<QSUB_SCRIPT > $QROOT.qsub
+#!/bin/bash
+
+### Grid Directives
+#PBS ${QNAME:+-N $QNAME} ${QQUEUE:+-q $QQUEUE}
+#PBS -S /bin/bash
+#PBS -v PATH 
+#PBS -d '$QDIR'
+#PBS -j oe -o '$QROOT.qlog'
+#PBS ${QSUBFLAGS}
+
+#$ -S /bin/bash
+#$ ${QNAME:+-N $QNAME} ${QPROJECT:+-P $QPROJECT}
+#$ -v PATH -cwd
+#$ -terse -j y -o '$QROOT.qlog'
+#$ ${QSUBFLAGS}
+### Grid Directives
+
+# Make absolutely sure we are in the working directory
+cd '$QDIR'              
+
+echo
+echo "==== Job Environment ===="
+echo "Date     = "\`date\`
+echo "Hostname = "\`hostname\`
+echo "Username = "\`whoami\`
+echo "PID      = \$\$"
+echo "PWD      = \$PWD"
+echo "PATH     = \$PATH"
+if [ "\$PBS_O_HOST" != "" ]; then
+   echo
+   echo "PBS_O_HOST    = \$PBS_O_HOST"
+   echo "PBS_O_QUEUE   = \$PBS_O_QUEUE"
+   echo "PBS_O_WORKDIR = \$PBS_O_WORKDIR"
+   echo "PBS_JOBID     = \$PBS_JOBID"
+   echo "PBS_JOBNAME   = \$PBS_JOBNAME"
+   echo "PBS_QUEUE     = \$PBS_QUEUE"
+fi
+if [ "\$SGE_ROOT" != "" ]; then
+   echo
+   echo "SGE_ROOT      = \$SGE_ROOT"
+   echo "SGE_O_HOST    = \$SGE_O_HOST"
+   echo "SGE_O_WORKDIR = \$SGE_O_WORKDIR"
+   echo "JOB_ID        = \$JOB_ID"
+   echo "JOB_NAME      = \$JOB_NAME"
+   echo "QUEUE         = \$QUEUE"
+fi
+echo "==== Job Environment ===="
+echo ""
+
+# Simple logging function
+qlog () { echo "[\`date --rfc-3339=sec\`] \$USER ($QNAME) \$*" >> $QLOG; }
+
+#
+# Function to catch errors/signals and report them
+#
+function on_exit() {
+   local exited=\${1:-\$?}
+   trap - HUP INT QUIT TERM ERR EXIT
+   
+   # Signal or error?
+   if [ \$exited -gt 128 -a \$exited -lt 166 ]; then
+      exited=\$(( exited - 128 ))
+      status="ABORT(\$exited)"
+   elif [ \$exited -ne 0 ]; then
+      status="ERROR(\$exited)"
+   else
+      status=DONE
+   fi
+   qlog \$status $QROOT.qsub
+   
+   # Remove job file (if it exists)
+   rm -f $QROOT.qjob
+
+   # Depending on cluster either exit with special abort code or just status
+   [ ! -z "\$SGE_ROOT" ] && exit 100    # SGE exit w/special 100 "abort" code 
+   exit \$exited                        # PBS exit status
+}
+trap on_exit HUP INT QUIT TERM ERR EXIT
+
+qlog START $QROOT.qsub
+
+QSUB_SCRIPT
+
+   # Add commands to run
+   echo "# Commands" >> $QROOT.qsub
+   while read; do  echo "$REPLY" >> $QROOT.qsub; done
+
+   # Add any extra commands
+   echo ""                         >> $QROOT.qsub
+   echo "# Run any extra commands" >> $QROOT.qsub
+   echo "$QEXTRA"                  >> $QROOT.qsub
+   
+   echo "exit 0" >> $QROOT.qsub
+
+   rm -f $QROOT.qlog
+   qlog SUBMIT $QROOT.qsub
+   qsub $QROOT.qsub | tee $QROOT.qjob
+   [ $? -ne 0 ] && exit 1
+}
+
+
+# -- POD DOCUMENTATION ------------------------------------------------------
+#
+# Documentation for wrappers.  Uses pod with a shell trick.  Simply run
+# pod2man on this file to get the documentation.
+#
+: <<POD
+
+=head1 NAME
+
+qtandem  - Submit a Tandem MS/MS peptide search to a cluster
+
+qsomssa  - Submit a OMSSA MS/MS peptide search to a cluster 
+
+qinspect - Submit a InsPect MS/MS peptide search to a cluster
+
+=head1 SYNOPSIS            
+
+q<search program> [options] *.mzXML|*.mzML [-- [qsub options]]
+
+ Options:
+   -h, --help           display this help and exit
+   --params <file>      file containing MS search algorithm parameters 
+   --dest <dir>         directory as the destination for output files  
+   
+ [qsub options]         all options following '--' are passed to qsub
+ 
+=head1 DESCRIPTION
+
+Collection of wrapper programs which can be used to submit MS/MS peptide
+spectra searches to a OpenPBS or Sun Grid Engine cluster.
+
+=head1 OPTIONS
+
+=over 5
+
+=item B<-h, --help>
+
+Print a brief help message and exit.
+
+=item B<--params FILE>
+
+Specify the parameters file to use in the search.  If not provided then the
+program will default to a file named "I<search>.params" in the local current
+directory, where I<search> is the name of the search program to invoke.
+E.g. qtandem will defaults to "tandem.params".
+
+=item B<--dest DIR>
+
+Specifies the destination directory of the search results.  If the destination
+directory doesn't exist it will be created first then links to all input files
+will be placed in this destination along with all output files from the search.
+No care is taken to prevent overwriting any existing files.
+
+=back
+
+Finally any options following "--" will be used as additional parameters to
+the qsub command.  So for example to change the queue that the job is submitted
+to you could run the command "qtandem *.mzXML -- -q serial".
+
+=head1 PREREQUSITES
+
+First off you can only invoke these commands on the head node of the cluster
+(or any system that has permissions to submit jobs to the cluster). Secondly,
+these programs require the appropriate search programs be installed on the
+cluster and in an accessible location for all nodes in the cluster.  Lastly each
+program assumes that local working directory is the same on all nodes and that
+any additional databases or auxiliary files are also available on all nodes.
+
+=over 5
+
+=item I<Sun Grid Engine>
+
+In order to use SGE you need to ensure that your shell environment is setup
+correctly.  In most cases your system administrator should already have provided
+local system settings in your login files.  You can check to see if your
+settings are correct by examining your environment for any variables beginning
+with the name "SGE_".
+
+=item I<PBS> 
+
+Nothing special.  Just need to have the qsub command in your path. 
+
+
+=item I<InsPect>
+
+Uses the InsPect program developed at the University of California, San Diego.
+You'll have to install this program locally on your cluster and ensure that
+the program is in your path. The location of the resource files  used by 
+I<inspect> will be inferred by the location of the program itself as this
+is the typical setup for a installation.
+
+=back
+
+=head1 OUTPUT
+
+All of these programs will print a job identifier for each search job submitted.
+See either OpenPBS qsub or Sun Grid Engine's qsub command manual for more 
+information about the format of the job id and how to use them.
+
+Each search program's output (both STDOUT and STDERR) will be captured in a 
+job specific "<input file>.<search>.qlog" file.  This file is written by the
+grid software when the job finishes.
+
+=head1 EXAMPLE 
+
+To run a search on the cluster you'll need to provide two things, a search 
+parameters file and one or more mzXML formatted files containing mass 
+spectroscopy data. 
+
+So for example:
+
+=over 5
+
+S<qtandem --params tandem.params foo.mzXML>
+
+=back
+
+This will submit a tandem search job to the cluster for foo.mzXML in the
+current directory.  The results will be written to the file foo.pep.xml and
+any output for the job will be captured in  foo.tandem.qlog.
+
+=head1 ADDITIONAL RESOURCES
+
+=item L<http://gridengine.sunsource.net/>
+
+Sun Grid Engine project.
+
+=item L<http://www.clusterresources.com/products/torque-resource-manager.php>
+
+TORQUE resource manager (OpenPBS)
+
+=item L<http://www.thegpm.org/tandem/index.html>
+
+X!Tandem  open source software.
+
+=item L<http://pubchem.ncbi.nlm.nih.gov/omssa/>
+
+The Open Mass Spectrometry Search Algorithm [OMSSA]
+
+=item L<http://proteomics.ucsd.edu/>
+
+InsPecT: A Proteomics Search Toolkit
+
+=head1 AUTHORS
+
+Joe Slagel E<lt>js...@sy...<gt>
+
+=cut
+
+POD

Modified: trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qinspect
===================================================================
--- trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qinspect	2010-08-24 22:43:24 UTC (rev 5119)
+++ trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qinspect	2010-08-25 00:08:14 UTC (rev 5120)
@@ -3,7 +3,7 @@
 # Program: TPP HPC Tools
 # Author:  Joe Slagel
 #
-# Copyright (C) 2009 by Joseph Slagel
+# Copyright (C) 2009-2010 by Joseph Slagel
 # 
 # This library is free software; you can redistribute it and/or             
 # modify it under the terms of the GNU Lesser General Public                
@@ -27,28 +27,18 @@
 # $Id: $
 #
 
-. qgrid_functions                               # Load grid functions
+# -- Local settings -----------------------------------------------------------
 
-# -- Local settings ------------------------------------------------------------
-
-   # What command tool are we
-   QJOBNAME=${QJOBNAME-TPP-inspect}
+   # Default queue to submit jobs to
+   QQUEUE=${QQUEUE:-xtandem}
    
-   # Submission log file
-   QLOG=${QLOG-/dev/null}
-   
-   # Default grid queue to submit searches to
-   QQUEUE=${QQUEUE-xtandem}
-   
-   # Additional commands to run when exiting
-   QEND=${QEND-}
-
-   # Parameters file
-   PARAMS=inspect.params
-   
+   # Default name for jobs in queue
+   QNAME=${QNAME:-TPP-inspect}
+    
    # Programs
-   QSUB=${QSUB-`which qsub 2>/dev/null`}
    INSPECT=${INSPECT-`which inspect 2>/dev/null`}
+   INSPECT2PEP=${INSPECT2PEP-`which InspectToPepXML.py 2>/dev/null`}
+   INSPECT2PEP=${INSPECT2PEP-`$(dirname $INSPECT)/InspectToPepXML.py`}
    
 # -----------------------------------------------------------------------------
 
@@ -61,41 +51,42 @@
 }
 
 #
-# Write search commands to qsub script
+# List of commands to run in the qsub script.  Commands are echo'ed out and
+# are intended to be read in by the qsubmit() function.
 #
-function write_qsub {
-   cat <<EOF > $1
-$(qsub_header $root.inspect.log)
-
-log "START $input"
-# inferred resource location from program name
+function cmds() {
+   cat <<EOF
 DIR=`dirname $INSPECT`          
 $INSPECT -r \$DIR -i $root.inspect.params -o $root.inspect -e $root.inspect.err
 if [ -f $root.inspect.err ]; then
    cat $root.inspect.err
    rm $root.inspect.err
 fi
-python \$DIR/InspectToPepXML.py -i $root.inspect -o $root.pep.xml
+python $INSPECT2PEP -i $root.inspect -o $root.pep.xml
 
-${QEND}
-log "DONE $input"
-
 EOF
 }
 
+# -- Main ----------------------------------------------------------------------
+
+source qgrid_functions                          # Load "q" grid functions
+
 # Check for necessary programs
-[ ! -e ${QSUB} ]         && usage "Error no qsub in your path"
-[ ! -e ${INSPECT} ]      && usage "Error no inspect in your path"
-search_input $*
+[ -x "${INSPECT}" ]     || die "Error no inspect in your path"
+[ -f "${INSPECT2PEP}" ] || die "Error no InspectToPepXML.py in your path"
 
+search_options inspect.params "*.mzML|*.mzXML" $*
+
 # Submit jobs
 for input in $INPUTS; do
    dir=$(dirname $(rel2abs "$input"))           # base directory of input
-   root=$(basename "${input%.mz*ML}")           # filename w/o extension
+   root=$(basename "${input%.*}")               # filename w/o extension
    
-   copy_params $PARAMS "$dir/$root.inspect.params" "$input"
-   write_qsub "$root.inspect.qsub"
+   rm -f $root.pep.xml                          # remove previous results
+   rm -f $root.inspect
    
-   log "SUBMIT $input"
-   $QSUB "$dir/$root.inspect.qsub"
+   copy_params $PARAMS "$dir/$root.inspect.params" "$input" 
+   
+   # Submit job
+   cmds | submit "$dir/$root.inspect" "$dir"
 done

Modified: trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qomssa
===================================================================
--- trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qomssa	2010-08-24 22:43:24 UTC (rev 5119)
+++ trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qomssa	2010-08-25 00:08:14 UTC (rev 5120)
@@ -3,7 +3,7 @@
 # Program: TPP HPC Tools
 # Author:  Joe Slagel
 #
-# Copyright (C) 2009 by Joseph Slagel
+# Copyright (C) 2009-2010 by Joseph Slagel
 # 
 # This library is free software; you can redistribute it and/or             
 # modify it under the terms of the GNU Lesser General Public                
@@ -27,78 +27,61 @@
 # $Id: $
 #
 
-. qgrid_functions                               # Load grid functions
-
-
 # -- Local settings ------------------------------------------------------------
 
-   # What command tool are we
-   QJOBNAME=${QJOBNAME-TPP-omssa}
+   # Default queue to submit jobs to
+   QQUEUE=${QQUEUE:-xtandem}
    
-   # Submission log file
-   QLOG=${QLOG-/dev/null}
-   
-   # Default grid queue to submit searches to
-   QQUEUE=${QQUEUE-xtandem}
-   
-   # Additional commands to run when exiting
-   QEND=${QEND-}
-   
-   # Parameters file
-   PARAMS=omssa.params
-   
+   # Default name for jobs in queue
+   QNAME=${QNAME:-TPP-omssa}
+
    # Default OMSSA flags
-   OMSSA_FLAGS=
+   OMSSA_FLAGS=${OMSSA_FLAGS-}
    
    # Programs
-   QSUB=${QSUB-`which qsub 2>/dev/null`}
    OMSSACL=${OMSSACL-`which omssacl 2>/dev/null`}
    MZXML2SEARCH=${MZXML2SEARCH-`which MzXML2Search 2>/dev/null`}
    
+# -----------------------------------------------------------------------------
    
-# Functions --------------------------------------------------------------------
-
 #
 # Read in omssa parameters file
 #
-function readparams () {
-   OMSSA_FLAGS+=`perl -ne 's/\s*\#.*$//; chop; push @a, $_ if $_; END { print join " ", @a }' $PARAMS`
+function read_params () {
+   OMSSA_FLAGS+=`perl -pe 's/\s*\#.*$//; s/\r//g; s/^\n//' $PARAMS`
+   OMSSA_FLAGS=`echo $OMSSA_FLAGS`    # remove newlines
 }
 
 #
-# Append search commands to qsub script
+# List of commands to run in the qsub script.  Commands are echo'ed out and
+# are intended to be read in by the qsubmit() function.
 #
-function write_qsub {
-   cat <<EOF > $1
-$(qsub_header $root.omssa.log)
-
-log "START $input"
+function cmds() {
+   cat <<EOF
 $MZXML2SEARCH -mgf $input
 $OMSSACL -fm $root.mgf -op $root.pep.xml $OMSSA_FLAGS
 
-${QEND}
-log "DONE $input"
-
 EOF
 }
 
-# Main ------------------------------------------------------------------------
+# -- Main ----------------------------------------------------------------------
 
+source qgrid_functions                          # Load "Q" grid functions
+
 # Check for necessary programs
-[ ! -e ${QSUB} ]         && usage "Error no qsub in your path"
-[ ! -e ${OMSSACL} ]      && usage "Error no omssacl in your path"
-[ ! -e ${MZXML2SEARCH} ] && usage "Error no MzXML2Search in your path"
-search_input $*
+[ -x ${OMSSACL} ]      || die "Error no omssacl in your path"
+[ -x ${MZXML2SEARCH} ] || die "Error no MzXML2Search in your path"
 
-readparams $PARAMS
+search_options omssa.params "*.mzML|*.mzXML" $*
+
+read_params omssa.params
    
 # Submit jobs
 for input in $INPUTS; do
    dir=$(dirname $(rel2abs "$input"))           # base directory of input
-   root=$(basename "${input%.mz*ML}")           # filename w/o extension
+   root=$(basename "${input%.*}")               # filename w/o extension
    
-   write_qsub "$root.omssa.qsub"
-   
-   log "SUBMIT $input"
-   $QSUB $dir/$root.omssa.qsub
+   rm -f $root.pep.xml                          # remove previous results
+    
+   cmds | submit "$dir/$root.omssa" "$dir"      # submit job
 done

Modified: trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qspectrast
===================================================================
--- trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qspectrast	2010-08-24 22:43:24 UTC (rev 5119)
+++ trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qspectrast	2010-08-25 00:08:14 UTC (rev 5120)
@@ -27,29 +27,20 @@
 # $Id: $
 #
 
-. qgrid_functions                               # Load grid functions
 
-
 # -- Local settings ------------------------------------------------------------
 
-   # What command tool are we
-   QJOBNAME=${QJOBNAME-TPP-spectrast}
-   
    # Default queue to submit jobs to
-   QQUEUE=${QQUEUE-xtandem}
+   QQUEUE=${QQUEUE:-xtandem}
    
-   # Additional commands to run at the end
-   QEND=${QEND-}
-   
-   # Params file
-   PARAMS=spectrast.params
+   # Default name for jobs in queue
+   QNAME=${QNAME:-TPP-spectrast}
 
-   # Default remote convert flags
-   SPECTRASTFLAGS=${SPECTRASTFLAGS-}
-   
    # Programs
-   QSUB=${QSUB-`which qsub 2>/dev/null`}
    SPECTRAST=`which spectrast 2>/dev/null`
+   
+   # Default spectrast flags
+   SPECTRAST_FLAGS=${SPECTRAST_FLAGS-}
 
 # -----------------------------------------------------------------------------
 
@@ -66,31 +57,28 @@
    echo "   -h, --help          help"
    echo "   -m, --manual        manual"
    echo
-   echo "[spectrast opts]	See spectrast -h for options"
-   echo "[qsub options]         All flags following '--' are passed to qsub"
+   echo "   [spectrast opts]	see spectrast -h for options"
+   echo "   [qsub options]      all flags following '--' are passed to qsub"
    exit 1
 }
 
 #
-# Write search commands to qsub script
+# List of commands to run in the qsub script.  Commands are echo'ed out and
+# are intended to be read in by the qsubmit() function.
 #
-function write_qsub {
-   f=$1; shift
-   cat <<EOF > $f
-$(qsub_header $root.spectrast.log)
+function cmds {
+   cat <<EOF
+${SPECTRAST} ${SPECTRAST_FLAGS} $*
+EOF
+}
 
-log "SPECTRAST started $input"
 
-${SPECTRAST} ${SPECTRASTFLAGS} $*
-${QEND}
-log "SPECTRAST done $input"
+# -- Main ----------------------------------------------------------------------
 
-EOF
-}
+source qgrid_functions                          # Load "Q" grid functions
 
 # Check for necessary programs
-[ ! -x ${QSUB} ]      && usage "Error no qsub in your path"
-[ ! -x ${SPECTRAST} ] && usage "Error no spectrast in your path"
+[ -x ${SPECTRAST} ] || die "Error no spectrast in your path"
 
 # Check options
 INPUTS=
@@ -110,21 +98,21 @@
       ;;
    -sL*|-sF*)
       MODE=search
-      SPECTRASTFLAGS="$SPECTRASTFLAGS $1 "
+      SPECTRAST_FLAGS+=" $1 "
       shift
       ;;
    -c*)
       MODE=create
-      SPECTRASTFLAGS="$SPECTRASTFLAGS $1 "
+      SPECTRAST_FLAGS+=" $1 "
       shift
       ;;
    -*)
-      SPECTRASTFLAGS="$SPECTRASTFLAGS $1 "
+      SPECTRAST_FLAGS+=" $1 "
       shift
       ;;
    *)
       [ -e "$1" ] || usage "Error $1 file not found";
-      INPUTS="$INPUTS $1"
+      INPUTS+=" $1"
       shift
       ;;
    esac
@@ -133,17 +121,14 @@
 # Spectrast requires a mode.  If one doesn't exist assume its search
 # and that the parameters file is the default
 if [ "$MODE" = "" ]; then
-      SPECTRASTFLAGS="$SPECTRASTFLAGS -sFspectrast.params"
+      SPECTRAST_FLAGS+=" -sFspectrast.params"
       MODE=search
 fi
 
 # Submit create job
 if [ "$MODE" = "create" ]; then
    dir=$(dirname $(rel2abs "$input"))           # base directory of input
-   log "SUBMIT $input"
-   write_qsub "spectrast.qsub" $INPUTS
-   $QSUB spectrast.qsub
-   [ $? -ne 0 ] && exit 1
+   cmds | submit "$dir/$root.spectrast" "$dir"  # submit job
    exit 0
 fi
 
@@ -152,12 +137,9 @@
    dir=$(dirname $(rel2abs "$input"))           # base directory of input
    root=$(basename "${input%.*}")               # filename w/o extension
    
-   write_qsub "$root.spectrast.qsub" "$input"
-   
-   log "SUBMIT $input"
-   $QSUB "$dir/$root.spectrast.qsub"
-   [ $? -ne 0 ] && exit 1
+   cmds | submit "$dir/$root.spectrast" "$dir"  # submit job
 done
+
 exit 0
 
 
@@ -178,6 +160,7 @@
 
  Options:
    -h, --help           display this help and exit
+   -m, --manual         display the manual and exit
    
 =head1 DESCRIPTION
 

Modified: trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qtandem
===================================================================
--- trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qtandem	2010-08-24 22:43:24 UTC (rev 5119)
+++ trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qtandem	2010-08-25 00:08:14 UTC (rev 5120)
@@ -3,7 +3,7 @@
 # Program: TPP HPC Tools
 # Author:  Joe Slagel
 #
-# Copyright (C) 2009 by Joseph Slagel
+# Copyright (C) 2009-2010 by Joseph Slagel
 # 
 # This library is free software; you can redistribute it and/or             
 # modify it under the terms of the GNU Lesser General Public                
@@ -27,34 +27,20 @@
 # $Id: $
 #
 
-. qgrid_functions                       # Load grid functions
-
-
 # -- Local settings -----------------------------------------------------------
 
-   # What command tool are we
-   QJOBNAME=${QJOBNAME:-TPP-tandem}
-    
-   # Submission log file
-   QLOG=${QLOG-/dev/null}
-   
    # Default queue to submit jobs to
-   QQUEUE=${QQUEUE-xtandem}
+   QQUEUE=${QQUEUE:-xtandem}
    
-   # Additional commands to run when exiting
-   #QEND=${QEND-}
-   
-   # Parameters file
-   PARAMS=tandem.params
-   
+   # Default name for jobs in queue
+   QNAME=${QNAME:-TPP-tandem}
+    
    # Programs
-   QSUB=${QSUB-`which qsub 2>/dev/null`}
    TANDEM=${TANDEM-`which tandem.exe 2>/dev/null`}
    TANDEM2XML=${TANDEM2XML-`which Tandem2XML 2>/dev/null`}
    
+# -----------------------------------------------------------------------------
    
-# -- Functions ----------------------------------------------------------------
-
 #
 # Copy tandem.params file 
 #
@@ -62,7 +48,7 @@
    src=$1
    dst=$2
    
-   # use brute force copy the parameter file and replace tags
+   # Use brute force to copy the parameter file and replace tags
    SPT_REGEX='(<note type="input"\s+label="spectrum,\s*path">).*?(</note>)'
    OUT_REGEX='(<note type="input"\s+label="output,\s*path">).*?(</note>)'
    SEQ_REGEX='(<note type="input"\s+label="output,\s*sequence path">).*?(</note>)'
@@ -75,42 +61,40 @@
 }
 
 #
-# Write out a qsub script
+# List of commands to run in the qsub script.  Commands are echo'ed out and
+# are intended to be read in by the qsubmit() function.
 #
-function write_qsub {
-   cat <<EOF > $1
-$(qsub_header $root.tandem.log)
-
-log "START $input"
+function cmds() {
+   cat <<EOF
 $TANDEM $root.tandem.params 
 $TANDEM2XML $root.tandem $root.pep.xml 
-
-${QEND}
-log "DONE $input"
 EOF
 }
 
 
 # -- Main ----------------------------------------------------------------------
 
+source qgrid_functions                          # Load "q" grid functions
+
 # Check for necessary programs
-[ ! -x ${QSUB} ]        && usage "Error no qsub in your path"
-[ ! -x ${TANDEM} ]      && usage "Error no tandem.exe in your path"
-[ ! -x ${TANDEM2XML} ]  && usage "Error no Tandem2XML in your path"
-search_input $*
+[ -x ${TANDEM} ]     || die "Error no tandem.exe in your path"
+[ -x ${TANDEM2XML} ] || die "Error no Tandem2XML in your path"
 
+# Assigns PARAMS INPUTS
+search_options tandem.params '*.mzML|*.mzXML|*.mgf' $*
+
 # Submit jobs
 for input in $INPUTS; do
    input=$(rel2abs "$input")                    # use abs path
    dir=$(dirname "$input") 		        # base directory of input
-   root=$(basename "${input%.mz*ML*}")          # filename w/o extension
+   root=$(basename "${input%.*}")               # filename w/o extension
    
-   rm -f $root.tandem.log                       # remove previous results
-   rm -f $root.pep.xml
+   rm -f $root.pep.xml                          # remove previous results
+   rm -f $root.output_sequences
+   rm -f $root.tandem
    
    copy_params $PARAMS $root.tandem.params
-   write_qsub $root.tandem.qsub
    
-   log "SUBMIT $input"
-   $QSUB $root.tandem.qsub
+   # Submit job
+   cmds | submit "$dir/$root.tandem" "$dir"
 done

Modified: trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qtpp
===================================================================
--- trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qtpp	2010-08-24 22:43:24 UTC (rev 5119)
+++ trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/qtpp	2010-08-25 00:08:14 UTC (rev 5120)
@@ -3,7 +3,7 @@
 # Program: TPP HPC Tools
 # Author:  Joe Slagel
 #
-# Copyright (C) 2009 by Joseph Slagel
+# Copyright (C) 2009-2010 by Joseph Slagel
 # 
 # This library is free software; you can redistribute it and/or             
 # modify it under the terms of the GNU Lesser General Public                
@@ -27,19 +27,16 @@
 # $Id: $
 #
 
-. qgrid_functions                       # include grid functions
-
-
 # -- Local settings ------------------------------------------------------------
 
-   QCMD=analyze
+   # Default queue to submit jobs to
+   QQUEUE=${QQUEUE:-serial}
    
-   # QMAIL is a comma delimited list of users to email when results are done
-   QMAIL=${QMAIL-}
+   # Default name for jobs in queue
+   QNAME=${QNAME:-TPP}
+    
+   # Programs
    
-   # Command to run when the job is finished
-   QEND=${QEND-}
-   
 # -- Functions -----------------------------------------------------------------
 
 # 
@@ -49,66 +46,24 @@
 function usage() {
    [ "$1" ] && echo "$(basename $0): $*" 1>&2
    echo "Usage:"
-   echo "$(basename $0) [qsub options] *.pep.xml file(s)"
+   echo "$(basename $0) [options] *.pep.xml file(s) [-- [qsub options]]"
    echo
    echo "Options:"
    echo "   -h, --help          help"
-   echo "   --depends <jobids>  list of job ids to depend on"
+   echo "   -m, --manual        manual"
    echo
-   echo "[qsub options]         All flags following '--' are passed to qsub"
+   echo "   [qsub options]      all flags following '--' are passed to qsub"
    echo
    exit 1
 }
 
-
-# -- Main ----------------------------------------------------------------------
-
 #
-# Check program parameters
+# List of commands to run in the qsub script.  Commands are echo'ed out and
+# are intended to be read in by the qsubmit() function.
 #
-PEPXMLS=
-QSUBFLAGS=
-QSUBDEPENDS=
-while [ "$1" ]; do
-   case "$1" in 
-      --)
-         shift
-         QSUBFLAGS=$*
-         break
-         ;;
-      -h|--help)
-         usage
-         ;;
-      --depends)
-         if [ -z "$SGE_ROOT" ]; then
-            QSUBDEPENDS="-W depend=\"afterok:$2\""
-         else
-            QSUBDEPENDS=`echo "-hold_jid $2" | perl -pe 's/:/,/gm;'`
-         fi
-         shift 2
-         ;;
-      *.pep.xml)
-         PEPXMLS+="$1 "
-         shift
-         ;;
-      *)
-	 echo "Invalid argument: $1"
-	 usage
-         ;;
-   esac
-done
-[ ! -z "$PEPXMLS" ] || usage "Error missing one or more pep.xml files"
-
-#
-# Make qsub script.  Be very careful about escaping variables in here
-#
-cat <<EOF > TPP.qsub
-$(qsub_header TPP.log)
-
-log "START $PWD"
-
-echo "Starting \`date\`"
-
+function cmds() {
+   cat <<EOF
+   
 # Read in additional command parameters
 PEPXMLS='$PEPXMLS'
 [ -f xinteract.params ] && XPARAMS=\`cat xinteract.params\`
@@ -160,18 +115,56 @@
 calctppstat.pl -i interact-ipro.pep.xml --FDRthresh --full --write \$CALCPARAMS
 echo 
 
-echo "Finished \`date\`"
+EOF
+}
 
-log "DONE $PWD"
+# -- Main ----------------------------------------------------------------------
 
-EOF
+source qgrid_functions                          # Load "q" grid functions
 
 #
-# Submit the job
+# Check program parameters
 #
-log "SUBMIT $PWD"
-$QSUB TPP.qsub ${QSUBDEPENDS}
+PEPXMLS=
+QSUBFLAGS=
+QSUBDEPENDS=
+while [ "$1" ]; do
+   case "$1" in 
+      --)
+         shift
+         QSUBFLAGS=$*
+         break
+         ;;
+      -h|--help)
+         usage
+         ;;
+      -m|--manual)
+         pod2man -n 'HPC TOOLS' -c 'Trans-Proteomic Pipeline (TPP)' $0 | nroff -man | less
+         exit
+         ;;
+      --depends)
+         if [ -z "$SGE_ROOT" ]; then
+            QSUBDEPENDS="-W depend=\"afterok:$2\""
+         else
+            QSUBDEPENDS=`echo "-hold_jid $2" | perl -pe 's/:/,/gm;'`
+         fi
+         shift 2
+         ;;
+      *.pep.xml)
+         PEPXMLS+="$1 "
+         shift
+         ;;
+      *)
+	 echo "Invalid argument: $1"
+	 usage
+         ;;
+   esac
+done
+[ ! -z "$PEPXMLS" ] || usage "Error missing one or more pep.xml files"
 
+# Submit job
+cmds | submit "$PWD/TPP" "$PWD"
+
 exit 0
 
 
@@ -257,6 +250,8 @@
 
 =item qinspect - submit one or more inspect search jobs
 
+=item qmyrimatch - submit one or more myrimatch search jobs
+
 =item qsub, OpenPBS L<http://www.openpbs.org>, 
 
 =item Sun Grid Engine L<http://gridengine.sunsource.net>

Modified: trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/runtpp
===================================================================
--- trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/runtpp	2010-08-24 22:43:24 UTC (rev 5119)
+++ trunk/trans_proteomic_pipeline/extern/hpctools/grid/bin/runtpp	2010-08-25 00:08:14 UTC (rev 5120)
@@ -3,7 +3,7 @@
 # Program: TPP HPC Tools
 # Author:  Joe Slagel
 #
-# Copyright (C) 2009 by Joseph Slagel
+# Copyright (C) 2009-2010 by Joseph Slagel
 # 
 # This library is free software; you can redistribute it and/or             
 # modify it under the terms of the GNU Lesser General Public                
@@ -31,62 +31,63 @@
 # Run MS/MS searches followed by a TPP analysis
 #
 
-. qgrid_functions                       # include grid functions
+# -- Main ----------------------------------------------------------------------
 
+source qgrid_functions                          # Load "q" grid functions
 
-# -- Local settings ------------------------------------------------------------
-
-  # QMAIL is a comma delimited list of users to email when results are done
-  #QMAIL=
-
-
-# -- Main ----------------------------------------------------------------------
-
-# Pick which search program to use
+# Pick a default search program to use
 for PARAMS in {tandem,omssa,inspect,sequest}.params; do
    [ -f "$PARAMS" ] && break
    PARAMS=
 done
 
-# Check input and submit search
-check_input $*
-rm -f .jobs
+# Assigns PARAMS INPUTS and submit search jobs
+search_options $PARAMS '*.mzML|*.mzXML' $*
 case "${PARAMS%.params}" in
    tandem)
       echo "Queuing tandem searches:"
-      qtandem $INPUTS $QSUB_FLAGS | tee .jobs
+      JOBS=`qtandem $INPUTS $QSUB_FLAGS`
       ;;
    omssa)
       echo "Queuing omssa searches:"
-      qomssa $INPUTS $QSUB_FLAGS | tee .jobs
+      JOBS=`qomssa $INPUTS $QSUB_FLAGS`
       ;;
    inspect)
       echo "Qeuing inspect searches:"
-      qinspect $INPUTS $QSUB_FLAGS | tee .jobs
+      JOBS=`qinspect $INPUTS $QSUB_FLAGS`
       ;;
    sequest)
       echo "Submitting sequest searches:"
-      qsequest $INPUTS $QSUB_FLAGS | tee .jobs
+      JOBS=`qsequest $INPUTS $QSUB_FLAGS`
       ;;
    *)
       usage "Unrecognized parameter file ${PARAMS}"
       ;;
 esac
-echo
+echo "$JOBS"
+echo 
 
 # Send email on tpp finishing
-QEND='email_notify "TPP $QSTATUS in $PWD" "'$QMAIL\"
+#QEND='email_notify "TPP $QSTATUS in $PWD" "'$QMAIL\"
 
+# Add search job dependencies to QSUB flags list
+if [ -z "$SGE_ROOT" ]; then
+   JOBS=`echo -e $JOBS | perl -pe 's/ /:/gm;'`
+   QSUB_FLAGS="$QSUB_FLAGS -W depend=\"afterok:$JOBS\""
+else
+   JOBS=`echo -e $JOBS | perl -pe 's/ /,/gm;'`
+   QSUB_FLAGS="$QSUB_FLAGS -hold_jid $JOBS"
+fi
+
+# Queue up tpp
 echo "Queuing TPP analysis of search results:"
-JOBS=`cat .jobs`
-JOBS=`echo -e $JOBS | perl -pe 's/ /:/gm;'`
 PEPXML=${INPUTS//.mzXML/.pep.xml}
 PEPXML=${PEPXML//.mzML/.pep.xml}
-qtpp $PEPXML --depends $JOBS $QSUB_FLAGS
-rm -f .jobs
+qtpp $PEPXML -- $QSUB_FLAGS
 
 exit 0
 
+
 # -- POD DOCUMENTATION ------------------------------------------------------
 #
 # Documentation.  Uses pod with a shell trick.  Simply run pod2man on this file
@@ -100,7 +101,7 @@
 
 =head1 SYNOPSIS            
 
-runtpp [options] *.mzXML
+runtpp [options] *.mzXML|*.mzML
 
  Options:
    -h, --help          print this help
@@ -146,6 +147,8 @@
 
 =item qinspect - submit one or more inspect search jobs
 
+=item qmyrimatch - submit one or more myrimatch search jobs
+
 =item qtpp - submit a TPP analysis 
 
 =item qsub, OpenPBS L<http://www.openpbs.org>, 


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.