#!/bin/ksh
#
# LSF batch script to run an MPI application
#
#BSUB -P P48503002             # project code
#BSUB -W 00:10                 # wall-clock time (hrs:mins)
#BSUB -n 16                    # number of tasks in job         
#BSUB -R "span[ptile=16]"      # run 16 MPI tasks per node
#BSUB -J gsi                   # job name
#BSUB -o gsi.%J.out          # output file name in which %J is replaced by the job ID
#BSUB -e gsi.%J.err          # error file name in which %J is replaced by the job ID
## #BSUB -q regular               # queue
#BSUB -q small               # queue

#
# GSIPROC = processor number used for GSI analysis
#------------------------------------------------
  GSIPROC=16
  ARCH='LINUX_LSF' 
# Supported configurations:
            # IBM_LSF,
            # LINUX, LINUX_LSF, LINUX_PBS,
            # DARWIN_PGI
#
#####################################################
# case set up (users should change this part)
#####################################################
#
# ANAL_TIME= analysis time  (YYYYMMDDHH)
# WORK_ROOT= working directory, where GSI runs
# PREPBURF = path of PreBUFR conventional obs
# BK_DIR   = path of first guess generation dirctory
# OBS_ROOT = path of observations files
# FIX_ROOT = path of fix files
# GSI_EXE  = path and name of the gsi executable 
# ROOTDIR  = RTMA root dirctory: util/RTMA
  ANAL_TIME=2012052811
  WORK_ROOT=/glade/p/work/mhu/gsi/rtma/rtma/RTMA/rtmaprd
  BK_DIR=/glade/p/work/mhu/gsi/rtma/rtma/RTMA/rtmagus
  OBS_ROOT=/glade/p/work/mhu/gsi/rtma/RTMA_test/rtmaprd
  PREPBUFR=${OBS_ROOT}/newgblav.20120528.rap.t11z.prepbufr
  FIX_ROOT=/glade/p/work/mhu/gsi/code/fix_releaseV32
  GSI_EXE=/glade/p/work/mhu/gsi/code/release_V3.2_intel13.0/run/gsi.exe
  ROOTDIR=/glade/p/work/mhu/gsi/rtma/rtma/RTMA

#------------------------------------------------
# if_clean = clean  : delete temperal files in working directory (default)
#            no     : leave running directory as is (this is for debug only)
  if_clean=clean
#
. ${ROOTDIR}/rtma_envir.sh
#
#####################################################
# Users should NOT change script after this point
#####################################################
#

case $ARCH in
   'IBM_LSF')
      ###### IBM LSF (Load Sharing Facility)
      RUN_COMMAND="mpirun.lsf " ;;

   'LINUX')
      if [ $GSIPROC = 1 ]; then
         #### Linux workstation - single processor
         RUN_COMMAND=""
      else
         ###### Linux workstation -  mpi run
        RUN_COMMAND="mpirun -np ${GSIPROC} -machinefile ~/mach "
      fi ;;

   'LINUX_LSF')
      ###### LINUX LSF (Load Sharing Facility)
      RUN_COMMAND="mpirun.lsf " ;;

   'LINUX_PBS')
      #### Linux cluster PBS (Portable Batch System)
      RUN_COMMAND="mpirun -np ${GSIPROC} " ;;

   'DARWIN_PGI')
      ### Mac - mpi run
      if [ $GSIPROC = 1 ]; then
         #### Mac workstation - single processor
         RUN_COMMAND=""
      else
         ###### Mac workstation -  mpi run
         RUN_COMMAND="mpirun -np ${GSIPROC} -machinefile ~/mach "
      fi ;;

   * )
     print "error: $ARCH is not a supported platform configuration."
     exit 1 ;;
esac


##################################################################################
# Check GSI needed environment variables are defined and exist
#
 
# Make sure ANAL_TIME is defined and in the correct format
if [ ! "${ANAL_TIME}" ]; then
  echo "ERROR: \$ANAL_TIME is not defined!"
  exit 1
fi

# Make sure WORK_ROOT is defined and exists
if [ ! "${WORK_ROOT}" ]; then
  echo "ERROR: \$WORK_ROOT is not defined!"
  exit 1
fi

# Make sure the background file exists
if [ ! "${BK_DIR}" ]; then
  echo "ERROR: ${BK_DIR} does not exist!"
  exit 1
fi

# Make sure OBS_ROOT is defined and exists
if [ ! "${OBS_ROOT}" ]; then
  echo "ERROR: \$OBS_ROOT is not defined!"
  exit 1
fi
if [ ! -d "${OBS_ROOT}" ]; then
  echo "ERROR: OBS_ROOT directory '${OBS_ROOT}' does not exist!"
  exit 1
fi

# Set the path to the GSI static files
if [ ! "${FIX_ROOT}" ]; then
  echo "ERROR: \$FIX_ROOT is not defined!"
  exit 1
fi
if [ ! -d "${FIX_ROOT}" ]; then
  echo "ERROR: fix directory '${FIX_ROOT}' does not exist!"
  exit 1
fi

# Make sure the GSI executable exists
if [ ! -x "${GSI_EXE}" ]; then
  echo "ERROR: ${GSI_EXE} does not exist!"
  exit 1
fi

# Check to make sure the number of processors for running GSI was specified
if [ -z "${GSIPROC}" ]; then
  echo "ERROR: The variable $GSIPROC must be set to contain the number of processors to run GSI"
  exit 1
fi

#
##################################################################################
# Create the ram work directory and cd into it

workdir=${WORK_ROOT}
echo " Create working directory:" ${workdir}

if [ -d "${workdir}" ]; then
  rm -rf ${workdir}
fi
mkdir -p ${workdir}
cd ${workdir}

#
##################################################################################

echo " Copy GSI executable, background file, and link observation bufr to working directory"

# Save a copy of the GSI executable in the workdir
cp ${GSI_EXE} gsi.exe

# Bring over background field (it's modified by GSI so we can't link to it)
cp ${BK_DIR}/twodvar_input_bi ./wrf_inout

# Link to the prepbufr data
ln -s ${PREPBUFR} ./prepbufr

## ln -s ${OBS_ROOT}/rtma.satwnd.tm00.bufr_d satwnd
 
# copy grid information from first guess generating step
## cp ${BK_DIR}/gridname_input .
##################################################################################

echo " Copy fixed files and link CRTM coefficient files to working directory"

# Set fixed files
#   berror   = forecast model background error statistics
#   pcpinfo  = text file with information about assimilation of prepcipitation rates
#   ozinfo   = text file with information about assimilation of ozone data
#   errtable = text file with obs error for conventional data (regional only)
#   convinfo = text file with information about assimilation of conventional data
#   bufrtable= text file ONLY needed for single obs test (oneobstest=.true.)

BERROR=${FIX_ROOT}/new_rtma_regional_nmm_berror.f77.gcv
OBERROR=${FIX_ROOT}/new_rtma_nam_errtable.r3dv
ANAVINFO=${FIX_ROOT}/anavinfo_rtma_gust_vis_7vars
CONVINFO=${FIX_ROOT}/new_rtma_regional_convinfo.txt

mesonetuselist=${FIX_ROOT}/new_rtma_mesonet_uselist.txt
mesonet_stnuselist=${FIX_ROOT}/nam_mesonet_stnuselist.txt
slmask=${ROOTDIR}/fix/rtma_${cgrid}_slmask.dat
terrain=${ROOTDIR}/fix/rtma_${cgrid}_terrain.dat

#  copy Fixed fields to working directory
 cp $ANAVINFO anavinfo
 cp $BERROR   berror_stats
 cp $CONVINFO convinfo
 cp $OBERROR  errtable

 cp $mesonetuselist ./mesonetuselist
 cp $mesonet_stnuselist ./mesonet_stnuselist
 cp $slmask ./rtma_slmask.dat
 cp $terrain ./rtma_terrain.dat

#
## reject lists for obs

if [ -e ${FIX_ROOT}/new_rtma_t_rejectlist ] ; then
   cp ${FIX_ROOT}/new_rtma_t_rejectlist t_rejectlist 
fi
if [ -e ${FIX_ROOT}/new_rtma_q_rejectlist ] ; then
   cp ${FIX_ROOT}/new_rtma_q_rejectlist q_rejectlist
fi
if [ -e ${FIX_ROOT}/new_rtma_p_rejectlist ] ; then
   cp ${FIX_ROOT}/new_rtma_p_rejectlist p_rejectlist
fi
if [ -e ${FIX_ROOT}/new_rtma_w_rejectlist ] ; then
   cp ${FIX_ROOT}/new_rtma_w_rejectlist w_rejectlist
fi


# Only need this file for single obs test
 bufrtable=${FIX_ROOT}/prepobs_prep.bufrtable
 cp $bufrtable ./prepobs_prep.bufrtable

 cp ${FIX_ROOT}/new_rtma_random_flips    ./random_flips
#
##################################################################################
# Set some parameters for use by the GSI executable and to build the namelist
echo " Build the namelist "

export JCAP=62
export LEVS=60
export JCAP_B=62
export DELTIM=${DELTIM:-$((3600/($JCAP/20)))}

# Build the GSI namelist on-the-fly
cat << EOF > gsiparm.anl
 &SETUP
   miter=2,niter(1)=50,niter(2)=50
   write_diag(1)=.true.,write_diag(2)=.true.,write_diag(3)=.true.,
   gencode=78,qoption=1,tsensible=$tsensible,lgschmidt=${lgschmidt},
   factqmin=1.0,factqmax=1.0,deltim=$DELTIM,
   ndat=8,iguess=-1,
   oneobtest=.false.,retrieval=.false.,
   diag_rad=.false.,diag_pcp=.false.,diag_ozone=.false.,
   nhr_assimilation=1,min_offset=30,
   perturb_obs=.false.,perturb_fact=0.1,
   use_prepb_satwnd=.false.
 /
 &GRIDOPTS
   JCAP=$JCAP,NLAT=$NLAT,NLON=$LONA,nsig=$LEVS,
   wrf_nmm_regional=.false.,wrf_mass_regional=.false.,twodvar_regional=.true.,
   diagnostic_reg=.false.,
   filled_grid=.false.,half_grid=.true.,netcdf=.false.,
 /
 &BKGERR
   hzscl=1.414,1.000,0.707,
   vs=0.5,bw=0.0,
 /
 &ANBKGERR
   anisotropic=.true.,an_vs=0.5,ngauss=1,
   an_flen_u=-5.,an_flen_t=3.,an_flen_z=-200.,
   ifilt_ord=2,npass=3,normal=-200,grid_ratio=1.,nord_f2a=4,
   rtma_subdomain_option=$rtma_subdomain_option,triad4=.true.,nsmooth=0,nsmooth_shapiro=0,lreadnorm=$lreadnorm,
 /
 &JCOPTS
 /
 &STRONGOPTS
 /
 &OBSQC
   dfact=0.75,dfact1=3.0,noiqc=.false.,oberrflg=.false.,
   c_varqc=0.02,vadfile='prepbufr',hilbert_curve=$hilbert_curve
 /
 &OBS_INPUT
   dmesh(1)=60.0,dmesh(2)=60.0,dmesh(3)=60.0,dmesh(4)=60.0,time_window_max=3.0,
   dfile(01)='prepbufr',  dtype(01)='ps',  dplat(01)=' ', dsis(01)='ps',  dval(01)=1.0,  dthin(01)=0,
   dfile(02)='prepbufr'   dtype(02)='t',   dplat(02)=' ', dsis(02)='t',   dval(02)=1.0,  dthin(02)=0,
   dfile(03)='prepbufr',  dtype(03)='q',   dplat(03)=' ', dsis(03)='q',   dval(03)=1.0,  dthin(03)=0,
   dfile(04)='prepbufr',  dtype(04)='uv',  dplat(04)=' ', dsis(04)='uv',  dval(04)=1.0,  dthin(04)=0,
   dfile(05)='satwnd',    dtype(05)='uv',  dplat(05)=' ', dsis(05)='uv',  dval(05)=1.0,  dthin(05)=0,
   dfile(06)='prepbufr',  dtype(06)='spd', dplat(06)=' ', dsis(06)='spd', dval(06)=1.0,  dthin(06)=0,
   dfile(07)='prepbufr',  dtype(07)='gust',dplat(07)=' ', dsis(07)='gust',dval(07)=1.0,  dthin(07)=0,
   dfile(08)='prepbufr',  dtype(08)='vis', dplat(08)=' ', dsis(08)='vis', dval(08)=1.0,  dthin(08)=0,
 /
 &SUPEROB_RADAR
 /
 &LAG_DATA
 /
 &HYBRID_ENSEMBLE
 /
 &RAPIDREFRESH_CLDSURF
 /
 &CHEM
/
 &SINGLEOB_TEST
   maginnov=5.,magoberr=1.,oneob_type='v',
   oblat=36.,oblon=264.,obpres=1000.,obdattim=${ANAL_TIME},
   obhourset=0.,
 /
EOF
cat << EOF > parmcard_input
&parmcardanisof
    latdepend=.true.,
    scalex1=1.,
    scalex2=1.,
    scalex3=1.,
    afact0=1.,
    hsteep=500.,
    lsmoothterrain=.true.,
    hsmooth_len=1.0,
    volpreserve=.false.,
    lwater_scaleinfl=.true.,
    water_scalefactpsi=1.,
    water_scalefactchi=1.,
    water_scalefacttemp=2.,
    water_scalefactq=2.,
    water_scalefactpsfc=1.,
    water_scalefactgust=2.,
    water_scalefactvis=2.,
    water_scalefactpblh=2.,
    nhscale_pass=1,
    rltop_wind=50000000.,
    rltop_temp=1200.,
    rltop_q=1200.,
    rltop_psfc=1200.,
    rltop_gust=50000000.,
    rltop_vis=2400.,
    rltop_pblh=1200.,
    svpsi=0.21,
    svchi=0.2888,
    svpsfc=0.7,
    svtemp=0.6666,
    svshum=0.75,
    sclpsi=0.4725,
    sclchi=0.4725,
    sclpsfc=0.4658,
    scltemp=0.675,
    sclhum=0.60
    sclgust=0.675,
    sclvis=0.60,
    sclpblh=0.45
/
&parmcardreadprepb
    cgrid='${cgrid}',
    ladjusterr=.false.,
    oberrinflfact=5.0,
    ineighbour=3,
    jneighbour=3
    lshoreline=.false.
    slmland=0.9
/
&parmcardhcurve
    random_cvgrp=.true.
    usagecv=3.
    ngrps_tob=5
    ngrps_qob=5
    ngrps_psob=5
    ngrps_uvob=8
/
EOF

#
###################################################
#  run  GSI
###################################################
echo ' Run GSI with' ${bk_core} 'background'

case $ARCH in
   'IBM_LSF')
      ${RUN_COMMAND} ./gsi.exe < gsiparm.anl > stdout 2>&1  ;;

   * )
      ${RUN_COMMAND} ./gsi.exe > stdout 2>&1  ;;
esac

##################################################################
#  run time error check
##################################################################
error=$?

if [ ${error} -ne 0 ]; then
  echo "ERROR: ${GSI} crashed  Exit status=${error}"
  exit ${error}
fi
#
##################################################################
#
#   GSI updating satbias_in
#
# cp ./satbias_out ${FIX_ROOT}/ndas.t06z.satbias.tm03

# Copy the output to more understandable names
ln -s stdout      stdout.anl.${ANAL_TIME}
ln -s wrf_inout   wrfanl.${ANAL_TIME}
ln -s fort.201    fit_p1.${ANAL_TIME}
ln -s fort.202    fit_w1.${ANAL_TIME}
ln -s fort.203    fit_t1.${ANAL_TIME}
ln -s fort.204    fit_q1.${ANAL_TIME}

# Loop over first and last outer loops to generate innovation
# diagnostic files for indicated observation types (groups)
#
# NOTE:  Since we set miter=2 in GSI namelist SETUP, outer
#        loop 03 will contain innovations with respect to
#        the analysis.  Creation of o-a innovation files
#        is triggered by write_diag(3)=.true.  The setting
#        write_diag(1)=.true. turns on creation of o-g
#        innovation files.
#

loops="01 02 03"
for loop in $loops; do

case $loop in
  01) string=ges;;
  03) string=anl;;
   *) string=$loop;;
esac

#  Collect diagnostic files for obs types (groups) below
   listall="conv"
   for type in $listall; do
      count=0
      if [[ -f pe0000.${type}_${loop} ]]; then
         count=`ls pe*${type}_${loop}* | wc -l`
      fi
      if [[ $count -gt 0 ]]; then
         cat pe*${type}_${loop}* > diag_${type}_${string}.${ANAL_TIME}
      fi
   done
done

#  Clean working directory to save only important files 
if [ ${if_clean} = clean ]; then
  echo ' Clean working directory after GSI run'
  rm -f pe0*           # diag files on each processor
  rm -f obs_input.*    # observation middle files
  rm -f siganl sigf01  # background middle files
  rm -f fsize_*        # delete temperal file for bufr size
fi

exit 0
