#!/bin/ksh
#set -x
#
############### User's section #######################################
##
## - Define architecture
arch=Linux       # Linux/SX/ibm_power4/ibm_egee/SX8
libmpi=mpich    # For ibm_egee: openmpi/lammpi, for Linux: mpich/lammpi
host=`uname -n`
## - Define paths
srcdir=`pwd`
toyname=`basename $srcdir`
prismdir=/home/valcke/prism_trunk
user=`whoami`

if [ $arch == SX ]; then
    rundir=/import/ds9b/hpf/b/b340041/prism/work_$toyname
elif [ $arch == ibm_egee ]; then
    rundir=$srcdir/work_$toyname
elif [ $arch == SX8 ]; then
    rundir=/work/$user/$toyname/work2_$toyname
    expdir=/work/$user/$toyname/expce_$toyname
else
    rundir=/space/valcke/work_$toyname
fi

## - Define names of executables
# oasis3
oa3_exe=oasis3.MPI1.x

# toy models: Define number of processes to run each executable
if [ $toyname == gridless ]; then
    exe1=source.x
    exe2=target.x
    nproc_exe1=3
    nproc_exe2=2
    nproc_oa3=1
elif [ $toyname == benchmark ]; then
    exe1=appl-atm.x
    exe2=appl-ocn.x
    nproc_exe1=1
    nproc_exe2=1
    nproc_oa3=1
elif [ $toyname == simple-mg ]; then
    exe1=appl-atm.x
    exe2=appl-ocn.x
    nproc_exe1=1
    nproc_exe2=2
    nproc_oa3=1
elif [ $toyname == userdef ]; then
    exe1=src_main
    exe2=tgt_main
    nproc_exe1=3
    nproc_exe2=2
    nproc_oa3=1
elif [ $toyname == simple-mg2g ]; then
    exe1=appl-atm.x
    exe2=appl-ocn.x
    nproc_exe1=2
    nproc_exe2=1
    nproc_oa3=1
elif [ $toyname == simple-mg-io-forrd ]; then
    exe1=appl-atm.x
    exe2=appl-ocn.x
    nproc_exe1=1
    nproc_exe2=1
    nproc_oa3=1
elif [ $toyname == simple-mg-io-rd ]; then
    exe1=appl-atm.x
    exe2=appl-ocn.x
    nproc_exe1=1
    nproc_exe2=1
    nproc_oa3=1
elif [ $toyname == simple-mg-io-wrt ]; then
    exe1=appl-atm.x
    exe2=appl-ocn.x
    nproc_exe1=1
    nproc_exe2=1
    nproc_oa3=1
elif [ $toyname == simple-mg-io-wrt2 ]; then
    exe1=appl-atm.x
    exe2=appl-ocn.x
    nproc_exe1=1
    nproc_exe2=1
    nproc_oa3=1
elif [ $toyname == simple-mgred ]; then
    exe1=appl-atm.x
    exe2=appl-ocn.x
    nproc_exe1=1
    nproc_exe2=1
elif [ $toyname == c_grid_problem ]; then
    exe1=toyatm.x
    exe2=toyoce.x
    nproc_exe1=1
    nproc_exe2=1
    nproc_oa3=1
elif [ $toyname == simple_reg2regKM ]; then
    exe1=appl-atm.x
    exe2=appl-ocn.x
    nproc_exe1=1
    nproc_exe2=1
    nproc_oa3=1
elif [ $toyname == simple_oneway ]; then
    exe1=appl-ocn.x
    exe2=appl-atm.x
    nproc_exe1=1
    nproc_exe2=1
    nproc_oa3=1
elif [ $toyname == conserv ]; then
    exe1=appl-ocn.x
    exe2=appl-atm.x
    nproc_exe1=1
    nproc_exe2=1
    nproc_oa3=1
elif [ $toyname == conserv2 ]; then
    exe1=appl-ocn.x
    exe2=appl-atm.x
    nproc_exe1=1
    nproc_exe2=1
    nproc_oa3=1
elif [ $toyname == ripples ]; then
    exe1=appl-atm.x
    exe2=appl-ocn.x
    nproc_exe1=1
    nproc_exe2=1
    nproc_oa3=1
elif [ $toyname == gauss_red ]; then
    exe1=atmoa3.x
    exe2=oceoa3.x
    nproc_exe1=2
    nproc_exe2=1
    nproc_oa3=1
elif [ $toyname == test-trili ]; then
    exe1=appl-atm.x
    exe2=appl-ocn.x
    nproc_exe1=1
    nproc_exe2=1
    nproc_oa3=1
elif [ $toyname == ajs-toys-rgg ]; then
    exe1=appl-ctm.x
    exe2=appl-gcm.x
    nproc_exe1=1
    nproc_exe2=1
    nproc_oa3=1
elif [ $toyname == playback ]; then
    exe1=pseudo.x
    exe2=pseudo.x
    nproc_exe1=1
    nproc_exe2=1
    nproc_oa3=1
elif [ $toyname == create_restart ]; then
# exe1 : rest_toyoa3.x or rest_toy2.x 
    exe1=rest_toyoa3.x
    cp -f scc_toyoa3.xml scc.xml
#    exe1=rest_toy2.x
#    cp -f scc_toy2.xml scc.xml
    exe2=none
    nproc_exe1=1
    nproc_exe2=0
    oa3_exe=$exe1
elif [ $toyname == simple-cc ]; then
    exe1=appl.x
    exe2=appl.x
    nproc_exe1=2
    nproc_exe2=0
    nproc_oa3=1
elif [ $toyname == mytoys ]; then
    exe1=appl-ctm.x
    exe2=appl-gcm.x
    nproc_exe1=2
    nproc_exe2=2
    nproc_oa3=1
elif [ $toyname == interpolation-grilles-cicle ]; then
    exe1=source.x
    exe2=target.x
    nproc_exe1=1
    nproc_exe2=1
    nproc_oa3=1
else
    echo 'new example, fill in : '
    exe1=xxxx
    exe2=xxxx
    nproc_exe1=0
    nproc_exe2=0
fi

############### End of user's section ################################

echo ''
echo '*****************************************************************'
echo '***    toyoa3 '$run
echo ''
echo 'Rundir       :' $rundir
echo 'Architecture :' $arch
echo 'Host         : '$host
echo ''
######################################################################
###
### 1. Copy source example directory containing everything needed
###    into rundir

rm -fr $rundir
mkdir -p $rundir

cp -f $srcdir/namcouple $rundir/.
cp -f $srcdir/cf_name_table.txt  $rundir/.
cp -f $srcdir/*nc  $rundir/.
cp -f $srcdir/*.x  $rundir/.
cp -f $srcdir/$exe1  $rundir/.
cp -f $srcdir/$exe2  $rundir/.
if [ $toyname == gauss_red ]; then
    ln -s $prismdir/util/running/toyoa3/data/*_in.nc $rundir/.
fi

cd $rundir

######################################################################
###
### 3. Creation of configuration scripts
   
###
### Linux
###
if [ $arch == Linux ] && [ $toyname == create_restart ]; then
  cat <<EOF> $rundir/appl-linux.conf
$host 0 $rundir/$exe1
EOF


elif [ $arch == Linux ] && [ $libmpi == mpich ]; then
  cat <<EOF> $rundir/appl-linux.conf
$host 0 $rundir/$oa3_exe
EOF

  count=1
  while [[ $count -lt $nproc_oa3 ]];do
      cat <<EOF>> $rundir/appl-linux.conf
$host 1 $rundir/$oa3_exe
EOF
   (( count += 1 ))
  done

  count=0
  while [[ $count -lt $nproc_exe1 ]];do
   cat <<EOF>> $rundir/appl-linux.conf
$host 1 $rundir/$exe1
EOF
   (( count += 1 ))
  done

  count=0
  while [[ $count -lt $nproc_exe2 ]];do
   cat <<EOF>> $rundir/appl-linux.conf
$host 1 $rundir/$exe2
EOF
   (( count += 1 ))
  done

elif [ $arch == Linux ] && [ $libmpi == lammpi ]; then
echo 'LAMMPI on LINUX'

  cat <<EOF >$rundir/hostfile
$host
EOF
  cat <<EOF >$rundir/appl.conf
c0 -np 1 $oa3_exe
EOF
  count=0
  while [[ $count -lt $nproc_exe1 ]];do
   cat <<EOF>> $rundir/hostfile
$host
EOF
   cat <<EOF>> $rundir/appl.conf
c0 -np 1 $exe1
EOF
   (( count += 1 ))
  done

  count=0
  while [[ $count -lt $nproc_exe2 ]];do
   cat <<EOF>> $rundir/hostfile
$host
EOF
   cat <<EOF>> $rundir/appl.conf
c0 -np 1 $exe2
EOF
   (( count += 1 ))
  done

  (( nproc = 1 + $nproc_exe1 + $nproc_exe2 ))

###
### Meteo-France NEC SX-8
###

elif [ $arch == SX8 ] ; then

# - Define Maximum number of proc per node for arch=SX8
#
nprocmax=8

  (( nproc = $nproc_oa3 + $nproc_exe1 + $nproc_exe2 )) 

  # Automatic creation of the configuration file
  # maximizing the nodes fill rate

  \rm -f systemSX8.conf
  count=0
  node=0

  nn=1
  while [ $nn -le $nproc_oa3 ] ; do
    ((count+=1))
    if [ count -lt nn ] ; then proc=$count
    else proc=$nn ; fi
    cat <<EOF > temp.oa3.$node
-h $node -p $proc -e $oa3_exe
EOF
    ((nn+=1))
    if [ $count -gt $nprocmax ] ; then
      ((node+=1))
      count=0
    fi
  done

  nn=1
  while [ $nn -le $nproc_exe1 ] ; do
    ((count+=1))
    if [ count -lt nn ] ; then proc=$count
    else proc=$nn ; fi
    cat <<EOF > temp.exe1.$node
-h $node -p $proc -e $exe1
EOF
    ((nn+=1))
    if [ $count -gt $nprocmax ] ; then
      ((node+=1))
      count=0
    fi
  done

  nn=1
  while [ $nn -le $nproc_exe2 ] ; do
    ((count+=1))
    if [ count -lt nn ] ; then proc=$count
    else proc=$nn ; fi
    cat <<EOF > temp.exe2.$node
-h $node -p $proc -e $exe2
EOF
    ((nn+=1))
    if [ $count -gt $nprocmax ] ; then
      ((node+=1))
      count=0
    fi
  done

  for fich in `ls temp.oa3.*` ; do
  cat $fich >> systemSX8.conf ; done
  for fich in `ls temp.exe1.*` ; do
  cat $fich >> systemSX8.conf ; done
  for fich in `ls temp.exe2.*` ; do
  cat $fich >> systemSX8.conf ; done
  
  \rm -f temp.*.*

  echo 'systemSX8.conf created'

  ((nnode = $node+1))

  if [ $nproc -lt $nprocmax ] ; then
###    nproc_used=$nproc
    nproc_used=$nprocmax
    queue=vector
  else
    nproc_used=$nprocmax
    queue=vector
  fi
 
  cat <<EOF > $rundir/run_$toyname.$arch
#!/bin/ksh
#PBS -T mpisx                   # Batch job = MPI/SX job (mandatory for multi-node jobs !)
#PBS -q $queue                  # Queue name
#PBS -b $nnode                  # number of nodes
#PBS -l cpunum_job=$nproc_used  # number of CPUs (maximum number of CPUs = 8)
#PBS -l elapstim_req=00:10:00   # 1 hour realtime  
#PBS -l cputim_job=00:10:00     # 1 hour cputime
#PBS -l memsz_job=4gb           # 4 Gbyte memory
#PBS -j o                       # join err and out to out
#PBS -N r_$toyname              # job name
##PBS -m e                      # E-mail is sent when request execution ends
##PBS -M valcke@cerfacs.fr       # Mail send-to destinations

#
/bin/echo " job started at: " \\c
date
/bin/echo " ExecutionHost : " \\c
hostname                        # print name of current host

MPISUSPEND=ON; export MPISUSPEND

## - For performances analysis
MPIPROGINF=all_detail; export MPIPROGINF
XLFRTEOPTS=cpu_time_type=usertime; export XLFRTEOPTS
F_TRACE=yes; export F_TRACE

cd $rundir

#mpiexec -n $nproc_oa3 $oa3_exe : -n $nproc_exe1 $exe1 : \
#	 -n $nproc_exe2 $exe2 : -n $nproc_exe3 $exe3

mpirun -f systemSX8.conf

/bin/echo " job completed at: " \\c
date

EOF

###
### NEC SX-6
###

elif [ $arch == SX ] && [ $toyname == create_restart ] ; then

 (( nproc = $nproc_exe1 )) 

  cat <<EOF >$rundir/run_$toyname.$arch
#!/bin/ksh
#PBS -l cpunum_prc=$nproc        # number of CPUs (maximum number of CPUs 8)
#PBS -l cputim_job=00:30:00 # 30 min cputime
#PBS -l memsz_job=3gb       # 3 Gbyte memory
#PBS -j o                   # join err and out to out
#PBS -N $toyname              # job name
#
/bin/echo " job started at: " \\c
date
/bin/echo " ExecutionHost : " \\c
hostname                   # print name of current host

MPISUSPEND=ON; export MPISUSPEND

cd $rundir

mpiexec -n $nproc_exe1 $exe1 

/bin/echo " job completed at: " \\c
date

EOF



elif [ $arch == SX ] && [ $toyname == simple-cc ] ; then

 (( nproc = 1 + 2 )) 

  cat <<EOF >$rundir/run_$toyname.$arch
#!/bin/ksh
#PBS -l cpunum_prc=$nproc        # number of CPUs (maximum number of CPUs 8)
#PBS -l cputim_job=00:30:00 # 30 min cputime
#PBS -l memsz_job=3gb       # 3 Gbyte memory
#PBS -j o                   # join err and out to out
#PBS -N $toyname              # job name
#
/bin/echo " job started at: " \\c
date
/bin/echo " ExecutionHost : " \\c
hostname                   # print name of current host

MPISUSPEND=ON; export MPISUSPEND

cd $rundir

mpiexec -n 1 $oa3_exe : -n 2 $exe1 

/bin/echo " job completed at: " \\c
date

EOF


elif [ $arch == SX ]; then

 (( nproc = 1 + $nproc_exe1 + $nproc_exe2 )) 

  cat <<EOF >$rundir/run_$toyname.$arch
#!/bin/ksh
#PBS -l cpunum_prc=$nproc        # number of CPUs (maximum number of CPUs 8)
#PBS -l cputim_job=00:30:00 # 30 min cputime
#PBS -l memsz_job=3gb       # 3 Gbyte memory
#PBS -j o                   # join err and out to out
#PBS -N $toyname              # job name
#
/bin/echo " job started at: " \\c
date
/bin/echo " ExecutionHost : " \\c
hostname                   # print name of current host

MPISUSPEND=ON; export MPISUSPEND

cd $rundir

mpiexec -n 1 $oa3_exe : -n $nproc_exe1 $exe1 : -n $nproc_exe2 $exe2

/bin/echo " job completed at: " \\c
date

EOF

###
### IBM power4
###

elif  [ $arch == ibm_power4 ] && [ $toyname == create_restart ]; then

  cat <<EOF >$rundir/appl-aix.conf
$exe1
EOF

  cat <<EOF >$rundir/run_toy.$arch
#!/bin/ksh 
#
#  This script will launch toyoa3 with oasis3
#
#@ shell           = /usr/bin/ksh 
#@ class           = ns
#@ job_type        = parallel
#@ job_name        = create_restart
#@ output          = \$(job_name).\$(schedd_host).\$(jobid).out 
#@ error           = \$(job_name).\$(schedd_host).\$(jobid).out 
#@ notification    = never  
#@ resources       = ConsumableCpus(1) ConsumableMemory(100mb) 
#@ cpu_limit        = 00:30:00 
#@ wall_clock_limit = 00:30:00 
#@ queue 

MPI_DSM_VERBOSE=1; export MPI_DSM_VERBOSE
poe -pgmmodel mpmd -cmdfile appl-aix.conf

exit 0  

EOF


elif  [ $arch == ibm_power4 ]; then

  cat <<EOF >$rundir/appl-aix.conf
$oa3_exe
EOF

  count=0
  while [[ $count -lt $nproc_exe1 ]];do
   cat <<EOF>> $rundir/appl-aix.conf
$exe1
EOF
   (( count += 1 ))
  done

  count=0
  while [[ $count -lt $nproc_exe2 ]];do
   cat <<EOF>> $rundir/appl-aix.conf
$exe2
EOF
   (( count += 1 ))
  done


  (( nproc = 1 + $nproc_exe1 + $nproc_exe2 )) 

  cat <<EOF >$rundir/run_toy.$arch
#!/bin/ksh 
#
#  This script will launch toyoa3 with oasis3
#
#@ shell           = /usr/bin/ksh 
#@ class           = np 
#@ job_type        = parallel 
#@ job_name        = $toyname
#@ output          = \$(job_name).\$(schedd_host).\$(jobid).out 
#@ error           = \$(job_name).\$(schedd_host).\$(jobid).out 
#@ notification    = never  
#@ resources       = ConsumableCpus(1) ConsumableMemory(100mb) 
#@ node            = 1
#@ tasks_per_node  = $nproc
#@ cpu_limit        = 00:30:00 
#@ wall_clock_limit = 00:30:00 
#@ queue 

MPI_DSM_VERBOSE=1; export MPI_DSM_VERBOSE
poe -pgmmodel mpmd -cmdfile appl-aix.conf

exit 0  

EOF

elif  [ $arch == ibm_egee ] && [ $libmpi == openmpi ]; then
echo 'IBM_EGEE'
  cat <<EOF >$rundir/appl.conf
$rundir/$oa3_exe
EOF

  count=0
  while [[ $count -lt $nproc_exe1 ]];do
   cat <<EOF>> $rundir/appl.conf
$rundir/$exe1
EOF
   (( count += 1 ))
  done

  count=0
  while [[ $count -lt $nproc_exe2 ]];do
   cat <<EOF>> $rundir/appl.conf
$rundir/$exe2
EOF
   (( count += 1 ))
  done

  (( nproc = 1 + $nproc_exe1 + $nproc_exe2 )) 

if [ $nproc -gt 8  ]; then
    echo 'nproc cannot be greater than 8; change node and tasks_per_node'
    stop
fi
 
  cat <<EOF >$rundir/run_toy.$arch
#!/bin/sh 
#
#  This script will launch toyoa3 with oasis3
#
#@ class           = test 

#@ job_type        = parallel 
#@ job_name        = $toyname
#@ output          = \$(job_name).\$(schedd_host).\$(jobid).out 
#@ error           = \$(job_name).\$(schedd_host).\$(jobid).out 
#@ notify_user     = valcke@cerfacs.fr
#@ environment = COPY_ALL 
#@ node            = 1
#@ tasks_per_node  = $nproc
#@ cpu_limit        = 00:15:00 
#@ wall_clock_limit = 00:15:00 
#@ queue
 
cd \$LOADL_STEP_INITDIR
###export MP_IMPL=ompi

hfile=.hostfile_ll_$$
ll_get_machine_list >\$hfile
NPROCS=\$(wc -l \$hfile |awk '{print \$1}')

MPI_DSM_VERBOSE=1; export MPI_DSM_VERBOSE
paste -d " " \$hfile appl.conf > appl-aix1.conf
awk '{print "-np 1 --host " \$1 "  " \$2}' appl-aix1.conf  > appl-aix.conf
cat  appl-aix.conf

/usr/bin/mpirun -machinefile \$hfile -app appl-aix.conf > runjob.err

rm \$hfile
rm appl-aix1.conf

exit 0  

EOF

elif  [ $arch == ibm_egee ] && [ $libmpi == lammpi ]; then
echo 'IBM_EGEE'
  cat <<EOF >$rundir/appl.conf
-np 1 $rundir/$oa3_exe
EOF

  count=0
  while [[ $count -lt $nproc_exe1 ]];do
   cat <<EOF>> $rundir/appl.conf
-np 1 $rundir/$exe1
EOF
   (( count += 1 ))
  done

  count=0
  while [[ $count -lt $nproc_exe2 ]];do
   cat <<EOF>> $rundir/appl.conf
-np 1 $rundir/$exe2
EOF
   (( count += 1 ))
  done

  (( nproc = 1 + $nproc_exe1 + $nproc_exe2 )) 

if [ $nproc -gt 8  ]; then
    echo 'nproc cannot be greater than 8; change node and tasks_per_node'
    stop
fi
 
  cat <<EOF >$rundir/run_toy.$arch
#!/bin/sh 
#
#  This script will launch toyoa3 with oasis3
#
#@ class           = test 

#@ job_type        = parallel 
#@ job_name        = $toyname
#@ output          = \$(job_name).\$(schedd_host).\$(jobid).out 
#@ error           = \$(job_name).\$(schedd_host).\$(jobid).out 
#@ notify_user     = valcke@cerfacs.fr
#@ environment = COPY_ALL 
#@ node            = 1
#@ tasks_per_node  = $nproc
#@ cpu_limit        = 00:15:00 
#@ wall_clock_limit = 00:15:00 
#@ queue
 
cd \$LOADL_STEP_INITDIR
export MP_IMPL=lam
export MP_EUILIB=ip
hfile=.hostfile_ll_$$
ll_get_machine_list  | sed  's/egee/egee_myri/' >\$hfile
NPROCS=\$(wc -l \$hfile |awk '{print \$1}')

MPI_DSM_VERBOSE=1; export MPI_DSM_VERBOSE

/usr/bin/mpirun -machinefile \$hfile appl.conf > runjob.err

rm \$hfile

exit 0  

EOF

else 
  echo 'For architecture='$arch ' you have to add your execution'
  exit
fi 

######################################################################
###
### 4. Execute the model

if [ $arch == Linux ] && [ $libmpi == mpich ]; then

  echo 'Executing the model using /usr/local/pgi/linux86/9.0/mpi/mpich/bin/mpirun' 
  /usr/local/pgi/linux86/9.0/mpi/mpich/bin/mpirun -p4pg appl-linux.conf ./$oa3_exe > runjob.err

elif [ $arch == Linux ] && [ $libmpi == lammpi ]; then
  
echo 'Executing the model using /usr/local/lam7.1.4/bin/mpirun'
  /usr/local/lam7.1.4/bin/lamboot ./hostfile
  /usr/local/lam7.1.4/bin/mpirun -v appl.conf > runjob.err
  /usr/local/lam7.1.4/bin/lamhalt


elif [ $arch == SX ]; then

  echo 'Submitting the job to queue using qsub'
  qsub $rundir/run_$toyname.$arch
  qstat

elif [ $arch == SX8 ]; then

  echo 'Submitting the job to queue using qsub'
  qsub $rundir/run_$toyname.$arch
  qstat

elif  [ $arch == ibm_power4 ]; then
 
  echo 'Submitting the job to queue using llsubmit'    
  llsubmit $rundir/run_toy.$arch

elif  [ $arch == ibm_egee ]; then
 
  echo 'Submitting the job to queue using llsubmit'    
  llsubmit $rundir/run_toy.$arch    
fi

echo $toyname 'is executed or submitted to queue.'
echo 'Results are found in rundir : '$rundir 


exit

