#pragma section-numbers 2

= MPI-Start as EMI-ES ParallelEnvironment backend =

<<TableOfContents>>


== CREAM ==

Cream support for the !ParallelEnvironment uses mpi-start out of the box for its implementation. There is no need for extra configuration.
The default job generated by CREAM looks like this:
{{{
/usr/bin/mpi-start -t <ParallelEnvironmentType> -npnode <ProccessesPerHost> -d THREADS_PER_PROCESS=<ThreadsPerProcess> <UserExecutable> <UserExecutableArgs>
}}}



== UNICORE ==

The UNICORE EMI-ES !ParallelEnvironment is also a complete implementation of the specification. 

Configuration is done via xml files where the mapping from the job specification to the final executable is specified. For each mpi-flavor that needs to be supported by mpi-start an entry in the idb must be included. For example for OpenMPI:
{{{#!highlight xml

<ee:ExecutionEnvironment xmlns:ee="http://www.unicore.eu/unicore/jsdl-extensions">
    <ee:Name>OpenMPI</ee:Name>
    <ee:Version>1.5.4</ee:Version>
    <ee:Description>Runs OpenMPI job using mpi-start to start the job</ee:Description>
    <ee:ExecutableName>/usr/bin/mpi-start</ee:ExecutableName>
    <ee:CommandlineTemplate>#EXECUTABLE -t openmpi #ARGS #USERCOMMAND #USERARGS</ee:CommandlineTemplate>
    <ee:Argument>
      <ee:Name>Output</ee:Name>
      <ee:IncarnatedValue>-o </ee:IncarnatedValue>
      <ee:ArgumentMetadata>
        <ee:Type>string</ee:Type>
        <ee:Description>Write the job output to a FILE instead of to the standard output stream</ee:Description>
      </ee:ArgumentMetadata>
    </ee:Argument>
    <ee:Argument>
      <ee:Name>PostHook</ee:Name>
      <ee:IncarnatedValue>-post </ee:IncarnatedValue>
      <ee:ArgumentMetadata>
        <ee:Type>string</ee:Type>
        <ee:Description>Use the file as post hook</ee:Description>
      </ee:ArgumentMetadata>
    </ee:Argument>
    <ee:Argument>
      <ee:Name>PreHook</ee:Name>
      <ee:IncarnatedValue>-pre </ee:IncarnatedValue>
      <ee:ArgumentMetadata>
        <ee:Type>string</ee:Type>
        <ee:Description>Use the file as pre hook</ee:Description>
      </ee:ArgumentMetadata>
    </ee:Argument>
    <ee:Argument>
      <ee:Name>Error</ee:Name>
      <ee:IncarnatedValue>-e </ee:IncarnatedValue>
      <ee:ArgumentMetadata>
        <ee:Type>string</ee:Type>
        <ee:Description>Write the job error to a FILE instead of to the standard error stream</ee:Description>
      </ee:ArgumentMetadata>
    </ee:Argument>
    <ee:Option> 
      <ee:Name>Verbose</ee:Name> 
      <ee:IncarnatedValue>-v</ee:IncarnatedValue> 
      <ee:OptionMetadata> 
        <ee:Description>Enable verbose mode</ee:Description> 
      </ee:OptionMetadata> 
    </ee:Option> 
    <ee:Option> 
      <ee:Name>PerNode</ee:Name> 
      <ee:IncarnatedValue>-pnode</ee:IncarnatedValue> 
      <ee:OptionMetadata> 
        <ee:Description>Start one process per available node</ee:Description> 
      </ee:OptionMetadata> 
    </ee:Option> 
    <ee:Option> 
      <ee:Name>PerSocket</ee:Name> 
      <ee:IncarnatedValue>-psocket</ee:IncarnatedValue> 
      <ee:OptionMetadata> 
        <ee:Description>Start one process per available socket</ee:Description> 
      </ee:OptionMetadata> 
    </ee:Option> 
    <ee:Option> 
      <ee:Name>PerCore</ee:Name> 
      <ee:IncarnatedValue>-pcore</ee:IncarnatedValue> 
      <ee:OptionMetadata> 
        <ee:Description>Start one process per available core</ee:Description> 
      </ee:OptionMetadata> 
    </ee:Option> 
</ee:ExecutionEnvironment>
}}}

== ARC ==

ARC supports the !ParallelEnvironment through the !RunTimeEnvironments. You need to create a RTE for the !ParallelEnvironment that uses mpi-start:

{{{#!highlight sh
#!/bin/sh

case "$1" in
0)
        TYPE=`echo "$joboption_penv_type" | tr [:upper:] [:lower:]`
        OPTS="-t $TYPE"
        if [ "x$joboption_penv_procperslot" != "x" ] ; then
                OPTS="$OPTS -npnode $joboption_penv_procperslot"
        fi
        if [ "x$joboption_penv_threadsperslot" != "x" ] ; then
                OPTS="$OPTS -d THREADS_PER_CORE=$joboption_penv_threadsperslot"
        fi
        joboption_args="mpi-start $OPTS -- $joboption_args"
;;
1)
;;
2)
;;
*)
        return 1
;;
esac
}}}

In the user job, you should include both the RTE (e.g. MPISTART) and PE in the job description (only shown the relevant elements):
{{{#!highlight xml
<RuntimeEnvironment><Name>MPISTART</Name></RuntimeEnvironment>
<ParallelEnvironment>
  <Type>OpenMPI</Type>
  <!-- Any other PE options -->
</ParallelEnvironment>
}}}

== Testing ==

Testing the use of PEs with mpi-start was done as part of mpi-start 1.5.0 testing. Check [[https://bitbucket.org/enolfc/mpi-start/wiki/emi-MPIv1.5.0/integration|the integration tests]] for details.

== Sample Job ==

This is a sample job that can be submitted to the EMI-ES endpoints:
{{{#!highlight xml
<ActivityDescription xmlns="http://www.eu-emi.eu/es/2010/12/adl">
    <ActivityIdentification>
        <Name>test job</Name>
        <Description>A test job showing the features of EMI-ES</Description>
        <Type>single</Type>
        <Annotation>test</Annotation>
    </ActivityIdentification>
    <Application>
        <Executable><Path>cpi.c</Path></Executable>
        <Error>std.err</Error>
        <Output>std.out</Output>
        <Environment><Name>I2G_MPI_PRE_RUN_HOOK</Name><Value>pre.sh</Value></Environment>
        <Environment><Name>I2G_MPI_START_VERBOSE</Name><Value>1</Value></Environment>
    </Application>
    <Resources>
        <SlotRequirement>
            <NumberOfSlots>2</NumberOfSlots>
        </SlotRequirement>
        <ParallelEnvironment>
                <Type>OpenMPI</Type>
        </ParallelEnvironment>
    </Resources>
    <DataStaging>
        <InputFile> <Name>pre.sh</Name> <Source><URI>pre.sh</URI></Source> </InputFile>
        <InputFile> <Name>cpi.c</Name> <Source><URI>cpi.c</URI></Source> </InputFile>
    </DataStaging>
</ActivityDescription>
}}}

The hook (`pre.sh`) is the following:
{{{#!highlight sh
#!/bin/sh

pre_run_hook () {
  # Compile the program.
  info_msg "Compiling ${I2G_MPI_APPLICATION}"

  export I2G_MPI_APPLICATION=`echo $I2G_MPI_APPLICATION | sed -e "s/\.c$//"`
  # Actually compile the program.
  cmd="${MPI_MPICC} ${MPI_MPICC_OPTS} -o ${I2G_MPI_APPLICATION} ${I2G_MPI_APPLICATION}.c"
  info_msg $cmd
  $cmd
  st=$?
  if [ $st -ne 0 ]; then
    error_msg "Error compiling program.  Exiting..."
    return $st 
  fi

  # Everything's OK.
  info_msg "Successfully compiled ${I2G_MPI_APPLICATION}"
  return 0
}
}}}

And the C code (`cpi.c`):
{{{#!highlight c
#include "mpi.h"
#include <stdio.h>
#include <math.h>

double f( double );
double f( double a )
{
    return (4.0 / (1.0 + a*a));
}

int main( int argc, char *argv[])
{
   int n_intervals = 16384;

   int done = 0, n, myid, numprocs, i;
   double PI25DT = 3.141592653589793238462643;
   double mypi, pi, h, sum, x;
   double startwtime = 0.0, endwtime;
   int  namelen;
   char processor_name[MPI_MAX_PROCESSOR_NAME];

   MPI_Init(&argc,&argv);
   MPI_Comm_size(MPI_COMM_WORLD,&numprocs);
   MPI_Comm_rank(MPI_COMM_WORLD,&myid);
   MPI_Get_processor_name(processor_name,&namelen);

   fprintf(stderr,"Process %d on %s: n=%d\n",myid, processor_name,n);
   if (numprocs >= 1) {
       if( myid == 0 ) fprintf(stderr,"Using %d intervals\n",n_intervals);

       n = 0;
       while (!done)
       {
      if (myid == 0) {
         startwtime = MPI_Wtime();
      }
      if( n == 0  ) n = n_intervals; else n = 0;
      MPI_Bcast(&n, 1, MPI_INT, 0, MPI_COMM_WORLD);
      if (n == 0)
         done = 1;
      else
      {
         h   = 1.0 / (double) n;
         sum = 0.0;
         for (i = myid + 1; i <= n; i += numprocs)
         {
        x = h * ((double)i - 0.5);
        sum += f(x);
         }
         mypi = h * sum;

         MPI_Reduce(&mypi, &pi, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
         if (myid == 0)
         {
        printf("pi is approximately %.16f, Error is %.16f\n",
               pi, fabs(pi - PI25DT));
        endwtime = MPI_Wtime();
        printf("wall clock time = %f\n",
        endwtime-startwtime);
         }
      }
       }
   } else {
       fprintf(stderr,"Only 1 process, not doing anything");
   }
   MPI_Finalize();

   return 0;
}
}}}