This wiki is obsolete, see the NorduGrid web pages for up to date information.

LRMS Backends/Testbeds

From NorduGrid
Jump to navigationJump to search

Open Grid Scheduler (OGS/GE 2011.11p1)

Old name: Sun Grid Engine

Website: http://gridscheduler.sourceforge.net

Compute nodes

roczei@gemaster:~$ qstat -f
queuename                      qtype resv/used/tot. load_avg arch          states
---------------------------------------------------------------------------------
all.q@ogsexec1                 BIP   0/0/4          0.00     linux-x64     
---------------------------------------------------------------------------------
all.q@ogsexec2                 BIP   0/0/4          0.01     linux-x64     
---------------------------------------------------------------------------------
all.q@ogsexec3                 BIP   0/0/4          0.00     linux-x64     
---------------------------------------------------------------------------------
all.q@ogsexec4                 BIP   0/0/4          0.00     linux-x64     
roczei@gemaster:~$

Queue configuration

roczei@gemaster:~$ qconf -sq all.q
qname                 all.q
hostlist              ogsexec1 ogsexec2 ogsexec3 ogsexec4
seq_no                0,[ogsexec1=1],[ogsexec2=2],[ogsexec3=3],[ogsexec4=4]
load_thresholds       np_load_avg=1.75
suspend_thresholds    NONE
nsuspend              1
suspend_interval      00:05:00
priority              0
min_cpu_interval      00:05:00
processors            UNDEFINED
qtype                 BATCH INTERACTIVE
ckpt_list             NONE
pe_list               mpi openmp
rerun                 FALSE
slots                 4
tmpdir                /scratch/tmp
shell                 /bin/bash
prolog                NONE
epilog                NONE
shell_start_mode      unix_behavior
starter_method        NONE
suspend_method        NONE
resume_method         NONE
terminate_method      NONE
notify                00:00:60
owner_list            NONE
user_lists            NONE
xuser_lists           NONE
subordinate_list      NONE
complex_values        NONE
projects              NONE
xprojects             NONE
calendar              NONE
initial_state         default
s_rt                  01:50:00
h_rt                  02:00:00
s_cpu                 INFINITY
h_cpu                 INFINITY
s_fsize               INFINITY
h_fsize               INFINITY
s_data                INFINITY
h_data                INFINITY
s_stack               INFINITY
h_stack               INFINITY
s_core                0
h_core                0
s_rss                 INFINITY
h_rss                 INFINITY
s_vmem                INFINITY
h_vmem                INFINITY
roczei@gemaster:~$

Parallel enviroments

OpenMP

Open Grid Scheduler configuration

roczei@gemaster:~$ qconf -sp openmp
pe_name            openmp
slots              16
user_lists         NONE
xuser_lists        NONE
start_proc_args    NONE
stop_proc_args     NONE
allocation_rule    $pe_slots
control_slaves     TRUE
job_is_first_task  TRUE
urgency_slots      max
accounting_summary TRUE
roczei@gemaster:~$

ARC configuration

roczei@gemaster:/scratch/arc/rte/ENV/GCC$ cat OPENMP-2.5 
#!/bin/bash
parallel_env_name="openmp"
case "$1" in
0 ) # local LRMS specific settings
    i=0
    eval jonp=\${joboption_nodeproperty_$i}
    while [ ! -z $jonp ] ; do
        (( i++ ))
        eval jonp=\${joboption_nodeproperty_$i}
    done
    eval joboption_nodeproperty_$i=$parallel_env_name
;;
1 ) # nothing here
;;
2 ) # nothing here
;;
* ) # everything else is an error
    return 1
;;
esac
roczei@gemaster:/scratch/arc/rte/ENV/GCC$

Configuration of sge_shepherd

roczei@gemaster:/scratch/sge$ cat jail.sh 
#!/bin/bash

while IFS== read key val; do
    case "$key" in
        PE) PE="$val";;
        NSLOTS) NSLOTS="$val";;
    esac
done <environment

if [ ! -z "$PE" ]
then
   if [ "$PE" = "openmp" ]
   then
     export OMP_NUM_THREADS=$NSLOTS
   fi
fi

exec /home/ogs/bin/linux-x64/sge_shepherd $@
roczei@gemaster:/scratch/sge$ qconf -sconf | grep shepherd
shepherd_cmd                 /scratch/sge/jail.sh
roczei@gemaster:/scratch/sge$

MPI

Open Grid Scheduler configuration

 roczei@gemaster:~$ qconf -sp mpi
pe_name            mpi
slots              16
user_lists         NONE
xuser_lists        NONE
start_proc_args    NONE
stop_proc_args     NONE
allocation_rule    $fill_up
control_slaves     TRUE
job_is_first_task  FALSE
urgency_slots      max
accounting_summary FALSE
roczei@gemaster:~$

ARC configuration

roczei@gemaster:/scratch/arc/rte/ENV/GCC/MPI$ cat OPENMPI-1.6.3
#!/bin/bash
parallel_env_name="mpi"
case "$1" in
0 ) # local LRMS specific settings
    i=0
    eval jonp=\${joboption_nodeproperty_$i}
    while [ ! -z $jonp ] ; do
        (( i++ ))
        eval jonp=\${joboption_nodeproperty_$i}
    done
    eval joboption_nodeproperty_$i=$parallel_env_name
;;
1 ) # nothing here
    export PATH=/scratch/openmpi/bin:$PATH
    export LD_LIBRARY_PATH=/scratch/openmpi/lib:/scratch/openmpi/lib/openmpi:$LD_LIBRARY_PATH
;;
2 ) # nothing here
;;
* ) # everything else is an error
    return  1
;;
esac
roczei@gemaster:/scratch/arc/rte/ENV/GCC/MPI$

OpenMPI and Grid Engine integration

Configuration of consumable resources

roczei@gemaster:~$ qconf -se ogsexec1 | grep complex_values
complex_values        exclusive=true,h_vmem=4G,slots=4
roczei@gemaster:~$ qconf -se ogsexec2 | grep complex_values
complex_values        exclusive=true,h_vmem=4G,slots=4
roczei@gemaster:~$ qconf -se ogsexec3 | grep complex_values
complex_values        exclusive=true,h_vmem=4G,slots=4
roczei@gemaster:~$ qconf -se ogsexec4 | grep complex_values
complex_values        exclusive=true,h_vmem=4G,slots=4
roczei@gemaster:~$ qconf -sc | grep h_vmem
h_vmem              h_vmem     MEMORY      <=      YES         YES        1G     0
roczei@gemaster:~$ qconf -sc | grep exclusive
exclusive           excl       BOOL        EXCL    YES         YES        0        1000
roczei@gemaster:~$  clear

Configuration of the ARC client

roczei@gemaster:~/.arc$ cat client.conf
[common]

[computing/gemaster]
url=https://gemaster.grid.niif.hu:60000/arex
infointerface=org.nordugrid.wsrfglue2
submissioninterface=org.ogf.bes
default=yes
roczei@gemaster:~/.arc$

Test jobs

You can download the test files from here.

MPI

<JobDefinition
xmlns="http://schemas.ggf.org/jsdl/2005/11/jsdl"
xmlns:posix="http://schemas.ggf.org/jsdl/2005/11/jsdl-posix">

<JobDescription>
  <JobIdentification>
    <JobName>MPI test job</JobName>
  </JobIdentification>
  <Application>
    <posix:POSIXApplication>
      <posix:Executable>/scratch/openmpi/bin/mpirun</posix:Executable>
      <posix:Argument>-np</posix:Argument>
      <posix:Argument>16</posix:Argument>
      <posix:Argument>/scratch/arc/examples/mpi/connectivity</posix:Argument>
      <posix:Argument>-v</posix:Argument>
      <posix:Output>out.txt</posix:Output>
      <posix:Error>err.txt</posix:Error>
    </posix:POSIXApplication>
  </Application>
  <DataStaging>
    <FileName>out.txt</FileName>
    <DeleteOnTermination>false</DeleteOnTermination>
  </DataStaging>
  <DataStaging>
    <FileName>err.txt</FileName>
    <DeleteOnTermination>false</DeleteOnTermination>
  </DataStaging>
  <Resources>
      <SlotRequirement>
          <NumberOfSlots>16</NumberOfSlots>
       </SlotRequirement>
       <RunTimeEnvironment>
          <Software>
             <Name>ENV/GCC/MPI/OPENMPI</Name>
             <Version require="eq">1.6.3</Version>
          </Software>
       </RunTimeEnvironment>
  </Resources>
</JobDescription>
</JobDefinition>

OpenMP

<JobDefinition
xmlns="http://schemas.ggf.org/jsdl/2005/11/jsdl"
xmlns:posix="http://schemas.ggf.org/jsdl/2005/11/jsdl-posix">

<JobDescription>
  <JobIdentification>
    <JobName>OpenMP test job</JobName>
  </JobIdentification>
  <Application>
    <posix:POSIXApplication>
      <posix:Executable>/scratch/arc/examples/openmp/omp_mm</posix:Executable>
      <posix:Output>out.txt</posix:Output>
      <posix:Error>err.txt</posix:Error>
    </posix:POSIXApplication>
  </Application>
  <DataStaging>
    <FileName>out.txt</FileName>
    <DeleteOnTermination>false</DeleteOnTermination>
  </DataStaging>
  <DataStaging>
    <FileName>err.txt</FileName>
    <DeleteOnTermination>false</DeleteOnTermination>
  </DataStaging> 
  <Resources>
       <SlotRequirement>
          <NumberOfSlots>4</NumberOfSlots>
       </SlotRequirement>
       <RunTimeEnvironment>
          <Software>
             <Name>ENV/GCC/OPENMP</Name>
             <Version require="eq">2.5</Version>
          </Software>
       </RunTimeEnvironment>
  </Resources>
</JobDescription>
</JobDefinition>

Exclusive execution

<JobDefinition
xmlns="http://schemas.ggf.org/jsdl/2005/11/jsdl"
xmlns:posix="http://schemas.ggf.org/jsdl/2005/11/jsdl-posix">

<JobDescription>
  <JobIdentification>
    <JobName>Exclusive test job</JobName>
  </JobIdentification>
  <Application>
    <posix:POSIXApplication>
      <posix:Executable>/bin/sleep</posix:Executable>
      <posix:Argument>30</posix:Argument>
      <posix:Output>out.txt</posix:Output>
      <posix:Error>err.txt</posix:Error>
    </posix:POSIXApplication>
  </Application>
  <DataStaging>
    <FileName>out.txt</FileName>
    <DeleteOnTermination>false</DeleteOnTermination>
  </DataStaging>
  <DataStaging>
    <FileName>err.txt</FileName>
    <DeleteOnTermination>false</DeleteOnTermination>
  </DataStaging> 
  <Resources>
     <ExclusiveExecution>true</ExclusiveExecution>
  </Resources>
</JobDescription>
</JobDefinition>
Retrieved from "https://wiki.nbi.ku.dk/w/index.php?title=LRMS_Backends/Testbeds&oldid=6335"