#!/bin/bash
# setup_processor_group.sh
# To setup processor groups on Linux and Solaris.

CREATE_STX="-name <processor group name> -numa_nodes <num1[,num2[,...]]> | -cpus <num1|num1-num2[,...]>"
DELETE_STX="-name <processor group name>"

USAGE="Usage:
setup_processor_group.sh [command] [options]

Commands:
-prepare    Prepare the system for processor groups
             This sets up the system for processor groups usage and gives
             the relevant permissions to oracle user to be able to create
             and use the processor groups.
             This needs to be run as root.

-create     Create a processor group.
           Options syntax:
             $CREATE_STX

           -name <processor group name>
             This option specifies the processor group name.

           -numa_nodes <comma-separated list of NUMA node ids>
             This option specifies the NUMA nodes for this processor group.
             The processor group will include all CPUs from these NUMA nodes.
             Cannot be combined with -cpus option.

           -cpus <comma-separated list of CPU ids or CPU id ranges>
             This option specifies the CPUs for this processor group.  
             No numa affinity for the nodes is assigned for this
             processor group.
             Cannot be combined with -numa_nodes option.

-delete     Delete a processor group.
           Options syntax:
             $DELETE_STX

           -name <processor group name>
             This option specifies the processor group name.

-show       Print the CPU configuration and NUMA node configuration and the
           existing processor groups.

-help       Print this usage message.

Notes:
-----

Start with preparing the system for processor group setup by running as root:
$ setup_processor_group.sh -prepare

To look at the your current processor group configuration:
$ setup_processor_group.sh -show

To create a processor group pgroup23 with numa nodes 2 and 3 as follows:
$ setup_processor_group.sh -create -name pgroup23 -numa_nodes 2,3

To create a processor group pgroup1234 with cpus 1,2,3 and 4 as follows:
$ setup_processor_group.sh -create -name pgroup1234 -cpus 1,2,3,4

To delete a processor group:
$ setup_processor_group.sh -delete -name pgroup1234

To start an Oracle instance in the processor group pgroup23,
add the following init.ora parameter:
PROCESSOR_GROUP_NAME=pgroup23
"

PATH="/usr/gnu/bin:/bin:/usr/bin:/sbin:/usr/sbin"
export PATH

LINUX=1
SOLARIS=2

# os - The OS, 1:Linux, 2:Solaris
os=0

# USAGE: check_return expected_value exit_message
# Check the return value and exit on error.
function check_return
{
  RETURN_STATUS=$?
  EXP_VALUE=$1; shift
  EXIT_MESSAGE=$1; shift
  if [ "$RETURN_STATUS" -ne "$EXP_VALUE" ]
  then
    echo "Expected status $EXP_VALUE got ${RETURN_STATUS}."
    echo "$EXIT_MESSAGE"
    exit 1
  fi
}

# USAGE: syntax_message message command
# Print the syntax and exit.
function syntax_message
{
  MESSAGE=$1; shift
  COMMAND=$1; shift
  echo "$MESSAGE;"
  if [[ $COMMAND == "create" ]]
  then
          echo "-create command options syntax:"
          echo "$CREATE_STX"
  elif [[ $COMMAND == "delete" ]]
  then
          echo "-delete command options syntax:"
          echo "$DELETE_STX"
  fi
  exit "1"
}

# get_os - Get the OS, Linux or Solaris. 
function get_os()
{
  local os_string;

  os_string=`uname`;

  if [[ "$os_string" == "Linux" ]]; then
    os=$LINUX;
  else
    if [[ "$os_string" == "SunOS" ]]; then
      os=$SOLARIS;
    else
      echo "# Error:"
      echo "# OS: $os_string. This script only works on Linux and Solaris."
      exit
    fi 
  fi
}

# USAGE: solaris_prepare
function solaris_prepare
{
  return;
}

# USAGE: linux_prepare
function linux_prepare
{
  # create the cgroup directory and mount it.
  CGROUP_DIR=/dev/cgroup
  
  if [ ! -d "$CGROUP_DIR" ]
  then
    mkdir $CGROUP_DIR
    check_return 0 "Cannot make $CGROUP_DIR dir"
  fi

  # Change ownership to oracle user.
  chown -R oracle $CGROUP_DIR
  check_return 0 "Cannot change owner of $CGROUP_DIR dir to oracle"
  mount -t cgroup cpuset -ocpuset $CGROUP_DIR
  check_return 0 "Cannot mount $CGROUP_DIR dir"
}


# USAGE: solaris_create_nodes name node_list 
# node_list must be a comma seperated list
function solaris_create_nodes
{
  return;
}


# USAGE: linux_create_nodes name node_list 
# node_list must be a comma seperated list
function linux_create_nodes
{
    NAME=$1
    NODES=$2
    NODE_LIST=$(echo $NODES | awk -F"," '{$1=$1; print}')
    CGROUP_DIR=/dev/cgroup

    # cretae and mount the cgroup dir
    if [ ! -d "$CGROUP_DIR" ]
    then
       echo "/dev/cgroup not mounted"
       echo "Please run this script with -prepare option as root before doing this."
       exit "1"
    fi

    cd $CGROUP_DIR
    check_return 0 "Cannot cd to $CGROUP_DIR"

    if [ -d "$NAME" ]
    then
      echo "Processor group $NAME already exists. Delete it to create this."
      exit "1"
    else
      mkdir $NAME
      check_return 0 "Cannot make directory $CGROUP_DIR/$NAME"
      chown -R oracle $CGROUP_DIR
      check_return 0 "Cannot change owner of $CGROUP_DIR/$NAME dir to oracle"
    fi

    cd $NAME
    check_return 0 "Cannot cd to $CGROUP_DIR/$NAME"

    # collect the cpus for each node.
    for NODE in $NODE_LIST
    do
      NODE_CPULIST=`cat /sys/devices/system/node/node${NODE}/cpulist`;
      check_return 0 "Cannot get cpulist for node $NODE";
      if [ ! -n "$CPULIST" ]
      then
        CPULIST="$NODE_CPULIST";
      else
        CPULIST="$CPULIST,$NODE_CPULIST";
      fi
    done

    # Add the cpus and mems to the cgroup
    /bin/echo $CPULIST > cpuset.cpus
    check_return 0 "Cannot write to $CGROUP_DIR/$NAME/cpuset.cpus; Check for valid syntax or permissions."
    /bin/echo $NODES > cpuset.mems
    check_return 0 "Cannot write to $CGROUP_DIR/$NAME/cpuset.mems; Check for valid syntax or permissions."

    echo "Succesfully added CPUs $CPULIST and NUMA nodes $NODES to processor group $NAME"

}

# USAGE: solaris_create_cpus name cpu_list 
# node_list must be a comma seperated list of numbers or ranges
function solaris_create_cpus
{
  return;
}

# USAGE: linux_create_cpus name cpu_list 
# node_list must be a comma seperated list of numbers or ranges
function linux_create_cpus
{
    NAME=$1
    CPULIST=$2
    CGROUP_DIR=/dev/cgroup

    if [ ! -d "$CGROUP_DIR" ]
    then
       echo "/dev/cgroup not mounted"
       echo "Please run this script with -prepare option as root before doing this."
       exit "1"
    fi

    cd $CGROUP_DIR
    check_return 0 "Cannot cd to $CGROUP_DIR"
    if [ -d "$NAME" ]
    then
      echo "processor group $NAME already exists. Delete it to create this."
      exit "1"
    else
      mkdir $NAME
      check_return 0 "Cannot make directory $CGROUP_DIR/$NAME"
      chown -R oracle $CGROUP_DIR
      check_return 0 "Cannot change owner of $CGROUP_DIR/$NAME dir to oracle"
    fi

    cd $NAME
    check_return 0 "Cannot cd to $CGROUP_DIR/$NAME"

    # Add the cpus and mems to the cgroup
    /bin/echo $CPULIST > cpuset.cpus
    check_return 0 "Cannot write to $CGROUP/$NAME/cpuset.cpus; Check for valid syntax or permissions."
    NODES=`cat $CGROUP_DIR/cpuset.mems`
    /bin/echo $NODES > cpuset.mems
    check_return 0 "Cannot write to $CGROUP_DIR/$NAME/cpuset.mems; Check for valid syntax or permissions."

    echo "Succesfully added CPUs $CPULIST to processor group $NAME"
}

#USAGE solaris_delete name
function solaris_delete
{
  return;
}

#USAGE linux_delete name
function linux_delete
{
    NAME=$1
    CGROUP_DIR=/dev/cgroup/$NAME
    if [ ! -d "$CGROUP_DIR" ]
    then
      echo "processor group $NAME does not exist."
      exit "1"
    fi

    # deleting the cgroup is deleting the directory.
    rmdir $CGROUP_DIR
    check_return 0 "cannot rmdir $CGROUP_DIR; processor group $NAME cannot be deleted."
    echo "Succesfully deleted processor group $NAME"
}


# USAGE solaris_show
function solaris_show
{
  return;
}

# USAGE linux_show
function linux_show
{
  if [ -d "/sys/devices/system/node" ]
  then
    count=0
    echo "This is a NUMA system."
    echo ""

    # print NUMA configuration
    echo "NUMA configuration:"
    echo "-------------------"
    while [[ $count != "-1" ]]
    do
      if [ -d "/sys/devices/system/node/node$count" ]
      then
        echo "Node $count"
        cpus=`cat "/sys/devices/system/node/node$count/cpulist"`
        echo "  CPUs: $cpus"

        mem=`cat "/sys/devices/system/node/node$count/meminfo" |grep MemT | awk -F":" '{$1=$1; print}'`
        set $mem;
        total="$4 $5"
        mem=`cat "/sys/devices/system/node/node$count/meminfo" |grep MemF | awk -F":" '{$1=$1; print}'`
        set $mem;
        free="$4 $5"
        echo "  Memory Total: $total Free: $free"

        count=`expr $count + 1`
      else
        count=-1
      fi
    done
  else
    echo "This is not a NUMA system."
    echo ""
    CPU_LIST=`cat /proc/cpuinfo |grep processor |sed "s/processor//g" |sed "s/://g"`
    set $CPU_LIST;
    CPU_FIRST=$1;
    for CPU in $CPU_LIST
    do
      CPU_LAST=$CPU
    done
    echo "CPUs on this system:"
    echo "  $CPU_FIRST-$CPU_LAST"

    mem=`cat "/proc/meminfo" |grep MemT | awk -F":" '{$1=$1; print}'`
    set $mem;
    total="$2 $3"
    mem=`cat "/proc/meminfo" |grep MemF | awk -F":" '{$1=$1; print}'`
    set $mem;
    free="$2 $3"
    echo "  Memory Total: $total Free: $free"
  fi

  echo ""
  # check if cgroups are setup
  if [ -d "/dev/cgroup" ]
  then
    # check mounts
    mount=`mount|grep cgroup`
    set $mount
    if [[ $1 != "cpuset" ]]
    then
      echo "WARNING: This system is not prepared for processor groups."
      echo "WARNING: Run setup_processor_group.sh -prepare as root"
      return;
    fi

    # check permissions
    F=`ls -G -l -F /dev |grep '/' | grep cgroup`;set $F;
    if [[ $3 != "oracle" ]]
    then
      echo "WARNING: This system is not prepared for processor groups."
      echo "WARNING: Run setup_processor_group.sh -prepare as root"
      return;
    fi
   
    CG_LIST=$(echo `ls -F /dev/cgroup |grep '/'` | awk -F"/" '{$1=$1; print}')
    echo "Processor groups on this system:"
    echo "--------------------------------"
    for CG in $CG_LIST
    do
      echo "$CG"
      echo "  nodes: `cat /dev/cgroup/$CG/cpuset.mems`"
      echo "  cpus:  `cat /dev/cgroup/$CG/cpuset.cpus`"
    done
  else
    echo "WARNING: This system is not prepared for processor groups."
    echo "WARNING: Run setup_processor_group.sh -prepare as root"
  fi

}

# main
if [ $# -eq 0 ]
then
  echo "$USAGE"
  exit "1"
fi

get_os

CMD=$1; shift
case "$CMD" in
  -create)
    # -name <processor group name> -numa_nodes|-cpus <num1[,num2[...]]>

    while [ $# -ne 0 ]
    do
      OPTION="$1";shift
      case "$OPTION" in
        -name)
          NAME="$1";shift
          ;;
        -numa_nodes)
          NODES="$1";shift
          ;;
        -cpus)
          CPUS="$1";shift
          ;;
        *)
          syntax_message "Unexpected option $OPTION" $CMD
          ;;
      esac
    done

    if [[ -z $NAME ]]
    then
      syntax_message "-name option required" "create"
    fi

    if [[ -z $NODES && -z $CPUS ]]
    then
      syntax_message "-numa_nodes or -cpus option required" "create"
    fi

    if [[ -n $NODES && -n $CPUS ]]
    then
      syntax_message "Cannot specify both -numa_nodes and -cpus options" "create"
    fi

    if [[ -n $NODES ]]
    then
      NODE_LIST=$(echo $NODES | awk -F"," '{$1=$1; print}')
      #check the node list syntax
      for NODE in $NODE_LIST
      do
        if [[ $NODE != "${NODE//[^0-9]/}" ]]
        then
          syntax_message "-numa_nodes should be followed by a comma seperated list of numbers" "create"
        fi
      done

      if [[ "$os" == "$LINUX" ]]; then
        linux_create_nodes $NAME $NODES
      else
        solaris_create_nodes $NAME $NODES
      fi
    fi

    if [[ -n $CPUS ]]
    then
      CPU_LIST=$(echo $CPUS | awk -F"," '{$1=$1; print}')
      if [[ "$os" == "$LINUX" ]]; then
        linux_create_cpus $NAME $CPUS
      else
        solaris_create_cpus $NAME $CPUS
      fi
    fi

    echo "To start an Oracle instance in this processor group, add the following to the init.ora parameter file:"
    echo "PROCESSOR_GROUP_NAME=$NAME"
    
      ;;
  -delete)
    # -name <processor group name>"
    NBR_ARGS=2
    if [ $# -ne $NBR_ARGS ]
    then
      syntax_message "Wrong number of arguments" "delete"
    fi

    OPTION="$1";shift
    if [[ $OPTION != "-name" ]]
    then
      syntax_message "-name option must be specified" "delete"
    fi

    NAME="$1";shift
    if [[ "$os" == "$LINUX" ]]; then
      linux_delete $NAME
    else
      solaris_delete $NAME
    fi

    ;;
  -prepare)
    #Make sure it is run as root
    if [[ $EUID -ne 0 ]]; then
      echo "This command must be run as root"
      exit "1"
    fi

    if [[ "$os" == "$LINUX" ]]; then
      linux_prepare
    else
      solaris_prepare
    fi

    ;;
  -show)
    if [[ "$os" == "$LINUX" ]]; then
      linux_show
    else
      solaris_show
    fi
      ;;
  -help)
    echo "$USAGE"
      ;;
  *)
    echo "Unknown command $CMD"
    echo "$USAGE"
      ;;
esac

exit "0"