#!/bin/bash
### BEGIN INIT INFO
# Provides:          ovs-dpdk
# Required-Start:    $network $syslog
# Required-Stop:     $network $syslog
# Default-Start:     2 3 4 5
# Default-Stop:
# Short-Description: Open vSwitch DPDK
# Description:       Open vSwitch accelerate by DPDK
### END INIT INFO
# Starts and stops ovs with dpdk
#
echo "sourcing config"
source /etc/default/ovs-dpdk

BIND_PATH=${OVS_DPDK_DIR}/tools/dpdk-devbind.py
[ -e "$BIND_PATH" ] || BIND_PATH="${OVS_DPDK_DIR}/usertools/dpdk-devbind.py"


_XTRACE_OVS_DPDK_INIT=$(set +o | grep xtrace)
if [[ "${OVS_DPDK_SERVICE_DEBUG_OUTPUT}" == "True" ]]; then
    set -o xtrace
fi

is_ubuntu(){
    vendor=$(lsb_release -i -s 2> /dev/null)
    if [ "$vendor" == "Ubuntu" ]; then
        return 0
    else
        return 1
    fi
}

# Print line number and "message" then exits with return value from $1
# die "error_code" $LINENO "message"
die() {
    local exitcode=$1
    errXTRACE=$(set +o | grep xtrace)
    set +o xtrace
    local msg="[ERROR] ${BASH_SOURCE[2]}:$2 $3"
    echo $msg 1>&2;
    if [[ -n ${SCREEN_LOGDIR} ]]; then
        echo $msg >> "${SCREEN_LOGDIR}/error.log"
    fi
    $errXTRACE
    exit $exitcode
}

generate_pciwhitelist(){
    local _output=$1
    local MAPPINGS=${OVS_PCI_MAPPINGS//,/ }
    local ARRAY=( $MAPPINGS )
    local _Whitelist=''
    for pair in ${ARRAY[@]} ; do
        local addr=`echo $pair | cut -f 1 -d "#"`
        if [[ "$addr" =~ "0000:" ]]; then
            if [ "$_Whitelist" == '' ]; then
                _Whitelist="-w $addr"
            else
                _Whitelist="$_Whitelist -w $addr"
            fi
        fi
    done
    eval "$_output=\$_Whitelist"
}



init_db(){
    if [ ! -d $OVS_DB_CONF_DIR ]; then
        sudo mkdir -m 777 -p $OVS_DB_CONF_DIR
    elif [  -e $OVS_DB_CONF ]; then
        sudo rm -f $OVS_DB_CONF
    fi

    if [ ! -d $OVS_DB_SOCKET_DIR ]; then
        sudo mkdir -m 777 -p $OVS_DB_SOCKET_DIR
    fi

    if [ ! -d $OVS_LOG_DIR ]; then
        sudo mkdir -m 777 -p $OVS_LOG_DIR
    fi

    sudo ${OVS_INSTALL_DIR}/bin/ovsdb-tool create ${OVS_DB_CONF} ${OVS_INSTALL_DIR}/share/openvswitch/vswitch.ovsschema
    sudo ${OVS_INSTALL_DIR}/sbin/ovsdb-server  --detach --pidfile=${OVS_DB_SOCKET_DIR}/ovsdb-server.pid  --remote=punix:$OVS_DB_SOCKET --remote=db:Open_vSwitch,Open_vSwitch,manager_options
    sudo ovs-vsctl --db=unix:$OVS_DB_SOCKET --no-wait init
    sudo ovs-vsctl --no-wait set Open_vSwitch . other_config:pmd-cpu-mask=$OVS_PMD_CORE_MASK

    if [ "$OVS_INIT_POLICY" == "db" ]; then
        local pciAddressWhitelist
        generate_pciwhitelist pciAddressWhitelist
        sudo ovs-vsctl --no-wait set Open_vSwitch . other_config:pmd-cpu-mask=$OVS_PMD_CORE_MASK other_config:dpdk-init=True other_config:dpdk-lcore-mask=$OVS_CORE_MASK \
        other_config:dpdk-mem-channels=4 other_config:dpdk-socket-mem=$OVS_SOCKET_MEM other_config:dpdk-hugepage-dir=$OVS_HUGEPAGE_MOUNT  \
        other_config:dpdk-extra=" --proc-type primary $pciAddressWhitelist "
        [ -n "$OVS_VHOST_USER_SOCKET_DIR" ] && sudo ovs-vsctl --no-wait set Open_vSwitch . other_config:vhost-sock-dir=$OVS_VHOST_USER_SOCKET_DIR
    fi

}


restart_service(){
    sudo service $1 restart
}

remove_uio_pci_generic_module(){
    echo "Unloading uio_pci_generic UIO module"
    lsmod | grep -ws uio_pci_generic > /dev/null
    if [ $? -eq 0 ] ; then
        sudo modprobe -r uio_pci_generic
    fi
}

#
# Loads new uio_pci_generic.ko (and uio module if needed).
#
load_uio_pci_generic_module(){
    # UIO may be compiled into kernel, so it may not be an error if it can't
    # be loaded.

    echo "Loading uio_pci_generic UIO module"
    sudo modprobe uio_pci_generic
    if [ $? -ne 0 ] ; then
        die 1 $LINENO "## ERROR: Could not load uio_pci_generic."
    fi
}


remove_igb_uio_module(){
    echo "Unloading any existing DPDK UIO module"
    lsmod | grep -ws igb_uio > /dev/null
    if [ $? -eq 0 ] ; then
        sudo rmmod igb_uio
    fi
}

#
# Loads new igb_uio.ko (and uio module if needed).
#
load_igb_uio_module(){
    if [ ! -f $RTE_SDK/$RTE_TARGET/kmod/igb_uio.ko ];then
        echo "## ERROR: Target does not have the DPDK UIO Kernel Module."
        echo "       To fix, please try to rebuild target."
        return
    fi

    remove_igb_uio_module

    lsmod | grep -ws uio > /dev/null
    if [ $? -ne 0 ] ; then
        if  ls /lib/modules/$(uname -r)/kernel/drivers/uio/uio.* 1> /dev/null 2>&1 ; then
            echo "Loading uio module"
            sudo modprobe uio
        fi
    fi

    # UIO may be compiled into kernel, so it may not be an error if it can't
    # be loaded.

    echo "Loading DPDK UIO module"
    sudo insmod $RTE_SDK/$RTE_TARGET/kmod/igb_uio.ko
    if [ $? -ne 0 ] ; then
        die 1 $LINENO "## ERROR: Could not load kmod/igb_uio.ko."
    fi
}


remove_vfio_pci_module(){
    echo "Unloading vfio_pci module"
    lsmod | grep -ws vfio_pci > /dev/null
    if [ $? -eq 0 ] ; then
        sudo rmmod vfio_pci
    fi
}

#
# Loads new igb_uio.ko (and uio module if needed).
#
load_vfio_pci_module(){
    lsmod | grep -ws vfio_pci > /dev/null
    if [ ! $? -eq 0 ] ; then
        sudo modprobe vfio_pci
    fi
}

unbind_nics(){
    PCI_MAPPINGS=${OVS_PCI_MAPPINGS//,/ }
    PCI_ARRAY=( $PCI_MAPPINGS )

    # unbind nics from OVS_INTERFACE_DRIVER driver

    for pair in "${PCI_ARRAY[@]}"; do
        addr=`echo $pair | cut -f 1 -d "#"`

        echo "Removing driver_override from device $addr"
        echo | sudo tee /sys/bus/pci/devices/$addr/driver_override

        sudo ${BIND_PATH} -u $addr
        local interface=$(sudo ovs-vsctl --column name --bare find interface other_config:pci_address=$(echo $addr|  sed "s/:/\\\:/g" ))
        drv=$(sudo ovs-vsctl get interface $interface other_config:previous_driver)
        drv=$( echo $drv | sed "s/'//g" | sed "s/\"//g")
        sudo ${BIND_PATH} -b $drv $addr
    done

}


add_nics_to_bridges(){
    # bind nic to dpdk driver
    PCI_MAPPINGS=${OVS_PCI_MAPPINGS//,/ }
    PCI_ARRAY=( $PCI_MAPPINGS )

    local -A driver_map
    for pair in "${PCI_ARRAY[@]}"; do
        addr=`echo $pair | cut -f 1 -d "#"`
        nic=`echo $pair | cut -f 2 -d "#"`

        echo "Adding module ${OVS_INTERFACE_DRIVER} to driver_override for pci device $addr / $nic"
        echo "${OVS_INTERFACE_DRIVER}" | sudo tee /sys/bus/pci/devices/$addr/driver_override

        STATUS=$(${BIND_PATH} --status)
        while read line; do
            # Check if this physical nic it's using OVS_INTERFACE_DRIVER.
            if [[ "$line" =~ "unused="[[:graph:]]*"${OVS_INTERFACE_DRIVER}" ]] && [[ "$line" =~ "$addr" ]]; then
                # Check the status of the nic. If it's up, take it down.
                if [[ $line =~ "Active" ]]; then
                    sudo ifdown $nic || sudo ip link set dev $nic down
                fi
                oldDriver=$(echo $line | sed -r 's/[[:alnum:]]+=/\n&/g' | grep drv= | cut -d "=" -f 2)
                # Bind nic to OVS_INTERFACE_DRIVER.
                local cmd="${BIND_PATH} -b $OVS_INTERFACE_DRIVER $addr"
                echo $cmd
                sudo $cmd
                driver_map[$addr]="$oldDriver"

            fi
        done <<< "$STATUS"
    done


    # Add physical nics to bridges Keep the order of physical nic.
    phys_ofport=0
    STATUS=$(${BIND_PATH} --status)
    while read line; do
        for index in "${!PCI_ARRAY[@]}"; do
            tuple=${PCI_ARRAY[$index]}
            addr=`echo $tuple | cut -f 1 -d "#"`
            nic=`echo $tuple | cut -f 2 -d "#"`
            if [[ -z "$nic" ]]; then
                # If the nic interface is empty (the tuple in the array
                # is deleted), continue to the next register.
                continue
            fi
            bridge=$(find_bridge $nic $OVS_DPDK_PORT_MAPPINGS)
            if [[ "$line" =~ "drv=${OVS_INTERFACE_DRIVER}" ]] && [[ "$line" =~ "$addr" ]] && [[ "$bridge" != "" ]]; then
                # if nic is inside bond, it should be omitted from individual adding
                bonded_nic=False
                for k in "${!BONDS[@]}"; do
                    if [[ ${BONDS[$k]} =~ .*\($nic\).* ]]; then
                        bonded_nic=True
                        # replace nic with correct dpdk port name
                        BONDS[$k]=`printf '%s' "${BONDS[$k]}" | sed "s/${nic}/dpdk${phys_ofport}/"`
                    fi
                done
                if [ $bonded_nic == "True" ]; then
                    echo "interface $nic skipped from individual binding under dpdk$phys_ofport, it will be added inside bond"
                else
                    # Map this physical nic interface to the logical bridge.
                    local cmd="ovs-vsctl --no-wait --may-exist add-port $bridge dpdk$phys_ofport -- set Interface dpdk$phys_ofport type=dpdk other_config:pci_address=$addr  \
                                other_config:driver=${OVS_INTERFACE_DRIVER} other_config:previous_driver=${driver_map[$addr]}"
                    if [[ "$OVS_VSWITCHD_INIT" == "immediate" ]]; then
                        cmd="ovs-vsctl --may-exist add-port $bridge $nic -- set Interface $nic type=dpdk options:dpdk-devargs=$addr other_config:pci_address=$addr  \
                                other_config:driver=${OVS_INTERFACE_DRIVER} other_config:previous_driver=${driver_map[$addr]}"
                    fi
                    echo "add-port command: $cmd"
                    sudo $cmd
                fi
                # Remove element from list.
                PCI_ARRAY[$index]=''
                phys_ofport=$((phys_ofport+1))
            fi
        done
    done <<< "$STATUS"

    # TODO remove as this is nolonger needed for ci.
    # Add ports for virtual nic. Read non deleted elements from PCI_ARRAY.
    for tuple in "${PCI_ARRAY[@]}"; do
        if [[ ! -z "$tuple" ]]; then
            nic=`echo $tuple | cut -f 2 -d "#"`
            bridge=`echo $tuple | cut -f 3 -d "#"`
            check_nic_presence $nic
            echo "sudo ovs-vsctl --no-wait -- --may-exist add-port $bridge $nic"
            sudo ovs-vsctl --no-wait -- --may-exist add-port $bridge $nic
        fi
    done
}

setup_ovs(){
    # Loop over network definitions, create bridges
    # Extract nic name, bind it with OVS_INTERFACE_DRIVER driver, and add it to a bridge
    # dpdk sorts physical ports on pci addresses,
    # need to keep that order when adding phys ports to ovs
    add_bridges $OVS_BRIDGE_MAPPINGS
    if [ $OVS_DPDK_BIND_PORT == 'False' ]; then
        echo "Not binding nic to OVS_INTERFACE_DRIVER"
    else
        add_nics_to_bridges
    fi

}

generate_bond_array(){
    # bonds pre-processing
    declare -A BONDS
    local PORTS=${OVS_BOND_PORTS//,/ }
    local PORTS_ARRAY=( $PORTS )

    for pair in "${PORTS_ARRAY[@]}"; do
        name="${pair%%:*}"
        nic="${pair##*:}"
        if [[ ${BONDS[$name]} == "" ]]; then
            BONDS[$name]="($nic)"
        else
            BONDS[$name]=${BONDS[$name]},"($nic)"
        fi
    done
}

add_bonds() {
    generate_bond_array
    declare -A BONDS_MODE
    local MODES=${OVS_BOND_MODE//,/ }
    local MODES_ARRAY=( $MODES )

    for pair in "${MODES_ARRAY[@]}"; do
        name="${pair%%:*}"
        mode="${pair##*:}"
        if [[ ${BONDS_MODE[$name]} == "" ]]; then
            BONDS_MODE[$name]="$mode"
        fi
    done

    # Add ports for bonds using agregated nic skipped from adding earlier
    for k in "${!BONDS[@]}"; do
        bridge=$(echo $OVS_DPDK_PORT_MAPPINGS | sed -e "s/\(.*\)$k:\([^,]*\).*/\2/g")
        ports=${BONDS[$k]//,/ }
        ports_mdf=$(echo $ports | sed -e "s/(//g" | sed -e "s/)//g")
        ports_mdf_array=( $ports_mdf )
        args=""
        for port in ${ports_mdf_array[@]}; do
            args=$args" -- set Interface $port type=dpdk"
        done

        if [ ${BONDS_MODE[$k]} != "" ]; then
            mode=${BONDS_MODE[$k]}
            echo "sudo ovs-vsctl add-bond $bridge $k $ports_mdf bond_mode=$mode $args"
            sudo ovs-vsctl add-bond $bridge $k $ports_mdf bond_mode=$mode $args
        else
            echo "sudo ovs-vsctl add-bond $bridge $k $ports_mdf $args"
            sudo ovs-vsctl add-bond $bridge $k $ports_mdf $args
        fi
    done
}

remove_bonds() {
    generate_bond_array
    # remove bonds
    for k in "${!BONDS[@]}"; do
        echo "sudo ovs-vsctl del-port $bridge $k"
        sudo ovs-vsctl del-port $bridge $k
    done
}

check_nic_presence () {
    # This function will use the command "ip link" to find if the nic
    # name, passed as an argument, exists.
    # List computer interfaces.
    interfaces=$(ip link)
    if [[ "$interfaces" =~ ": $nic: " ]]; then
        return 0
    fi
    die 2 $LINENO "The nic interface $nic is not present in the system"
}


find_bridge() {
    # Check argument is supplied.
    if [[ -z "$1" ]]; then
        die 3 $LINENO "No argument supplied, nic expected"
    fi
    if [[ -z "$2" ]]; then
        # OVS_DPDK_PORT_MAPPINGS not present, use the nic interface name
        # to build the bridge name.
        echo "br-"$1
        return 0
    fi

    # Loop OVS_DPDK_PORT_MAPPINGS tuple list to find the pairing bridge.
    NIC_MAPPINGS=${2//,/ }
    NIC_ARRAY=( $NIC_MAPPINGS )
    for pair in "${NIC_ARRAY[@]}"; do
        nic=`echo $pair | cut -f 1 -d ":"`
        if [[ "$nic" == "$1" ]]; then
            bridge=`echo $pair | cut -f 2 -d ":"`
            echo $bridge
            return 0
        fi
    done
    echo ""
}


add_bridges() {
    # Check argument is supplied.
    if [[ -z "$1" ]]; then
        die 4 $LINENO "No argument supplied, OVS_BRIDGE_MAPPINGS expected"
    fi

    # Adds the OVS bridge, given the mapping list OVS_BRIDGE_MAPPINGS.
    NIC_MAPPINGS=${1//,/ }
    NIC_ARRAY=( $NIC_MAPPINGS )
    for pair in "${NIC_ARRAY[@]}"; do
        bridge=`echo $pair | cut -f 2 -d ":"`
        sudo ovs-vsctl --no-wait -- --may-exist add-br $bridge -- set Bridge $bridge datapath_type=netdev
    done
}


free_hugepages() {
    HUGEPAGE_SIZE=$(grep Hugepagesize /proc/meminfo | awk '{ print $2 }')
    grep -ws $OVS_HUGEPAGE_MOUNT /proc/mounts > /dev/null
    if [ $? -ne 0 ]; then
       echo "Hugepages not mounted, nothing to clean"
       return 0
    fi
    #remove ovs reserved hugepages
    if [ -d $OVS_HUGEPAGE_MOUNT ]; then
       sudo rm -rf ${OVS_HUGEPAGE_MOUNT}/rtemap*
    fi

    #unmount ovs mountpoint
    sudo umount ${OVS_HUGEPAGE_MOUNT}

    # de-allocate hugepages
    if [ $OVS_ALLOCATE_HUGEPAGES == 'True' ]; then
       for d in /sys/devices/system/node/node? ; do
          echo 0 | sudo tee $d/hugepages/hugepages-${HUGEPAGE_SIZE}kB/nr_hugepages
       done
    fi

    if is_ubuntu; then
        restart_service libvirt-bin
    else
        restart_service libvirtd
    fi


}

alloc_hugepages() {
    HUGEPAGE_SIZE=$(grep Hugepagesize /proc/meminfo | awk '{ print $2 }')
    sudo mkdir -p $OVS_HUGEPAGE_MOUNT

    if [ $OVS_NUM_HUGEPAGES -eq 0 ]; then
        die 6 $LINENO "OVS_NUM_HUGEPAGES not set"
    fi
    grep -ws $OVS_HUGEPAGE_MOUNT /proc/mounts > /dev/null

    if [ $? -eq 0 ]; then
        free_hugepages
    fi
    #allocate hugepages
    if [ $OVS_ALLOCATE_HUGEPAGES == 'True' ]; then
        for d in /sys/devices/system/node/node? ; do
            echo $OVS_NUM_HUGEPAGES | sudo tee $d/hugepages/hugepages-${HUGEPAGE_SIZE}kB/nr_hugepages
        done
    fi

    if is_ubuntu; then
        qemu_user_id=$(id -u libvirt-qemu)
        qemu_group_id=$(id -g libvirt-qemu)
    else
        qemu_user_id=$(id -u qemu)
        qemu_group_id=$(id -g qemu)
    fi

    grep -ws $OVS_HUGEPAGE_MOUNT /proc/mounts > /dev/null
    if [ $? -ne 0 ] ; then
        if [ -n "$OVS_HUGEPAGE_MOUNT_PAGESIZE" ]; then
            sudo mount -t hugetlbfs -o uid=$qemu_user_id,gid=$qemu_group_id,pagesize=$OVS_HUGEPAGE_MOUNT_PAGESIZE  nodev $OVS_HUGEPAGE_MOUNT
        else
            sudo mount -t hugetlbfs -o uid=$qemu_user_id,gid=$qemu_group_id  nodev $OVS_HUGEPAGE_MOUNT
        fi
    fi
    if is_ubuntu; then
        restart_service libvirt-bin
    else
        restart_service libvirtd
    fi
}

get_ovs_status(){
    result=0
    pids_files=( "$OVS_DB_SOCKET_DIR/ovs-vswitchd.pid"  "$OVS_DB_SOCKET_DIR/ovsdb-server.pid" )
    for file in ${pids_files[@]}
    do
        if [ ! -e $file ] || [ ! -e "/proc/`cat $file`" ] ;then
            echo "$file is not running"
            result=$((result+1))
        fi
    done
    if [[ "$result" == "0" ]]; then
        echo "ovs alive"
        tail --lines 20 $OVS_LOG_DIR/ovs-vswitchd.log
    elif [[ "$result" == "1" ]]; then
        echo "Not all processes are running restart!!!"
    fi
    echo $result
    return $result

}

stop_ovs_deamon(){
    sudo kill -9 $(cat $OVS_DB_SOCKET_DIR/ovs-vswitchd.pid)
    sudo rm -f $OVS_DB_SOCKET_DIR/ovs-vswitchd.pid
}

stop_ovsdb(){
    sudo kill -9 $(cat $OVS_DB_SOCKET_DIR/ovsdb-server.pid)
    sudo rm -f $OVS_DB_SOCKET_DIR/ovsdb-server.pid
}

start_ovsdb(){
    if [ ! -d $OVS_DB_SOCKET_DIR ]; then
        sudo mkdir -m 777 -p $OVS_DB_SOCKET_DIR
    fi


    if [ ! -e $OVS_DB_SOCKET_DIR/ovsdb-server.pid ]; then
        sudo ${OVS_INSTALL_DIR}/sbin/ovsdb-server  --detach --pidfile=$OVS_DB_SOCKET_DIR/ovsdb-server.pid  --remote=punix:$OVS_DB_SOCKET --remote=db:Open_vSwitch,Open_vSwitch,manager_options
    fi
}

start_ovs_vswitchd(){
    MAPPINGS=${OVS_BRIDGE_MAPPINGS//,/ }
    ARRAY=( $MAPPINGS )
    PHYS_PORTS=${#ARRAY[@]}

    sudo rm -f $OVS_LOG_DIR/ovs-vswitchd.log

    qemu_group="qemu"
    if is_ubuntu; then
        qemu_group="kvm"
    fi
    [ -e "${OVS_DB_SOCKET_DIR}/${OVS_VHOST_USER_SOCKET_DIR}" ] ||  sudo sg $qemu_group -c "mkdir -m 777 -p ${OVS_DB_SOCKET_DIR}/${OVS_VHOST_USER_SOCKET_DIR}"
    # the permeisions must be adjusted to allow qemu to create vhost-user sockets when in server mode.
    sudo chown root:$qemu_group ${OVS_DB_SOCKET_DIR}
    sudo chmod 777 ${OVS_DB_SOCKET_DIR}
    start_cmd="screen -dms ovs-vswitchd sudo sg $qemu_group -c \"umask 002; ${OVS_INSTALL_DIR}/sbin/ovs-vswitchd --pidfile=${OVS_DB_SOCKET_DIR}/ovs-vswitchd.pid -- unix:$OVS_DB_SOCKET 2>&1 | tee ${OVS_LOG_DIR}/ovs-vswitchd.log\""
    if [ "$OVS_INIT_POLICY" == "cmd" ]; then
        local pciAddressWhitelist
        generate_pciwhitelist pciAddressWhitelist
        start_cmd="screen -dms ovs-vswitchd sudo sg $qemu_group -c \"umask 002; ${OVS_INSTALL_DIR}/sbin/ovs-vswitchd --dpdk -vhost_sock_dir ${OVS_DB_SOCKET_DIR}/${OVS_VHOST_USER_SOCKET_DIR} -c $OVS_CORE_MASK -n $OVS_MEM_CHANNELS  --proc-type primary  --huge-dir $OVS_HUGEPAGE_MOUNT --socket-mem $OVS_SOCKET_MEM $pciAddressWhitelist -- unix:$OVS_DB_SOCKET --pidfile=${OVS_DB_SOCKET_DIR}/ovs-vswitchd.pid 2>&1 | tee ${OVS_LOG_DIR}/ovs-vswitchd.log\""
    fi


    if [ -n "$OVS_LOCK_DIR" ]; then
        retry_count=1
        max_retry_count=60
        while [ "$retry_count" -le "$max_retry_count" ]; do
            if [[ ! -d $OVS_LOCK_DIR ]]; then
                sudo mkdir $OVS_LOCK_DIR
                if [ $? == 0 ]; then
                    echo "start ovs-vswitchd $retry_count/$max_retry_count"
                    echo $start_cmd; eval $start_cmd
                    break
                fi
            fi
            let retry_count="$retry_count+1"
            sleep 10
        done
    else
        echo $start_cmd; eval $start_cmd
    fi

    retry_count=1
    max_retry_count=60
    while [ "$retry_count" -le "$max_retry_count" ]; do
        PID=$(pidof ovs-vswitchd)
        ret=$?
        if [[ ${ret} -ne 0 ]]; then
            echo "Waiting for PID of ovs-vswitchd $retry_count/$max_retry_count"
        else
            echo "PID obtained ${PID}"
            break
        fi
        let retry_count="$retry_count+1"
        sleep 1
    done
    if [ ${ret} -eq 0 ]; then
        echo "ovs-vswitchd started with $PID"
    else
        free_hugepages
        if [ -n "$OVS_LOCK_DIR" ]; then
            sudo rm -rf $OVS_LOCK_DIR
        fi
        die 7 $LINENO "ovs-vswitchd application failed to start $PID"
    fi
    while [ $(grep -c "unix.*connected" ${OVS_LOG_DIR}/ovs-vswitchd.log) -eq 0 ]; do
        PID=$(pidof ovs-vswitchd)
        if [ $? -eq 0 ]; then
            echo "Waiting for ovs-vswitchd to start..."
            sleep 1
        else
            free_hugepages
            if [ -n "$OVS_LOCK_DIR" ]; then
                sudo rm -rf $OVS_LOCK_DIR
            fi
            die 8 $LINENO "ovs-vswitchd application failed to start $PID"
        fi
    done
    sudo rm -rf $OVS_LOCK_DIR

}

add_cidr(){
    if [[ ! -z "$OVS_TUNNEL_CIDR_MAPPING" ]]; then
        bridge=`echo $OVS_TUNNEL_CIDR_MAPPING | cut -f 1 -d ":"`
        cidr=`echo $OVS_TUNNEL_CIDR_MAPPING | cut -f 2 -d ":"`
        counter=0
        ip link show  $bridge
        while [ $? -ne 0 -a $counter -lt 120 ]; do
            echo "Waiting for $bridge interface to be created...."
            sleep 1
            counter=$((counter+1))
            ip link show $bridge
        done
        if [[ $counter -ge 120 ]]; then
            die 9 $LINENO "Interface $bridge not created!"
        fi
        sudo ip addr add $cidr dev $bridge
        sudo ip link set $bridge up
    fi
}

start_ovs(){
    # TODO remove as this is nolonger needed for ci.
    sudo modprobe veth

    start_ovsdb
    if [[ "$OVS_VSWITCHD_INIT" == "immediate" ]]; then
        start_ovs_vswitchd
        setup_ovs
    else
        setup_ovs
        start_ovs_vswitchd
    fi
    add_cidr
    # WA: adding bonds -  it's failing to configure bonds before ovs-vswitchd startup
    add_bonds

}

cmd_start(){
    get_ovs_status
    ret=$?
    if [[ $ret -eq 0 ]]; then
        echo Accelerated ovs already started
        return 0
    fi
    #if huge pages are not mounted allocate hugepages
    echo "mounting hugepages"
    alloc_hugepages
    #if uio diver is not loaded load
    echo "loading OVS_INTERFACE_DRIVER diver"
    if [[ "$OVS_INTERFACE_DRIVER" == "igb_uio" ]]; then
        load_igb_uio_module
    elif [[ "$OVS_INTERFACE_DRIVER" == "vfio-pci" ]]; then
        load_vfio_pci_module
    elif [[ "$OVS_INTERFACE_DRIVER" == "uio_pci_generic" ]]; then
        load_uio_pci_generic_module
    fi

    # store pid of each process in $OVS_LOG_DIR/*
    echo "starting ovs db"
    echo "binding nics"
    echo "starting vswitchd"
    start_ovs
}

cmd_stop(){
    remove_bonds

    #if switch is stopped no op/error message
    #else
    # retrive pid of each process in $OVS_LOG_DIR/*
    echo "stopping vswitchd"
    stop_ovs_deamon

    #if physical nics bindings are defined, bind nics with linux driver
    echo "rebinding nics to linux_driver"
    unbind_nics

    echo "stopping ovs db"
    stop_ovsdb

    echo "unloading OVS_INTERFACE_DRIVER"
    if [[ "$OVS_INTERFACE_DRIVER" == "igb_uio" ]]; then
        remove_igb_uio_module
    elif [[ "$OVS_INTERFACE_DRIVER" =~ "vfio-pci" ]]; then
        remove_vfio_pci_module
    elif [[ "$OVS_INTERFACE_DRIVER" =~ "uio_pci_generic" ]]; then
        remove_uio_pci_generic_module
    fi

    echo "unmounting hugepages"
    free_hugepages

}

cmd_status(){
    get_ovs_status
}


cmd_init(){
    init_db
}

case "$1" in
    start)
        cmd_start
    ;;

    stop)
        cmd_stop
    ;;

    restart)
        cmd_stop
        cmd_start
    ;;

    status)
        cmd_status
    ;;

    init)
        cmd_init
    ;;

    *)
        echo "Usage: $0 {start|stop|restart|status|init}"
        ${_XTRACE_OVS_DPDK_INIT}
        exit 1
esac

${_XTRACE_OVS_DPDK_INIT}
