#!/bin/bash
#
# common functions for ovs based plugin
# -------------------------------------

# Save trace setting
OVSB_XTRACE=$(set +o | grep xtrace)
set +o xtrace



function start_ovs_dpdk {
    if  [ -e /etc/init.d/ovs-dpdk ]; then
        sudo service ovs-dpdk start
    fi
}

function stop_ovs_dpdk {
    if  [ -e /etc/init.d/ovs-dpdk ]; then
        sudo service ovs-dpdk stop
    fi
}



function ovs_dpdk_configure_bridge_datapath {
    local bridge=$1
    if [[ $OVS_DATAPATH_TYPE != "" ]]; then
        sudo ovs-vsctl --no-wait set Bridge $bridge datapath_type=${OVS_DATAPATH_TYPE}
    fi
}

function ovs_dpdk_add_bridge {
    local bridge=$1
    sudo ovs-vsctl --no-wait -- --may-exist add-br $bridge
    ovs_dpdk_configure_bridge_datapath $bridge
    sudo ovs-vsctl --no-wait br-set-external-id $bridge bridge-id $bridge
}


function ovs_dpdk_setup_bridge {
    local bridge=$1
    neutron-ovs-cleanup
    ovs_dpdk_add_bridge $bridge
}

function uninstall_libvirt_CentOS {

    if [ "$os_VENDOR" == CentOS ] ; then

        sudo rm /usr/libexec/qemu-kvm
        sudo rm /usr/libexec/qemu-kvm.orig

        sudo yum remove -y libvirt* qemu* libcard* glusterfs*
    fi
}

function install_qemu_kvm {

    if [ "$os_VENDOR" == CentOS ] ; then
        echo "Configuring libvirt for CentOS"
        sudo service libvirtd stop

        sudo yum remove -y libvirt* qemu* libcacard* glusterfs*

        if [ ! -d /opt/stack/rpms ]; then
            sudo mkdir -m 777 -p /opt/stack/rpms
        fi

        RPMS=/opt/stack/rpms

        wget_cmd=" wget -nc -P"

        if [ "$RECLONE" == True ] ; then
            sudo rm -f $RPMS/*
            wget_cmd=" wget -P"
        fi

        for file in ${CentOS_libvirt_repo[@]} ; do
            $wget_cmd $RPMS $file
        done

        for file in ${CentOS_qemu_repo[@]} ; do
            $wget_cmd $RPMS $file
        done

        sudo yum install -y $RPMS/*

        sudo service libvirtd start
    else
        install_package qemu-kvm
    fi

}



function ovs_dpdk_create_kvm_wrapper(){

    install_qemu_kvm

    if [ -e /usr/bin/kvm ]; then
        KVM_CMD="/usr/bin/kvm"
    elif [ -e /usr/bin/qemu-kvm ]; then
        KVM_CMD="/usr/bin/qemu-kvm"
    elif [ -e /usr/libexec/qemu-kvm ]; then
        KVM_CMD="/usr/libexec/qemu-kvm"

    fi

    sudo mv $KVM_CMD $KVM_CMD.orig

cat << 'EOF' | sudo tee $KVM_CMD
#!/bin/bash -
VIRTIO_OPTIONS="csum=off,gso=off,guest_tso4=off,guest_tso6=off,guest_ecn=off"
VHOST_FORCE="vhostforce"
SHARE="share=on"
add_mem=False
i=0
while [ $# -gt 0 ]; do
     case "$1" in

     -device)
        args[i]="$1"
        (( i++ ))
        shift
        if [[ $1 =~ "vhost-user" ]]
        then
                args[i]=${1},${VHOST_FORCE}
                (( i++))
                shift

        fi
        if [[ $1 == virtio-net-pci* ]]; then
                args[i]=${1},${VIRTIO_OPTIONS}
                (( i++))
                shift

        fi
        ;;
    -object)
        args[i]="$1"
        (( i++ ))
        shift
        if [[ $1 =~ "memory-backend-file" ]]
        then
                args[i]=${1},${SHARE}
                (( i++))
                shift

        fi
        ;;

     *)
         args[i]="$1"
         (( i++ ))
         shift ;;
     esac
done
echo "qemu ${args[@]}"  > /tmp/qemu.orig
if [ -e /usr/bin/qemu-system-x86_64 ]; then
    exec /usr/bin/qemu-system-x86_64  "${args[@]}"
elif [ -e /usr/libexec/qemu-kvm.orig ]; then
    exec /usr/libexec/qemu-kvm.orig  "${args[@]}"
fi
EOF

    sudo chmod +x $KVM_CMD
}


function ovs_dpdk_db_cleanup {
    # remove all OVS ports that look like Neutron created ports
    for port in $(sudo ovs-vsctl list port | grep -o -e tap[0-9a-f\-]* -e q[rg]-[0-9a-f\-]*); do
        sudo ovs-vsctl --no-wait del-port ${port}
    done

    # remove all OVS bridges created by Neutron
    for bridge in $(sudo ovs-vsctl list-br | grep -o -e ${OVS_BRIDGE} -e ${PUBLIC_BRIDGE}); do
        sudo ovs-vsctl --no-wait del-br ${bridge}
    done
    if [ -e /usr/bin/kvm ]; then
        KVM_CMD="/usr/bin/kvm"
    elif [ -e /usr/bin/qemu-kvm ]; then
        KVM_CMD="/usr/bin/qemu-kvm"
    fi
    if [ -e $KVM_CMD.orig ]; then
        sudo mv $KVM_CMD.orig $KVM_CMD
    fi
}

function ovs_dpdk_clean(){
    sudo rm -f /usr/bin/ovs*
    sudo rm -f $OVS_DB_CONF_DIR/*
    sudo rm -f $OVS_DB_SOCKET_DIR/*
    sudo rm -f ${OVS_DIR}/BUILD_COMPLETE
}


function clone_ovs_dpdk(){
    if "$OFFLINE" != True && [ "$RECLONE" = True  ||  ! -e ${OVS_DIR}/BUILD_COMPLETE ]; then
        if [ ! -d ${OVS_DIR} ] || [ "$RECLONE" = True ]; then
            if [ -d ${OVS_DIR} ]; then
                rm -rf ${OVS_DIR}
            fi
            if [[ $OVS_GIT_TAG != "" ]]; then
                git_clone ${OVS_GIT_REPO} ${OVS_DIR} $OVS_GIT_TAG
            else
                git_clone ${OVS_GIT_REPO} ${OVS_DIR}
            fi
        fi

        if [ ! -d "${OVS_DPDK_DIR}" ] || [ "$RECLONE" = True ]; then
            git_clone ${OVS_DPDK_GIT_REPO} ${OVS_DPDK_DIR} ${OVS_DPDK_GIT_TAG}
        fi
        if [[ '' != $OVS_PATCHES ]]; then
            local patches=( $OVS_PATCHES )
            pushd  ${OVS_DIR}
            git clean -f -x -d
            git reset --hard $OVS_GIT_TAG
            for url in "${patches[@]}"; do
                curl $url | patch -p1
            done
            popd
        fi
        if [[ '' != $OVS_DPDK_PATCHES ]]; then
            local patches=( $OVS_DPDK_PATCHES )
            pushd  ${OVS_DPDK_DIR}
            git clean -f -x -d
            git reset --hard ${OVS_DPDK_GIT_TAG}
            for url in "${patches[@]}"; do
                curl $url | patch -p1
            done
            popd
        fi
    fi
}

function build_ovs_dpdk(){
   if [  -e ${OVS_DIR}/BUILD_COMPLETE ]; then
       echo "Build alread done."
       cd ${OVS_DIR}
       sudo make install
       return
   fi
   cd ${OVS_DPDK_DIR}
   make config T=${RTE_TARGET}
   if [ -e ${OVS_DPDK_DIR}/${RTE_TARGET} ]; then
      rm $RTE_TARGET
   fi
   ln -s -f build $RTE_TARGET
   sed "s/CONFIG_RTE_BUILD_COMBINE_LIBS=n/CONFIG_RTE_BUILD_COMBINE_LIBS=y/" -i ${OVS_DPDK_DIR}/build/.config
   sed "s/CONFIG_RTE_MAX_MEMSEG=.*$/CONFIG_RTE_MAX_MEMSEG=${OVS_DPDK_MEM_SEGMENTS}/" -i ${OVS_DPDK_DIR}/build/.config
   sed "s/CONFIG_RTE_LIBRTE_VHOST=.*$/CONFIG_RTE_LIBRTE_VHOST=${OVS_DPDK_RTE_LIBRTE_VHOST}/" -i ${OVS_DPDK_DIR}/build/.config
   sed "s/CONFIG_RTE_LIBRTE_VHOST_DEBUG=.*$/CONFIG_RTE_LIBRTE_VHOST_DEBUG=${OVS_DPDK_VHOST_USER_DEBUG}/" -i ${OVS_DPDK_DIR}/build/.config

   make -j `nproc`
   sudo cp ${OVS_DPDK_DIR}/build/lib/*dpdk.* /lib
   cd ${OVS_DIR}
   ./boot.sh
   ./configure --with-dpdk=${OVS_DPDK_DIR}/${RTE_TARGET} --prefix=/usr --with-rundir=$OVS_DB_SOCKET_DIR
   make -j `nproc` CFLAGS='-O3 -march=native'
   touch ${OVS_DIR}/BUILD_COMPLETE
   sudo make install
}

function ovs_dpdk_write_conf {
    sudo cp $NETWOKING_OVS_DPDK_DIR/devstack/ovs-dpdk/ovs-dpdk-conf /etc/default/ovs-dpdk

    sudo sed "s#RTE_SDK=.*#RTE_SDK=$OVS_DPDK_DIR#" -i /etc/default/ovs-dpdk
    sudo sed "s#RTE_TARGET=.*#RTE_TARGET=$RTE_TARGET#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_SRC_DIR=.*#OVS_SRC_DIR=$OVS_DIR#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_NUM_HUGEPAGES=.*#OVS_NUM_HUGEPAGES=$OVS_NUM_HUGEPAGES#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_HUGEPAGE_MOUNT=.*#OVS_HUGEPAGE_MOUNT=$OVS_HUGEPAGE_MOUNT#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_HUGEPAGE_MOUNT_PAGESIZE=.*#OVS_HUGEPAGE_MOUNT_PAGESIZE=$OVS_HUGEPAGE_MOUNT_PAGESIZE#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_DB_SOCKET_DIR=.*#OVS_DB_SOCKET_DIR=$OVS_DB_SOCKET_DIR#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_DB_CONF_DIR=.*#OVS_DB_CONF_DIR=$OVS_DB_CONF_DIR#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_DB_SOCKET=.*#OVS_DB_SOCKET=$OVS_DB_SOCKET#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_DB_CONF=.*#OVS_DB_CONF=$OVS_DB_CONF#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_SOCKET_MEM=.*#OVS_SOCKET_MEM=$OVS_SOCKET_MEM#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_MEM_CHANNELS=.*#OVS_MEM_CHANNELS=$OVS_MEM_CHANNELS#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_CORE_MASK=.*#OVS_CORE_MASK=$OVS_CORE_MASK#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_PMD_CORE_MASK=.*#OVS_PMD_CORE_MASK=$OVS_PMD_CORE_MASK#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_ALLOCATE_HUGEPAGES=.*#OVS_ALLOCATE_HUGEPAGES=$OVS_ALLOCATE_HUGEPAGES#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_LOG_DIR=.*#OVS_LOG_DIR=$OVS_LOG_DIR#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_LOCK_DIR=.*#OVS_LOCK_DIR=$OVS_LOCK_DIR#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_INTERFACE_DRIVER=.*#OVS_INTERFACE_DRIVER=$OVS_INTERFACE_DRIVER#" -i /etc/default/ovs-dpdk

    if [ $OVS_SOCKET_MEM == "auto" ] ; then
        for d in /sys/devices/system/node/node? ; do
            if [ $OVS_SOCKET_MEM == "auto" ]; then
                    OVS_SOCKET_MEM=2048
            else
                    OVS_SOCKET_MEM=$OVS_SOCKET_MEM,2048
            fi
        done
    fi

    sudo sed -e "s#OVS_SOCKET_MEM=.*#OVS_SOCKET_MEM=$OVS_SOCKET_MEM#g" -i /etc/default/ovs-dpdk
    # creates an array of pci addres to interface names delimeted by # e.g. <pci_address>#<interface name>
    PAIRS=( `ls -al /sys/class/net/* | grep pci | awk '{print $NF;}' | awk '{n=split($0,a,"/");print a[n-2] "#" a[n] }'` )
    # populates OVS_BRIDGE_MAPPINGS if $PHYSICAL_NETWORK and $OVS_PHYSICAL_BRIDGE are used instead.
    if [[ "$OVS_DATAPATH_TYPE" != "" ]] && [[ "$OVS_BRIDGE_MAPPINGS" = "" ]] && [[ "$PHYSICAL_NETWORK" != "" ]] && [[ "$OVS_PHYSICAL_BRIDGE" != "" ]]; then
        OVS_BRIDGE_MAPPINGS=$PHYSICAL_NETWORK:$OVS_PHYSICAL_BRIDGE
    fi
    MAPPINGS=${OVS_BRIDGE_MAPPINGS//,/ }
    ARRAY=( $MAPPINGS )
    NICS=""
    for net in "${ARRAY[@]}"; do
         KEY="${net%%:*}"
         VALUE="${net##*:}"
         printf "%s in %s\n" "$KEY" "$VALUE"
         nic=${VALUE/br-/}
         for pair in "${PAIRS[@]}"; do
            if [[ $nic == `echo $pair | cut -f 2 -d "#"` ]]; then
                if [[ $NICS == "" ]]; then
                    NICS=$pair
                else
                    NICS=$NICS,$pair
                fi
            fi
         done
    done

    sudo sed "s/OVS_BRIDGE_MAPPINGS=.*/OVS_BRIDGE_MAPPINGS=$NICS/" -i /etc/default/ovs-dpdk

}

function install_ovs_dpdk {
    # Install deps
    set +o errexit
    if is_ubuntu; then
        stop_service openvswitch-switch
        uninstall_package openvswitch-switch openvswitch-datapath-dkms openvswitch-common
        install_package autoconf libtool libfuse-dev
    else
        stop_service openvswitch
        uninstall_package openvswitch
        install_package pciutils autoconf libtool fuse-devel
    fi
    # This function exits on an error so that errors don't compound and you see
    # only the first error that occurred.
    sudo rmmod openvswitch
    set -o errexit

    build_ovs_dpdk

    sudo cp $NETWOKING_OVS_DPDK_DIR/devstack/ovs-dpdk/ovs-dpdk-init /etc/init.d/ovs-dpdk
    ovs_dpdk_write_conf
    ovs_dpdk_create_kvm_wrapper

    sudo chmod +x /etc/init.d/ovs-dpdk
    sudo service ovs-dpdk init
}


function ovs_dpdk_configure_debug_command {
    if [ "$Q_USE_PROVIDERNET_FOR_PUBLIC" = "True" ]; then
        iniset $NEUTRON_TEST_CONFIG_FILE DEFAULT external_network_bridge ""
    else
        iniset $NEUTRON_TEST_CONFIG_FILE DEFAULT external_network_bridge $PUBLIC_BRIDGE
    fi
}

function ovs_dpdk_configure_firewall_driver {
    iniset /$Q_PLUGIN_CONF_FILE securitygroup firewall_driver neutron.agent.firewall.NoopFirewallDriver
}

function ovs_dpdk_configure_l3_agent {
    if [ "$Q_USE_PROVIDERNET_FOR_PUBLIC" = "True" ]; then
        iniset $Q_L3_CONF_FILE DEFAULT external_network_bridge ""
    else
        iniset $Q_L3_CONF_FILE DEFAULT external_network_bridge $PUBLIC_BRIDGE
    fi
    if [[ "$Q_USE_PUBLIC_VETH" = "True" ]]; then
        ip link show $Q_PUBLIC_VETH_INT > /dev/null 2>&1 ||
        sudo ip link add $Q_PUBLIC_VETH_INT type veth peer name $Q_PUBLIC_VETH_EX
        sudo ip link set $Q_PUBLIC_VETH_INT up
        sudo ip link set $Q_PUBLIC_VETH_EX up
        sudo ip addr flush dev $Q_PUBLIC_VETH_EX
    else
        ovs_dpdk_add_bridge $PUBLIC_BRIDGE
    fi
}

function set_vcpu_pin_set {
    OVS_CORE_MASK=$(echo $OVS_CORE_MASK | sed 's/^0x//')
    OVS_PMD_CORE_MASK=$(echo $OVS_PMD_CORE_MASK | sed 's/^0x//')
    BAD_CORES=$((`echo $((16#${OVS_CORE_MASK}))` | `echo $((16#${OVS_PMD_CORE_MASK}))`))
    TOTAL_CORES=`nproc`
    vcpu_pin_set=""

    for cpu in $(seq 0 `expr $TOTAL_CORES - 1`);do
        tmp=`echo 2^$cpu | bc`
        if [ $(($tmp & $BAD_CORES)) -eq 0 ]; then
            vcpu_pin_set+=$cpu","
        fi
    done
    vcpu_pin_set=${vcpu_pin_set::-1}

    if is_service_enabled nova; then
        iniset $NOVA_CONF DEFAULT vcpu_pin_set $vcpu_pin_set
    fi
}

function update_ovs_pmd_core_mask {
    OVS_CORE_MASK=$(echo $OVS_CORE_MASK | sed 's/^0x//')
    OVS_PMD_CORE_MASK=$(echo $OVS_PMD_CORE_MASK | sed 's/^0x//')

    if [ $OVS_PMD_CORE_MASK -eq 4 ]; then
        #default value, check for siblings in case of hyperthreading enabled
        SIBLINGS=""
        RESULT=0
        FILE="/sys/devices/system/cpu/cpu3/topology/thread_siblings_list"
        if [ -e $FILE ]; then
            SIBLINGS=`cat $FILE`
        else
            echo "warning: don't know how to check siblings"
            SIBLINGS=3
        fi

        for SIBLING in $(echo $SIBLINGS | sed -n 1'p' | tr ',' '\n'); do
            SIBLING_CORE=`echo "obase=10;$((1<<($SIBLING-1)))" | bc`
            RESULT=$(($RESULT | $SIBLING_CORE))
        done

        OVS_PMD_CORE_MASK=`printf "%x" $RESULT`
    fi
}


# Restore xtrace
$OVSB_XTRACE
