#!/bin/bash
#
# common functions for ovs based plugin
# -------------------------------------

#
# start/stop service
#


# if we are installing ovs with dpdk dont install the standard ovs packages.
if [ $OVS_DPDK_INSTALL == 'True' ]; then
function _neutron_ovs_base_install_agent_packages {
    : # no op
}
fi

function start_ovs_dpdk {
    if [ -e /etc/init.d/ovs-dpdk ]; then
        sudo service ovs-dpdk start
    fi
}

function stop_ovs_dpdk {
    if [ -e /etc/init.d/ovs-dpdk ]; then
        sudo service ovs-dpdk stop
    fi
}

#
# ovs initialisation
#

function ovs_dpdk_configure_bridge_datapath {
    local bridge=$1
    if [ "$OVS_DATAPATH_TYPE" != "" ]; then
        sudo ovs-vsctl --no-wait set Bridge $bridge datapath_type=${OVS_DATAPATH_TYPE}
    fi
}

function ovs_dpdk_add_bridge {
    local bridge=$1
    sudo ovs-vsctl --no-wait -- --may-exist add-br $bridge
    ovs_dpdk_configure_bridge_datapath $bridge
    sudo ovs-vsctl --no-wait br-set-external-id $bridge bridge-id $bridge
}


function ovs_dpdk_setup_bridge {
    local bridge=$1
    neutron-ovs-cleanup
    ovs_dpdk_add_bridge $bridge
}


function ovs_dpdk_create_kvm_wrapper {

    install_qemu_kvm

    if [ -e /usr/bin/kvm ]; then
        KVM_CMD="/usr/bin/kvm"
    elif [ -e /usr/bin/qemu-kvm ]; then
        KVM_CMD="/usr/bin/qemu-kvm"
    elif [ -e /usr/libexec/qemu-kvm ]; then
        KVM_CMD="/usr/libexec/qemu-kvm"
    fi

    sudo mv $KVM_CMD $KVM_CMD.orig
    if [ "$os_VENDOR" == "CentOS" ]; then
        if [ ! -e /usr/bin/qemu-system-x86_64.orig ]; then
            sudo mv /usr/bin/qemu-system-x86_64 /usr/bin/qemu-system-x86_64.orig
        fi
        if [ -e /usr/bin/qemu-system-x86_64 ]; then
            sudo rm -f /usr/bin/qemu-system-x86_64
        fi
        sudo ln -s $KVM_CMD /usr/bin/qemu-system-x86_64
    fi

cat << 'EOF' | sudo tee  $KVM_CMD
#!/bin/bash -
VIRTIO_OPTIONS="csum=off,gso=off,guest_tso4=off,guest_tso6=off,guest_ecn=off"
VHOST_FORCE="vhostforce"
SHARE="share=on"
add_mem=False
i=0
while [ $# -gt 0 ]; do
    case "$1" in
    -device)
        args[i]="$1"
        (( i++ ))
        shift
        if [[ "$1" =~ "vhost-user" ]]; then
                args[i]=${1},${VHOST_FORCE}
                (( i++))
                shift

        fi
        if [[ $1 == virtio-net-pci* ]]; then
                args[i]=${1},${VIRTIO_OPTIONS}
                (( i++))
                shift

        fi
        ;;
    -object)
        args[i]="$1"
        (( i++ ))
        shift
        if [[ "$1" =~ "memory-backend-file" ]]; then
                args[i]=${1},${SHARE}
                (( i++))
                shift
        fi
        ;;
    -netdev)
        args[i]="$1"
        (( i++ ))
        shift
        if [[ "$1" =~ "vhost-user" ]]; then
                args[i]=${1},${VHOST_FORCE}
                (( i++))
                shift
        fi
        ;;
    *)
         args[i]="$1"
         (( i++ ))
         shift ;;
    esac
done

if [ -e /usr/libexec/qemu-kvm.orig ]; then
    exec /usr/libexec/qemu-kvm.orig  "${args[@]}"
elif [ -e /usr/bin/qemu-system-x86_64 ]; then
    exec /usr/bin/qemu-system-x86_64  "${args[@]}"
fi

EOF

    sudo chmod +x $KVM_CMD

}

#
# ovs cleanup
#

function cleanup_qemu_kvm_wrapper {
    if [ -e /usr/bin/kvm ]; then
        KVM_CMD="/usr/bin/kvm"
    elif [ -e /usr/bin/qemu-kvm ]; then
        KVM_CMD="/usr/bin/qemu-kvm"
    elif [ -e /usr/libexec/qemu-kvm ]; then
        KVM_CMD="/usr/libexec/qemu-kvm"
    fi
    if [ -e "$KVM_CMD.orig" ]; then
        sudo mv $KVM_CMD.orig $KVM_CMD
    fi
    if [ -e /usr/bin/qemu-system-x86_64.orig ]; then
            sudo mv /usr/bin/qemu-system-x86_64.orig /usr/bin/qemu-system-x86_64
    fi
}

function ovs_dpdk_db_cleanup {
    # remove all OVS ports that look like Neutron created ports
    for port in $(sudo ovs-vsctl list port | grep -o -e tap[0-9a-f\-]* -e q[rg]-[0-9a-f\-]*); do
        sudo ovs-vsctl --no-wait del-port ${port}
    done

    # remove all OVS bridges created by Neutron
    for bridge in $(sudo ovs-vsctl list-br | grep -o -e ${OVS_BRIDGE} -e ${PUBLIC_BRIDGE}); do
        sudo ovs-vsctl --no-wait del-br ${bridge}
    done
    if [ "${OVS_DPDK_USE_QEMU_KVM_WRAPPER}" == "True" ]; then
        cleanup_qemu_kvm_wrapper
    fi

}

function ovs_dpdk_clean(){
    sudo rm -f /usr/bin/ovs*
    sudo rm -f $OVS_DB_CONF_DIR/*
    sudo rm -f $OVS_DB_SOCKET_DIR/*
    sudo rm -f ${OVS_DIR}/BUILD_COMPLETE
}

#
# ovs installation
#

# 'git clone' only if directory doesn't exist already. Since ``DEST`` might not
# be owned by the installation user, we create the directory and change the
# ownership to the proper user.
# Set global ``RECLONE=yes`` to simulate a clone when dest-dir exists
# Set global ``ERROR_ON_CLONE=True`` to abort execution with an error if the
# git repo does not exist (default is False, meaning the repo will be cloned).
# Uses globals ``ERROR_ON_CLONE``, ``OFFLINE``, ``RECLONE``
# git_clone remote dest-dir branch
function git_clone_or_update {
    local git_remote=$1
    local git_dest=$2
    local git_ref=$3
    local orig_dir
    orig_dir=$(pwd)

    if [[ "$OFFLINE" = "True" ]]; then
        echo "Running in offline mode, clones already exist"
        # print out the results so we know what change was used in the logs
        cd $git_dest
        git show --oneline | head -1
        cd $orig_dir
        return
    fi

    if echo $git_ref | egrep -q "^refs"; then
        # If our branch name is a gerrit style refs/changes/...
        if [[ ! -d $git_dest ]]; then
            if [[ "$ERROR_ON_CLONE" = "True" ]]; then
                echo "The $git_dest project was not found; if this is a gate job, add"
                echo "the project to the \$PROJECTS variable in the job definition."
                die $LINENO "Cloning not allowed in this configuration"
            fi
            git_timed clone $git_remote $git_dest
        fi
        cd $git_dest
        git_timed fetch $git_remote $git_ref && git checkout FETCH_HEAD
    else
        # do a full clone only if the directory doesn't exist
        if [[ ! -d $git_dest ]]; then
            if [[ "$ERROR_ON_CLONE" = "True" ]]; then
                echo "The $git_dest project was not found; if this is a gate job, add"
                echo "the project to the \$PROJECTS variable in the job definition."
                die $LINENO "Cloning not allowed in this configuration"
            fi
            git_timed clone $git_remote $git_dest
            cd $git_dest
            git checkout -f $git_ref # clean modified tracked files
            git clean -d -f # clean untracked files
        elif [[ "$RECLONE" = "True" ]]; then
            # if it does exist then simulate what clone does if asked to RECLONE
            cd $git_dest
            # set the url to pull from and fetch
            git remote set-url origin $git_remote
            git_timed fetch origin
            # remove the existing ignored files (like pyc) as they cause breakage
            # (due to the py files having older timestamps than our pyc, so python
            # thinks the pyc files are correct using them)
            find $git_dest -name '*.pyc' -delete
            git checkout -f $git_ref # clean modified tracked files
            git clean -d -f # clean untracked files
        fi
    fi

    # print out the results so we know what change was used in the logs
    cd $git_dest
    git show --oneline | head -1
    cd $orig_dir
}

function ovs_dpdk_pre_install(){
    OFFLINE=$(trueorfalse False OFFLINE)

    if [ "$OFFLINE" != True ]; then
        if is_ubuntu; then
            sudo apt-get install -y linux-headers-$(uname -r) fdutils libxtst6 libnuma-dev automake
            if [ "$OVS_INTERFACE_DRIVER" == "uio_pci_generic"  ]; then
                 sudo apt-get install -y linux-image-extra-$(uname -r)
            fi
        elif [ "$os_VENDOR" == "CentOS" ]; then
            # NOTE: uio_pci_generic is provided in default modules
            sudo yum install -y kernel-devel-$(uname -r) redhat-lsb-core automake numactl-devel
        fi
    fi
    # Check minimum Mellanox OFED Versions
    # OVS_DPDK_GIT_TAG=v16.07 requires minimum MLNX_OFED > 3.3
    # OVS_DPDK_GIT_TAG=v16.11 requires minimum MLNX_OFED > 3.4
    if [ "$OVS_INTERFACE_DRIVER" == "mlnx" ]; then
        ofed_installed_version=$(ofed_info -s|tr -d [MLNX_OFED_LINUX]|cut -d'-' -f2)

        # Check if OVS_DPDK_GIT_TAG is a version X.Y
        if [[ ! $OVS_DPDK_GIT_TAG =~ .*\..* ]]; then
            return 0
        fi
        ovs_dpdk_git_version=$(echo $OVS_DPDK_GIT_TAG|tr -d [v])

        if [ -z $ofed_installed_version ] || [[ $ofed_installed_version < 3.3 ]]; then
            die $LINENO "Mellanox OFED version >= 3.3 not found"
        elif [[ $ofed_installed_version < 3.4 ]] && [[ $ofed_installed_version > 3.3 ]] &&
             [[ $ovs_dpdk_git_version > 16.10 ]]; then
            die $LINENO "Wrong version used. For OVS_DPDK_GIT_TAG >= v16.11 use Mellanox OFED >= 3.4"
        fi
    fi
}

function clone_ovs_dpdk(){
    OFFLINE=$(trueorfalse False OFFLINE)
    RECLONE=$(trueorfalse False RECLONE)

    if [[ "$OFFLINE" != True && ( "$RECLONE" == True || ! -e ${OVS_DIR}/BUILD_COMPLETE ) ]]; then
        if [ ! -d ${OVS_DIR} ] || [ "$RECLONE" == True ]; then
            # if directory exists and should be just refreshed
            # it's not possible to use devstack git_clone with commit's id
            git_clone_or_update ${OVS_GIT_REPO} ${OVS_DIR} ${OVS_GIT_TAG}
        fi

        if [ ! -d "${OVS_DPDK_DIR}" ] || [ "$RECLONE" == True ]; then
            # if directory exists and should be just refreshed
            # it's not possible to use devstack git_clone with commit's id
            git_clone_or_update ${OVS_DPDK_GIT_REPO} ${OVS_DPDK_DIR} ${OVS_DPDK_GIT_TAG}
        fi

        if [ "$OVS_PATCHES" != "" ]; then
            local patches=( $OVS_PATCHES )
            pushd  ${OVS_DIR}
            git clean -f -x -d
            git reset --hard $OVS_GIT_TAG
            for url in "${patches[@]}"; do
                curl $url | patch -p1
            done
            popd
        fi
        if [ "$OVS_DPDK_PATCHES" != "" ]; then
            local patches=( $OVS_DPDK_PATCHES )
            pushd  ${OVS_DPDK_DIR}
            git clean -f -x -d
            git reset --hard ${OVS_DPDK_GIT_TAG}
            for url in "${patches[@]}"; do
                curl $url | patch -p1
            done
            popd
        fi
    fi

    if [ "$OVS_INIT_POLICY" == "auto" ] && [ $(grep dpdk-init "${OVS_DIR}/vswitchd/vswitch.xml" > /dev/null ; echo $?) == "0" ]; then
        OVS_INIT_POLICY='db'
    elif [ "$OVS_INIT_POLICY" == "auto" ]; then
        OVS_INIT_POLICY='cmd'
    fi

    pushd  ${OVS_DIR}
    if [ "$OVS_VSWITCHD_INIT" == "auto" ] && \
       [ $(git branch --no-abbrev -v --contains  "55e075e65ef9ecbd70e5e0fada2704c3d73724d8" | grep \* | grep "${OVS_GIT_TAG}"  > /dev/null ; echo $?) == "0" ]; then
        OVS_VSWITCHD_INIT='immediate'
    elif [ "$OVS_VSWITCHD_INIT" == "auto" ]; then
        OVS_VSWITCHD_INIT='delayed'
    fi
    popd
}

function patch_ovs_emc() {
    if [ ! -z "${OVS_EMC_SIZE}" ]; then
        grep '#define EM_FLOW_HASH_SHIFT' ${OVS_DIR}/lib/dpif-netdev.c || die $LINENO "Failed to patch dpif-netdev.c for EMC hash size";
        sed "s/#define EM_FLOW_HASH_SHIFT.*$/#define EM_FLOW_HASH_SHIFT ${OVS_EMC_SIZE}/" -i ${OVS_DIR}/lib/dpif-netdev.c
    fi
}

function build_ovs_dpdk(){
    local ADDFLAGS=""
    if [ -e "${OVS_DIR}/BUILD_COMPLETE" ]; then
        echo "Build alread done."
        cd ${OVS_DIR}
        sudo make install
        return
    fi
    cd ${OVS_DPDK_DIR}
    make config T=${RTE_TARGET}
    if [ -e "${OVS_DPDK_DIR}/${RTE_TARGET}" ]; then
       rm -rf $RTE_TARGET
    fi
    ln -s -f build $RTE_TARGET

    OVS_DPDK_RTE_LIBRTE_VHOST=$(trueorfalse True OVS_DPDK_RTE_LIBRTE_VHOST)
    if [ "$OVS_DPDK_RTE_LIBRTE_VHOST" == "True" ]; then
        OVS_DPDK_RTE_LIBRTE_VHOST='y'
    else
        OVS_DPDK_RTE_LIBRTE_VHOST='n'
    fi

    OVS_DPDK_VHOST_USER_DEBUG=$(trueorfalse False OVS_DPDK_VHOST_USER_DEBUG)
    if [ "$OVS_DPDK_VHOST_USER_DEBUG" == "True" ]; then
        OVS_DPDK_VHOST_USER_DEBUG='y'
    else
        OVS_DPDK_VHOST_USER_DEBUG='n'
    fi

    OVS_DPDK_BUILD_SHARED_LIBRARY=$(trueorfalse False OVS_DPDK_BUILD_SHARED_LIB)
    if [ "$OVS_DPDK_BUILD_SHARED_LIB" == "True" ]; then
        OVS_DPDK_BUILD_SHARED_LIB='y'
    else
        OVS_DPDK_BUILD_SHARED_LIB='n'
    fi

    OVS_DPDK_CONFIG_RTE_LIBRTE_VHOST_NUMA=$(trueorfalse True OVS_DPDK_CONFIG_RTE_LIBRTE_VHOST_NUMA)
    if [ "$OVS_DPDK_CONFIG_RTE_LIBRTE_VHOST_NUMA" == "True" ]; then
        OVS_DPDK_CONFIG_RTE_LIBRTE_VHOST_NUMA='y'
    else
        OVS_DPDK_CONFIG_RTE_LIBRTE_VHOST_NUMA='n'
    fi

    local enable_igb_uio='n'
    if [ "$OVS_INTERFACE_DRIVER" == "igb_uio" ]; then
        enable_igb_uio='y'
    fi

    sed "s/CONFIG_RTE_BUILD_COMBINE_LIBS=n/CONFIG_RTE_BUILD_COMBINE_LIBS=y/" -i ${OVS_DPDK_DIR}/build/.config
    sed "s/CONFIG_RTE_MAX_MEMSEG=.*$/CONFIG_RTE_MAX_MEMSEG=${OVS_DPDK_MEM_SEGMENTS}/" -i ${OVS_DPDK_DIR}/build/.config
    sed "s/CONFIG_RTE_LIBRTE_VHOST=.*$/CONFIG_RTE_LIBRTE_VHOST=${OVS_DPDK_RTE_LIBRTE_VHOST}/" -i ${OVS_DPDK_DIR}/build/.config
    sed "s/CONFIG_RTE_LIBRTE_VHOST_NUMA=.*$/CONFIG_RTE_LIBRTE_VHOST_NUMA=${OVS_DPDK_CONFIG_RTE_LIBRTE_VHOST_NUMA}/" -i ${OVS_DPDK_DIR}/build/.config
    sed "s/CONFIG_RTE_LIBRTE_VHOST_DEBUG=.*$/CONFIG_RTE_LIBRTE_VHOST_DEBUG=${OVS_DPDK_VHOST_USER_DEBUG}/" -i ${OVS_DPDK_DIR}/build/.config
    sed "s/CONFIG_RTE_LIBRTE_KNI=.*$/CONFIG_RTE_LIBRTE_KNI=n/" -i ${OVS_DPDK_DIR}/build/.config
    sed "s/CONFIG_RTE_KNI_KMOD=.*$/CONFIG_RTE_KNI_KMOD=n/" -i ${OVS_DPDK_DIR}/build/.config
    sed "s/CONFIG_RTE_EAL_IGB_UIO=.*$/CONFIG_RTE_EAL_IGB_UIO=${enable_igb_uio}/" -i ${OVS_DPDK_DIR}/build/.config
    sed "s/CONFIG_RTE_BUILD_SHARED_LIB=.*$/CONFIG_RTE_BUILD_SHARED_LIB=${OVS_DPDK_BUILD_SHARED_LIB}/" -i ${OVS_DPDK_DIR}/build/.config

    if [ "$OVS_INTERFACE_DRIVER" == "mlnx" ]; then
        sed "s/CONFIG_RTE_LIBRTE_MLX5_PMD=n/CONFIG_RTE_LIBRTE_MLX5_PMD=y/" -i ${OVS_DPDK_DIR}/build/.config
        sed "s/CONFIG_RTE_LIBRTE_MLX5_DEBUG=n/CONFIG_RTE_LIBRTE_MLX5_DEBUG=y/" -i ${OVS_DPDK_DIR}/build/.config
        sed "s/CONFIG_RTE_LIBRTE_MLX4_PMD=n/CONFIG_RTE_LIBRTE_MLX4_PMD=y/" -i ${OVS_DPDK_DIR}/build/.config
        sed "s/CONFIG_RTE_LIBRTE_MLX4_SGE_WR_N=4/CONFIG_RTE_LIBRTE_MLX4_SGE_WR_N=1/" -i ${OVS_DPDK_DIR}/build/.config
        ADDFLAGS="LDFLAGS=-libverbs"
    fi

    # uninstall flake8-import-order since it generates unwanted warnings during ovs build
    sudo -H pip uninstall -y flake8-import-order || true

    make -j $(nproc) EXTRA_CFLAGS='-fPIC'
    sudo make install

    cd ${OVS_DIR}
    ./boot.sh
    ./configure --with-dpdk=${OVS_DPDK_DIR}/${RTE_TARGET} --prefix=/usr --with-dbdir=$OVS_DB_CONF_DIR --with-rundir=$OVS_DB_SOCKET_DIR
    patch_ovs_emc
    make -j $(nproc) CFLAGS='-O3 -march=native -fPIC' $ADDFLAGS
    touch ${OVS_DIR}/BUILD_COMPLETE
    sudo make install
}

function ovs_dpdk_write_conf {
    sudo cp $NETWORKING_OVS_DPDK_DIR/devstack/ovs-dpdk/ovs-dpdk-conf /etc/default/ovs-dpdk

    OVS_ALLOCATE_HUGEPAGES=$(trueorfalse True OVS_ALLOCATE_HUGEPAGES)
    OVS_DPDK_SERVICE_DEBUG_OUTPUT=$(trueorfalse False OVS_DPDK_SERVICE_DEBUG_OUTPUT)

    sudo sed "s#RTE_SDK=.*#RTE_SDK=$OVS_DPDK_DIR#" -i /etc/default/ovs-dpdk
    sudo sed "s#RTE_TARGET=.*#RTE_TARGET=$RTE_TARGET#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_SRC_DIR=.*#OVS_SRC_DIR=$OVS_DIR#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_NUM_HUGEPAGES=.*#OVS_NUM_HUGEPAGES=$OVS_NUM_HUGEPAGES#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_HUGEPAGE_MOUNT=.*#OVS_HUGEPAGE_MOUNT=$OVS_HUGEPAGE_MOUNT#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_HUGEPAGE_MOUNT_PAGESIZE=.*#OVS_HUGEPAGE_MOUNT_PAGESIZE=$OVS_HUGEPAGE_MOUNT_PAGESIZE#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_DB_SOCKET_DIR=.*#OVS_DB_SOCKET_DIR=$OVS_DB_SOCKET_DIR#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_DB_CONF_DIR=.*#OVS_DB_CONF_DIR=$OVS_DB_CONF_DIR#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_DB_SOCKET=.*#OVS_DB_SOCKET=$OVS_DB_SOCKET#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_DB_CONF=.*#OVS_DB_CONF=$OVS_DB_CONF#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_SOCKET_MEM=.*#OVS_SOCKET_MEM=$OVS_SOCKET_MEM#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_MEM_CHANNELS=.*#OVS_MEM_CHANNELS=$OVS_MEM_CHANNELS#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_CORE_MASK=.*#OVS_CORE_MASK=$OVS_CORE_MASK#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_PMD_CORE_MASK=.*#OVS_PMD_CORE_MASK=$OVS_PMD_CORE_MASK#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_ALLOCATE_HUGEPAGES=.*#OVS_ALLOCATE_HUGEPAGES=$OVS_ALLOCATE_HUGEPAGES#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_LOG_DIR=.*#OVS_LOG_DIR=$OVS_LOG_DIR#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_LOCK_DIR=.*#OVS_LOCK_DIR=$OVS_LOCK_DIR#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_INTERFACE_DRIVER=.*#OVS_INTERFACE_DRIVER=$OVS_INTERFACE_DRIVER#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_TUNNEL_CIDR_MAPPING=.*#OVS_TUNNEL_CIDR_MAPPING=$OVS_TUNNEL_CIDR_MAPPING#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_DPDK_SERVICE_DEBUG_OUTPUT=.*#OVS_DPDK_SERVICE_DEBUG_OUTPUT=$OVS_DPDK_SERVICE_DEBUG_OUTPUT#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_INIT_POLICY=.*#OVS_INIT_POLICY=$OVS_INIT_POLICY#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_VSWITCHD_INIT=.*#OVS_VSWITCHD_INIT=$OVS_VSWITCHD_INIT#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_VHOST_USER_SOCKET_DIR=.*#OVS_VHOST_USER_SOCKET_DIR=$OVS_VHOST_USER_SOCKET_DIR#" -i /etc/default/ovs-dpdk

    if [ "$OVS_SOCKET_MEM" == "auto" ]; then
        for d in /sys/devices/system/node/node? ; do
            if [ "$OVS_SOCKET_MEM" == "auto" ]; then
                OVS_SOCKET_MEM=2048
            else
                OVS_SOCKET_MEM=$OVS_SOCKET_MEM,2048
            fi
        done
    fi

    sudo sed -e "s#OVS_SOCKET_MEM=.*#OVS_SOCKET_MEM=$OVS_SOCKET_MEM#g" -i /etc/default/ovs-dpdk

    # Creates an array of pci addres to interface names delimeted by # e.g. <pci_address>#<interface name>
    PAIRS=( `ls -al /sys/class/net/* | awk '$0 ~ /pci|virtual/ {n=split($NF,a,"/"); if (a[4] == "virtual") { a[n-2] = "virtual."NR}; print a[n-2] "#" a[n] }' ` )
    # Populates OVS_BRIDGE_MAPPINGS if $PHYSICAL_NETWORK and $OVS_PHYSICAL_BRIDGE are used instead.
    if [[ "$OVS_DATAPATH_TYPE" != "" ]] && [[ "$OVS_BRIDGE_MAPPINGS" = "" ]] && [[ "$PHYSICAL_NETWORK" != "" ]] && [[ "$OVS_PHYSICAL_BRIDGE" != "" ]]; then
        OVS_BRIDGE_MAPPINGS=$PHYSICAL_NETWORK:$OVS_PHYSICAL_BRIDGE
    fi

    if [[ -z "$OVS_DPDK_PORT_MAPPINGS" ]]; then
        OVS_BRIDGES=${OVS_BRIDGE_MAPPINGS//,/ }
        ARRAY=( $OVS_BRIDGES )
        for net in "${ARRAY[@]}"; do
            bridge="${net##*:}"
            nic=${bridge/br-/}
            if [[ -z "$OVS_DPDK_PORT_MAPPINGS" ]]; then
                OVS_DPDK_PORT_MAPPINGS=$nic:$bridge
            else
                OVS_DPDK_PORT_MAPPINGS=$OVS_DPDK_PORT_MAPPINGS,$nic:$bridge
            fi
        done
    fi

    # replace bonds with nic's
    # first collect nic's into associate array BONDS
    declare -A BONDS
    PORTS=${OVS_BOND_PORTS//,/ }
    PORTS_ARRAY=( $PORTS )

    for pair in "${PORTS_ARRAY[@]}"; do
        name="${pair%%:*}"
        nic="${pair##*:}"
        if [[ ${BONDS[$name]} == "" ]]; then
            BONDS[$name]="($nic)"
        else
            BONDS[$name]=${BONDS[$name]},"($nic)"
        fi
    done

    # adding nics from bonds into OVS_DPDK_PORT_MAPPINGS
    for k in "${!BONDS[@]}"; do
        if [[ ${OVS_DPDK_PORT_MAPPINGS} =~ .*$k.* ]]; then
            replace=""
            bridge=$(echo $OVS_DPDK_PORT_MAPPINGS | sed -e "s/\(.*\)$k:\([^,]*\).*/\2/g")
            ports=${BONDS[$k]//,/ }
            ports_mdf=$(echo $ports | sed -e "s/(//g" | sed -e "s/)//g")
            ports_mdf_array=( $ports_mdf )
            for nic in "${ports_mdf_array[@]}"; do
                if [ -z "$replace" ]; then
                    replace="$k:$bridge,$nic:$bridge"
                else
                    replace="$replace,$nic:$bridge"
                fi
            done
            OVS_DPDK_PORT_MAPPINGS=$(echo $OVS_DPDK_PORT_MAPPINGS | sed -e "s/$k:\([^,]*\)/$replace/g")
        else
            # potential misconfiguration
            echo "warning: bond $k specified but bridge not found in OVS_DPDK_PORT_MAPPINGS"
        fi
    done

    MAPPINGS=${OVS_DPDK_PORT_MAPPINGS//,/ }

    ARRAY=( $MAPPINGS )
    NICS=""
    for net in "${ARRAY[@]}"; do
         nic="${net%%:*}"
         bridge="${net##*:}"
         printf "%s in %s\n" "$nic" "$bridge"
         for pair in "${PAIRS[@]}"; do
            if [[ $nic == `echo $pair | cut -f 2 -d "#"` ]]; then
                if [[ $NICS == "" ]]; then
                    NICS=$pair
                else
                    NICS=$NICS,$pair
                fi
            fi
        done
    done
    OVS_DPDK_BIND_PORT=$(trueorfalse True OVS_DPDK_BIND_PORT)
    sudo sed "s/OVS_PCI_MAPPINGS=.*/OVS_PCI_MAPPINGS=$NICS/" -i /etc/default/ovs-dpdk
    sudo sed "s/OVS_BRIDGE_MAPPINGS=.*/OVS_BRIDGE_MAPPINGS=$OVS_BRIDGE_MAPPINGS/" -i /etc/default/ovs-dpdk
    sudo sed "s/OVS_DPDK_PORT_MAPPINGS=.*/OVS_DPDK_PORT_MAPPINGS=$OVS_DPDK_PORT_MAPPINGS/" -i /etc/default/ovs-dpdk
    sudo sed "s/OVS_BOND_PORTS=.*/OVS_BOND_PORTS=$OVS_BOND_PORTS/" -i /etc/default/ovs-dpdk
    sudo sed "s/OVS_BOND_MODE=.*/OVS_BOND_MODE=$OVS_BOND_MODE/" -i /etc/default/ovs-dpdk
    sudo sed "s/OVS_DPDK_BIND_PORT=.*/OVS_DPDK_BIND_PORT=$OVS_DPDK_BIND_PORT/" -i /etc/default/ovs-dpdk
}

function install_qemu_kvm {
        install_package qemu
        install_package qemu-kvm
}

function install_dpdk_modules {

    echo "Install dpdk modules"

    sudo mkdir -p /lib/modules/$(uname -r)/extra/dpdk
    for mod in ${OVS_DPDK_DIR}/build/kmod/*.ko ; do
        [[ "$mod" != "${OVS_DPDK_DIR}/build/kmod/*.ko" ]] && sudo cp -f -v $mod /lib/modules/$(uname -r)/extra/dpdk/
    done
    sudo depmod -a

    MODULE=$OVS_INTERFACE_DRIVER

    # rename to module name
    if [[ "$OVS_INTERFACE_DRIVER" == "vfio-pci" ]]; then
        MODULE="vfio_pci"
    fi

    # remove first if already exists
    uninstall_dpdk_modules
    if [ -e "/etc/modules-load.d" ]; then
        echo -e "# DO NOT MODIFY this file (can be replaced or deleted during stack/unstack)\n$MODULE" | sudo tee /etc/modules-load.d/ovs-dpdk.conf
    elif [ -e "/etc/modules" ]; then
        echo -e "# ADDED by OVS, this line +1 will be removed on unstack\n$MODULE" | sudo tee -a /etc/modules
    else
        echo "WARNING: Unable to detect type of module autoloading"
    fi
}

function uninstall_dpdk_modules {
    echo "Uninstall dpdk modules"

    if [ -e "/etc/modules-load.d" ]; then
        sudo rm -f /etc/modules-load.d/ovs-dpdk.conf
    elif [ -e "/etc/modules" ]; then
        sudo sed -i '/^# ADDED by OVS, this line +1 will be removed on unstack$/,+1 {d}' /etc/modules
    fi
}

function install_ovs_dpdk {
    # Install deps
    set +o errexit
    if is_ubuntu; then
        if $(dpkg -s openvswitch-switch 2>&1 | grep installed | grep -v -i "not installed"  &> /dev/null ); then
            stop_service openvswitch-switch
            uninstall_package openvswitch-switch openvswitch-datapath-dkms openvswitch-common
        fi
        install_package autoconf libtool libfuse-dev screen
    else
        if $(rpm -qa 2>&1 | grep openvswitch &> /dev/null ); then
            stop_service openvswitch
            uninstall_package openvswitch
        fi
        install_package pciutils autoconf libtool fuse-devel screen
    fi
    # This function exits on an error so that errors don't compound and you see
    # only the first error that occurred.
    lsmod | grep openvswitch > /dev/null && sudo rmmod openvswitch
    set -o errexit

    build_ovs_dpdk
    install_dpdk_modules

    sudo cp $NETWORKING_OVS_DPDK_DIR/devstack/ovs-dpdk/ovs-dpdk-init /etc/init.d/ovs-dpdk
    type systemctl 2>&1 >/dev/null && sudo systemctl daemon-reload
    ovs_dpdk_write_conf

    if [ "${OVS_DPDK_USE_QEMU_KVM_WRAPPER}" == "True" ]; then
        ovs_dpdk_create_kvm_wrapper
    fi

    sudo chmod +x /etc/init.d/ovs-dpdk
    sudo service ovs-dpdk init

    # Create integration bridge
    ovs_dpdk_add_bridge $OVS_BRIDGE
}

function ovs_dpdk_configure_firewall_driver {
    if [[ ! -z "$OVS_ENABLE_SG_FIREWALL_MULTICAST" && "$OVS_ENABLE_SG_FIREWALL_MULTICAST" == "True" ]]; then
        iniset /$Q_PLUGIN_CONF_FILE ovs enable_sg_firewall_multicast $OVS_ENABLE_SG_FIREWALL_MULTICAST
    fi
}

function ovs_dpdk_configure_l3_agent {
    Q_USE_PUBLIC_VETH=$(trueorfalse False Q_USE_PUBLIC_VETH)
    if [ "$Q_USE_PUBLIC_VETH" == "True" ]; then
        ip link show $Q_PUBLIC_VETH_INT > /dev/null 2>&1 ||
        sudo ip link add $Q_PUBLIC_VETH_INT type veth peer name $Q_PUBLIC_VETH_EX
        sudo ip link set $Q_PUBLIC_VETH_INT up
        sudo ip link set $Q_PUBLIC_VETH_EX up
        sudo ip addr flush dev $Q_PUBLIC_VETH_EX
    else
        ovs_dpdk_add_bridge $PUBLIC_BRIDGE
    fi
}

function set_cpu_dedicated_set {
    OVS_CORE_MASK=$(echo $OVS_CORE_MASK | sed 's/^0x//')
    OVS_PMD_CORE_MASK=$(echo $OVS_PMD_CORE_MASK | sed 's/^0x//')
    BAD_CORES=$((`echo $((16#${OVS_CORE_MASK}))` | `echo $((16#${OVS_PMD_CORE_MASK}))`))
    TOTAL_CORES=`nproc`
    vcpu_pin_set=""

    for cpu in $(seq 0 `expr $TOTAL_CORES - 1`);do
        tmp=`echo 2^$cpu | bc`
        if [ $(($tmp & $BAD_CORES)) -eq 0 ]; then
            vcpu_pin_set+=$cpu","
        fi
    done
    vcpu_pin_set=${vcpu_pin_set::-1}

    if is_service_enabled nova; then
        iniset $NOVA_CONF compute cpu_dedicated_set $vcpu_pin_set
    fi
}

function update_ovs_pmd_core_mask {
    OVS_CORE_MASK=$(echo $OVS_CORE_MASK | sed 's/^0x//')
    OVS_PMD_CORE_MASK=$(echo $OVS_PMD_CORE_MASK | sed 's/^0x//')

    if [ $OVS_PMD_CORE_MASK -eq 4 ]; then
        #default value, check for siblings in case of hyperthreading enabled
        SIBLINGS=""
        RESULT=0
        FILE="/sys/devices/system/cpu/cpu3/topology/thread_siblings_list"
        if [ -e $FILE ]; then
            SIBLINGS=`cat $FILE`
        else
            echo "warning: don't know how to check siblings"
            SIBLINGS=3
        fi

        for SIBLING in $(echo $SIBLINGS | sed -n 1'p' | tr ',' '\n'); do
            SIBLING_CORE=`echo "obase=10;$((1<<($SIBLING-1)))" | bc`
            RESULT=$(($RESULT | $SIBLING_CORE))
        done

        OVS_PMD_CORE_MASK=`printf "%x" $RESULT`
    fi
}

function configure_multicast(){
    # Read multicast support variable status.
    if [[ -z $OVS_ENABLE_SG_FIREWALL_MULTICAST || "$OVS_ENABLE_SG_FIREWALL_MULTICAST" != "True" ]]; then
        return 0
    fi

    # List bridges.
    for bridge in $( sudo ovs-vsctl --columns name find bridge | grep name | awk '{ print $3 }' | tr -d '\"' ); do
        # Enable multicast snooping.
        sudo ovs-vsctl set bridge $bridge mcast_snooping_enable=true
        # Setup the multicast aging time.
        sudo ovs-vsctl set bridge $bridge other_config:mcast-snooping-aging-time=$OVS_MULTICAST_SNOOPING_AGING_TIME
        # Disable unregistered multicast (disable multicast flooding).
        sudo ovs-vsctl set Bridge $bridge other_config:mcast-snooping-disable-unregistered=true
    done

    # List patch ports.
    for patch_port in $( sudo ovs-vsctl --columns name find interface type=patch | grep name | awk '{ print $3 }' | tr -d '\"' ); do
        # Enable multicast snooping flood reports (IGMP reports) in all patch
        # port between all bridges.
        sudo ovs-vsctl set Port $patch_port other_config:mcast-snooping-flood-reports=true
    done
}
