@@ -1,28 +1,91 @@
# setting up irq affinity according to /proc/interrupts
# 2008-11-25 Robert Olsson
# 2009-02-19 updated by Jesse Brandeburg
#! /bin/bash
#
# > Dave Miller:
# (To get consistent naming in /proc/interrups)
# I would suggest that people use something like:
# char buf[IFNAMSIZ+6];
# Copyright (c) 2014, Intel Corporation
#
# sprintf(buf, "%s-%s-%d",
# netdev->name,
# (RX_INTERRUPT ? "rx" : "tx"),
# queue->index);
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# Assuming a device with two RX and TX queues.
# This script will assign:
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of Intel Corporation nor the names of its contributors
# may be used to endorse or promote products derived from this software
# without specific prior written permission.
#
# eth0-rx-0 CPU0
# eth0-rx-1 CPU1
# eth0-tx-0 CPU0
# eth0-tx-1 CPU1
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Affinitize interrupts to cores
#
# typical usage is (as root):
# set_irq_affinity -x local eth1 <eth2> <eth3>
#
# to get help:
# set_irq_affinity
usage ()
{
echo
echo " Usage: $0 [-x] {all|local|remote|one|custom} [ethX] <[ethY]>"
echo " options: -x Configure XPS as well as smp_affinity"
echo " options: {remote|one} can be followed by a specific node number"
echo " Ex: $0 local eth0"
echo " Ex: $0 remote 1 eth0"
echo " Ex: $0 custom eth0 eth1"
echo " Ex: $0 0-7,16-23 eth0"
echo
exit 1
}
if [ " $1 " == " -x" ; then
XPS_ENA=1
shift
fi
num=' ^[0-9]+$'
# Vars
AFF=$1
shift
case " $AFF " in
remote) [[ $1 =~ $num ]] && rnode=$1 && shift ;;
one) [[ $1 =~ $num ]] && cnt=$1 && shift ;;
all) ;;
local) ;;
custom) ;;
[0-9]* ) ;;
-h|--help) usage ;;
" "
* ) IFACES=$AFF && AFF=all ;; # Backwards compat mode
esac
# append the interfaces listed to the string with spaces
while [ " $# " -ne " 0" ; do
IFACES+=" $1 "
shift
done
# for now the user must specify interfaces
if [ -z " $IFACES " ; then
usage
exit 1
fi
# support functions
set_affinity ()
{
VEC=$core
if [ $VEC -ge 32 ]
then
MASK_FILL=" "
@@ -35,24 +98,126 @@ set_affinity()
let " VEC -= 32 * $IDX "
MASK_TMP=$(( 1 << $VEC ))
MASK=` printf " %X%s" $MASK_TMP $MASK_FILL `
MASK=$( printf " %X%s" $MASK_TMP $MASK_FILL )
else
MASK_TMP=$(( 1 << $VEC ))
MASK=` printf " %X" $MASK_TMP `
MASK=$( printf " %X" $MASK_TMP )
fi
printf " %s mask=%s for /proc/irq/%d/smp_affinity\n" $DEV $MASK $IRQ
printf " %s" $MASK > /proc/irq/$IRQ /smp_affinity
printf " %s" $MASK > /proc/irq/$IRQ /smp_affinity
printf " %s %d %s -> /proc/irq/$IRQ /smp_affinity\n" $IFACE $core $MASK
if ! [ -z " $XPS_ENA " ; then
printf " %s %d %s -> /sys/class/net/%s/queues/tx-%d/xps_cpus\n" $IFACE $core $MASK $IFACE $(( n- 1 ))
printf " %s" $MASK > /sys/class/net/$IFACE /queues/tx-$(( n- 1 ))
fi
}
if [ " $1 " = " " ; then
echo " Description:"
echo " This script attempts to bind each queue of a multi-queue NIC"
echo " to the same numbered core, ie tx0|rx0 --> cpu0, tx1|rx1 --> cpu1"
echo " usage:"
echo " $0 eth0 [eth1 eth2 eth3]"
fi
# Allow usage of , or -
#
parse_range () {
RANGE=${@// ,/ }
RANGE=${RANGE// -/ ..}
LIST=" "
for r in $RANGE ; do
# eval lets us use vars in {#..#} range
[[ $r =~ ' ..' && r=" $( eval echo {$r }) "
LIST+=" $r "
done
echo $LIST
}
# Affinitize interrupts
#
setaff ()
{
CORES=$( parse_range $CORES )
ncores=$( echo $CORES | wc -w)
n=1
# this script only supports interrupt vectors in pairs,
# modification would be required to support a single Tx or Rx queue
# per interrupt vector
queues=" ${IFACE} -.*TxRx"
irqs=$( grep " $queues " | cut -f1 -d:)
[ -z " $irqs " && irqs=$( grep $IFACE /proc/interrupts | cut -f1 -d:)
[ -z " $irqs " && irqs=$( for i in ` ls -Ux /sys/class/net/$IFACE /device/msi_irqs` ; \
do grep " $i :.*TxRx" | grep -v fdir | cut -f 1 -d : ; \
done)
[ -z " $irqs " && echo " Error: Could not find interrupts for $IFACE "
echo " IFACE CORE MASK -> FILE"
echo " ======================="
for IRQ in $irqs ; do
[ " $n " -gt " $ncores " && n=1
j=1
# much faster than calling cut for each
for i in $CORES ; do
[ $(( j++ )) -ge $n ] && break
done
core=$i
set_affinity
(( n++ ))
done
}
# now the actual useful bits of code
# these next 2 lines would allow script to auto-determine interfaces
# [ -z "$IFACES" ] && IFACES=$(ls /sys/class/net)
# [ -z "$IFACES" ] && echo "Error: No interfaces up" && exit 1
# echo IFACES is $IFACES
CORES=$( < /sys/devices/system/cpu/online)
[ " $CORES " || CORES=$( grep ^proc /proc/cpuinfo | cut -f2 -d:)
# Core list for each node from sysfs
node_dir=/sys/devices/system/node
for i in $( ls -d $node_dir /node* ) ; do
i=${i/* node/ }
corelist[$i ]=$( < $node_dir /node${i} /cpulist)
done
for IFACE in $IFACES ; do
# echo $IFACE being modified
dev_dir=/sys/class/net/$IFACE /device
[ -e $dev_dir /numa_node ] && node=$( < $dev_dir /numa_node)
[ " $node " && [ " $node " -gt 0 ] || node=0
case " $AFF " in
local)
CORES=${corelist[$node]}
;;
remote)
[ " $rnode " || { [ $node -eq 0 ] && rnode=1 || rnode=0; }
CORES=${corelist[$rnode]}
;;
one)
[ -n " $cnt " || cnt=0
CORES=$cnt
;;
all)
CORES=$CORES
;;
custom)
echo -n " Input cores for $IFACE (ex. 0-7,15-23): "
read CORES
;;
[0-9]* )
CORES=$AFF
;;
* )
usage
exit 1
;;
esac
# call the worker function
setaff
done
# check for irqbalance running
IRQBALANCE_ON=` ps ax | grep -v grep | grep -q irqbalance; echo $? `
@@ -61,35 +226,4 @@ if [ "$IRQBALANCE_ON" == "0" ] ; then
echo " likely override this script's affinitization."
echo " Please stop the irqbalance service and/or execute"
echo " 'killall irqbalance'"
fi
#
# Set up the desired devices.
#
for DEV in $*
do
for DIR in rx tx TxRx
do
MAX=` grep $DEV -$DIR /proc/interrupts | wc -l`
if [ " $MAX " == " 0" ; then
MAX=` egrep -i " $DEV :.*$DIR " | wc -l`
fi
if [ " $MAX " == " 0" ; then
echo no $DIR vectors found on $DEV
continue
fi
for VEC in ` seq 0 1 $MAX `
do
IRQ=` cat /proc/interrupts | grep -i $DEV -$DIR -$VEC " $" | cut -d: -f1 | sed " s/ //g" `
if [ -n " $IRQ " ; then
set_affinity
else
IRQ=` cat /proc/interrupts | egrep -i $DEV :v$VEC -$DIR " $" | cut -d: -f1 | sed " s/ //g" `
if [ -n " $IRQ " ; then
set_affinity
fi
fi
done
done
done
fi
SaveTheRbtz revised this gist