Hello, All !
I'm testing Infiniband on Citrix XenServer 6.2 SP1, for using iSCSI via IPoIB.
the installation was successfull, and ibstat shows HCA is installed but the server can't find ib0.
here are some command output :
[root@Epiclesis ~]# service openibd start
Loading HCA driver and Access Layer: [ OK ]
Setting up InfiniBand network interfaces:
Setting up service network . . . [ done ]
[root@Epiclesis ~]# ibstat
CA 'mthca0'
CA type: MT25204
Number of ports: 1
Firmware version: 1.2.0
Hardware version: a0
Node GUID: 0x0008f10403992cd4
System image GUID: 0x0008f10403992cd7
Port 1:
State: Active
Physical state: LinkUp
Rate: 10
Base lid: 2
LMC: 0
SM lid: 1
Capability mask: 0x02510a68
Port GUID: 0x0008f10403992cd5
Link layer: InfiniBand
[root@Epiclesis ~]# ifconfig -a
eth0 Link encap:Ethernet HWaddr A0:B3:CC:E5:E4:B0
inet6 addr: fe80::a2b3:ccff:fee5:e4b0/64 Scope:Link
UP BROADCAST RUNNING PROMISC MULTICAST MTU:1500 Metric:1
RX packets:866 errors:0 dropped:0 overruns:0 frame:0
TX packets:512 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:94173 (91.9 KiB) TX bytes:64368 (62.8 KiB)
Interrupt:16 Memory:fb9e0000-fba00000
eth1 Link encap:Ethernet HWaddr A0:B3:CC:E5:E4:B1
inet6 addr: fe80::a2b3:ccff:fee5:e4b1/64 Scope:Link
UP BROADCAST RUNNING PROMISC MULTICAST MTU:1500 Metric:1
RX packets:257 errors:0 dropped:0 overruns:0 frame:0
TX packets:253 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:229736 (224.3 KiB) TX bytes:22646 (22.1 KiB)
Interrupt:17 Memory:fbae0000-fbb00000
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
inet6 addr: ::1/128 Scope:Host
UP LOOPBACK RUNNING MTU:16436 Metric:1
RX packets:24 errors:0 dropped:0 overruns:0 frame:0
TX packets:24 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:8485 (8.2 KiB) TX bytes:8485 (8.2 KiB)
xenbr0 Link encap:Ethernet HWaddr A0:B3:CC:E5:E4:B0
inet addr:192.168.0.14 Bcast:192.168.0.255 Mask:255.255.255.0
inet6 addr: fe80::a2b3:ccff:fee5:e4b0/64 Scope:Link
UP BROADCAST RUNNING PROMISC MULTICAST MTU:1500 Metric:1
RX packets:586 errors:0 dropped:0 overruns:0 frame:0
TX packets:510 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:77373 (75.5 KiB) TX bytes:64480 (62.9 KiB)
xenbr1 Link encap:Ethernet HWaddr A0:B3:CC:E5:E4:B1
inet addr:10.0.1.11 Bcast:10.0.1.255 Mask:255.255.255.0
inet6 addr: fe80::a2b3:ccff:fee5:e4b1/64 Scope:Link
UP BROADCAST RUNNING PROMISC MULTICAST MTU:1500 Metric:1
RX packets:257 errors:0 dropped:0 overruns:0 frame:0
TX packets:245 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:229736 (224.3 KiB) TX bytes:22142 (21.6 KiB)
[root@Epiclesis ~]# lsmod|grep ib
ib_ucm 9570 0
ib_ipoib 103667 0
ib_uverbs 43116 2 rdma_ucm,ib_ucm
ib_umad 9676 0
mlx5_ib 82608 0
mlx5_core 64288 1 mlx5_ib
mlx4_ib 141956 0
mlx4_core 269999 2 mlx4_ib,mlx4_en
ib_iser 38017 0
rdma_cm 33217 2 rdma_ucm,ib_iser
ib_cm 35503 3 ib_ucm,ib_ipoib,rdma_cm
ib_sa 29654 5 rdma_ucm,ib_ipoib,mlx4_ib,rdma_cm,ib_cm
libiscsi_tcp 14456 1 iscsi_tcp
libiscsi 38939 3 ib_iser,iscsi_tcp,libiscsi_tcp
scsi_transport_iscsi 31891 4 ib_iser,iscsi_tcp,libiscsi
ib_mthca 122160 0
ib_mad 37437 5 ib_umad,mlx4_ib,ib_cm,ib_sa,ib_mthca
ib_core 97173 14 rdma_ucm,ib_ucm,ib_ipoib,ib_uverbs,ib_umad,mlx5_ib,mlx4_ib,ib_iser,rdma_cm,ib_cm,iw_cm,ib_sa,ib_mthca,ib_mad
ib_addr 6214 3 ib_uverbs,rdma_cm,ib_core
ipv6 239245 33 ib_ipoib,mlx4_ib,ib_addr
compat 35265 19 rdma_ucm,ib_ucm,ib_ipoib,ib_uverbs,ib_umad,mlx5_ib,mlx5_core,mlx4_ib,mlx4_en,mlx4_core,ib_iser,rdma_cm,ib_cm,iw_cm,ib_sa,ib_mthca,ib_mad,ib_core,ib_addr
rtc_lib 2285 1 rtc_core
libata 153762 1 ahci
scsi_mod 145948 9 ib_iser,iscsi_tcp,libiscsi,scsi_transport_iscsi,sg,hpsa,libata,usb_storage,sd_mod
usb_libusual 9511 1 usb_storage
usbcore 138861 7 usbhid,usb_storage,usb_libusual,uhci_hcd,ohci_hcd,ehci_hcd
[root@Epiclesis ~]# ps -ef|grep ib
root 1175 2 0 22:59 ? 00:00:00 [pciback_workque]
root 1176 2 0 22:59 ? 00:00:00 [pciback_workque]
root 4365 2 0 22:59 ? 00:00:00 [ib_addr]
root 4367 2 0 22:59 ? 00:00:00 [infiniband/0]
root 4368 2 0 22:59 ? 00:00:00 [infiniband/1]
root 4844 2 0 22:59 ? 00:00:00 [ib_mad1]
root 5645 1 0 22:59 ? 00:00:00 /opt/xensource/libexec/fe -daemon -pidfile /var/run/fe.pid
root 5671 1 0 23:00 ? 00:00:00 syslogd -p /dev/reallog -f /var/lib/syslog.conf
root 5716 2 0 23:00 ? 00:00:00 [ib_mcast]
root 5717 2 0 23:00 ? 00:00:00 [ib_inform]
root 5721 2 0 23:00 ? 00:00:00 [ib_cm/0]
root 5722 2 0 23:00 ? 00:00:00 [ib_cm/1]
root 5843 1 0 23:00 ? 00:00:00 /opt/xensource/libexec/xcp-networkd -daemon -pidfile /var/run/xcp-networkd.pid
root 5851 1 0 23:00 ? 00:00:00 /opt/xensource/libexec/fe -daemon -pidfile /var/run/fe.pid
root 6263 1 0 23:00 ? 00:00:00 /opt/xensource/libexec/xcp-rrdd -daemon -pidfile /var/run/xcp-rrdd.pid
root 6318 1 0 23:00 ? 00:00:00 /opt/xensource/libexec/xcp-rrdd-plugins/xcp-rrdd-gpumon -daemon -pidfile /var/run/xcp-rrdd-gpumon.pid
root 6368 1 0 23:00 ? 00:00:00 /opt/xensource/libexec/v6d -daemon -pidfile /var/run/v6d.pid
root 6370 6368 0 23:00 ? 00:00:00 /opt/xensource/libexec/v6d -daemon -pidfile /var/run/v6d.pid
root 6425 1 0 23:00 ? 00:00:00 /opt/xensource/libexec/squeezed -daemon -pidfile /var/run/squeezed.pid
root 6450 1 0 23:00 ? 00:00:00 /opt/xensource/libexec/xenopsd -daemon -config /etc/xenopsd.conf
root 6486 1 0 23:00 ? 00:00:00 /opt/xensource/libexec/fe -daemon -pidfile /var/run/fe.pid
root 6809 1 0 23:00 tty1 00:00:01 /usr/bin/python /usr/lib/xsconsole/XSConsole.py -f root
root 6813 1 0 23:00 ? 00:00:00 /usr/lib/xen/bin/vncterm -v 127.0.0.1:1 -r -x /local/domain/0/console -T -c /usr/lib/xen/bin/dom0term.sh
root 6825 6813 0 23:00 pts/0 00:00:00 /bin/bash /usr/lib/xen/bin/dom0term.sh
root 9127 2 0 23:09 ? 00:00:00 [mlx4_ib]
root 9128 2 0 23:09 ? 00:00:00 [mlx4_ib_mcg]
root 9179 1 0 23:09 pts/1 00:00:00 /bin/bash /etc/init.d/openibd start
root 9211 2 0 23:09 ? 00:00:00 [ipoib]
root 9212 2 0 23:09 ? 00:00:00 [ipoib_auto_mode]
root 9341 8226 0 23:09 pts/1 00:00:00 grep ib
[root@Epiclesis ~]#
[root@Epiclesis ~]# ibhosts
Ca : 0x0008f10403992cd0 ports 1 "Voltaire HCA410Ex"
Ca : 0x0008f10403992cd4 ports 1 "Voltaire HCA410Ex"
[root@Epiclesis ~]# ibping 1
Pong from SR-R (Lid 1): time 0.422 ms
Pong from SR-R (Lid 1): time 0.316 ms
Pong from SR-R (Lid 1): time 0.302 ms
Pong from SR-R (Lid 1): time 0.405 ms
Pong from SR-R (Lid 1): time 0.365 ms
^C
--- SR-R (Lid 1) ibping statistics ---
5 packets transmitted, 5 received, 0% packet loss, time 4080 ms
rtt min/avg/max = 0.302/0.362/0.422 ms
[root@Epiclesis ~]#
[root@Epiclesis ~]# lspci|grep Mella
07:00.0 InfiniBand: Mellanox Technologies MT25204 [InfiniHost III Lx HCA] (rev a0)
[root@Epiclesis ~]# lspci -v -t
-[0000:00]-+-00.0 Intel Corporation Xeon E3-1200 Processor Family DRAM Controller
+-01.0-[04]----00.0 Hewlett-Packard Company Smart Array G6 controllers
+-06.0-[07]----00.0 Mellanox Technologies MT25204 [InfiniHost III Lx HCA]
+-1a.0 Intel Corporation 6 Series/C200 Series Chipset Family USB Enhanced Host Controller #2
+-1c.0-[0a]--
+-1c.4-[02]----00.0 Intel Corporation 82574L Gigabit Network Connection
+-1c.5-[03]----00.0 Intel Corporation 82574L Gigabit Network Connection
+-1c.6-[0d]--
+-1c.7-[01]--+-00.0 Hewlett-Packard Company Integrated Lights-Out Standard Slave Instrumentation & System Support
| +-00.1 Matrox Electronics Systems Ltd. MGA G200EH
| +-00.2 Hewlett-Packard Company Integrated Lights-Out Standard Management Processor Support and Messaging
| \-00.4 Hewlett-Packard Company Integrated Lights-Out Standard Virtual USB Controller
+-1d.0 Intel Corporation 6 Series/C200 Series Chipset Family USB Enhanced Host Controller #1
+-1e.0-[13]--
+-1f.0 Intel Corporation C204 Chipset Family LPC Controller
\-1f.2 Intel Corporation 6 Series/C200 Series Chipset Family SATA AHCI Controller
[root@Epiclesis network-scripts]# cat /etc/modprobe.d/mlx4
install mlx4_core /sbin/modprobe --ignore-install mlx4_core && /sbin/modprobe mlx4_en
[root@Epiclesis network-scripts]# cat /etc/modprobe.d/mlnx.conf
# Module parameters for MLNX_OFED kernel modules
blacklist mlx4_core
blacklist mlx4_en
blacklist mlx5_core
blacklist mlx5_ib
[root@Epiclesis network-scripts]# cat /etc/modprobe.conf
alias eth0 tg3
#alias ib0 ib_umad
[root@Epiclesis network-scripts]# ifup ib0
Device ib0 does not seem to be present, delaying initialization.
[root@Epiclesis network-scripts]# cat /etc/sysconfig/network-scripts/ifcfg-ib0
DEVICE="ib0"
ONBOOT="yes"
IPADDR=10.0.2.11
NETMASK=255.255.255.0
I installed MLNX_OFED_LINUX-2.2-1.0.1-xenserver6.x-i686.tgz from official Mellanox download webpage, and repeated reinstall serveral times. but no luck
strange thing is, ibping works perfectly. and there's no file like ifcfg-ib0, so I made it by myself.
I've also change modprobe.conf, but it doesn't help.
and heres what happened when I load ib_ipoib :
May 11 20:22:17 localhost kernel: [ 506.126577] mthca0: failed to create parent QP
May 11 20:22:17 localhost kernel: [ 506.127705] mthca0: ipoib_transport_dev_init failed
May 11 20:22:17 localhost kernel: [ 506.148221] mthca0: failed to initialize port 1 (ret = -12)
May 11 20:22:17 localhost kernel: [ 506.148226] ipoib_add_one: Failed to init ib port, removing it
what should I do?!
I really appreciate any help. thanks.