legacy-wiki
Lab iscsi
Recovered from the older tannerjc.net wiki snapshot dated January 23, 2016.
Setting up an iSCSI target and initiators:
http://www.howtoforge.com/creating-a-cluster-testbed-using-centos-5-virtualization-and-iscsi-p3
Configuring multipath:
http://linfrastructure.blogspot.com/2008/02/multipath-and-equallogic-iscsi.html
initiator network config
- Add GATEWAY=router to /etc/sysconfig/network-scripts/ifcfg-eth0, so that the routing table sets the correct gateway device
[root@localhost network-scripts]# ip route show
192.168.52.0/24 dev eth3 proto kernel scope link src 192.168.52.1
192.168.50.0/24 dev eth1 proto kernel scope link src 192.168.50.1
192.168.51.0/24 dev eth2 proto kernel scope link src 192.168.51.1
192.168.1.0/24 dev eth0 proto kernel scope link src 192.168.1.58
169.254.0.0/16 dev eth3 scope link
default via 192.168.1.1 dev eth0
[root@localhost network-scripts]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
192.168.52.0 0.0.0.0 255.255.255.0 U 0 0 0 eth3
192.168.50.0 0.0.0.0 255.255.255.0 U 0 0 0 eth1
192.168.51.0 0.0.0.0 255.255.255.0 U 0 0 0 eth2
192.168.1.0 0.0.0.0 255.255.255.0 U 0 0 0 eth0
169.254.0.0 0.0.0.0 255.255.0.0 U 0 0 0 eth3
0.0.0.0 192.168.1.1 0.0.0.0 UG 0 0 0 eth0
[root@localhost network-scripts]# ifconfig -a | fgrep -e encap -e inet addr
eth0 Link encap:Ethernet HWaddr 00:16:36:22:91:5E
inet addr:192.168.1.58 Bcast:192.168.1.255 Mask:255.255.255.0
eth1 Link encap:Ethernet HWaddr 54:52:00:49:41:C6
inet addr:192.168.50.1 Bcast:192.168.50.255 Mask:255.255.255.0
eth2 Link encap:Ethernet HWaddr 54:52:00:1C:8D:88
inet addr:192.168.51.1 Bcast:192.168.51.255 Mask:255.255.255.0
eth3 Link encap:Ethernet HWaddr 54:52:00:29:43:52
inet addr:192.168.52.1 Bcast:192.168.52.255 Mask:255.255.255.0
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
sit0 Link encap:IPv6-in-IPv4
[root@initiator network-scripts]# ip route show
192.168.52.0/24 dev eth3 proto kernel scope link src 192.168.52.2
192.168.50.0/24 dev eth1 proto kernel scope link src 192.168.50.2
192.168.51.0/24 dev eth2 proto kernel scope link src 192.168.51.2
192.168.1.0/24 dev eth0 proto kernel scope link src 192.168.1.66
169.254.0.0/16 dev eth3 scope link
default via 192.168.1.1 dev eth0
[root@initiator network-scripts]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
192.168.52.0 0.0.0.0 255.255.255.0 U 0 0 0 eth3
192.168.50.0 0.0.0.0 255.255.255.0 U 0 0 0 eth1
192.168.51.0 0.0.0.0 255.255.255.0 U 0 0 0 eth2
192.168.1.0 0.0.0.0 255.255.255.0 U 0 0 0 eth0
169.254.0.0 0.0.0.0 255.255.0.0 U 0 0 0 eth3
0.0.0.0 192.168.1.1 0.0.0.0 UG 0 0 0 eth0
[root@initiator network-scripts]# ifconfig -a | fgrep -e encap -e inet addr
eth0 Link encap:Ethernet HWaddr 00:16:36:53:90:66
inet addr:192.168.1.66 Bcast:192.168.1.255 Mask:255.255.255.0
eth1 Link encap:Ethernet HWaddr 00:16:36:3B:3A:86
inet addr:192.168.50.2 Bcast:192.168.50.255 Mask:255.255.255.0
eth2 Link encap:Ethernet HWaddr 00:16:36:1B:59:08
inet addr:192.168.51.2 Bcast:192.168.51.255 Mask:255.255.255.0
eth3 Link encap:Ethernet HWaddr 00:16:36:4E:DE:66
inet addr:192.168.52.2 Bcast:192.168.52.255 Mask:255.255.255.0
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
sit0 Link encap:IPv6-in-IPv4
Create targets
[root@localhost network-scripts]# pvcreate /dev/sda
Physical volume /dev/sda successfully created
[root@localhost network-scripts]# vgcreate iscsi-vg
Please enter physical volume name(s)
Run `vgcreate --help' for more information.
[root@localhost network-scripts]# vgcreate iscsi-vg /dev/sda
Volume group iscsi-vg successfully created
[root@localhost network-scripts]# lvcreate -L 1G -n target0 iscsi-vg
Logical volume target0 created
[root@localhost network-scripts]# lvs iscsi-vg
LV VG Attr LSize Origin Snap% Move Log Copy% Convert
target0 iscsi-vg -wi-a- 1.00G
- add cluster-storage channel
- yum install scsi-target-utils
- ERROR: tgtadm: can’t send the request to the tgt daemon, Transport endpoint is not connected – need to start the tgtd service
## Add target
[root@localhost network-scripts]# tgtadm --lld iscsi --op new --mode target --tid 1 -T iqn.2010-28.com.example:disk1
[root@localhost network-scripts]#
## Add device
[root@localhost network-scripts]# tgtadm --lld iscsi --op new --mode logicalunit --tid 1 --lun 1 -b /dev/iscsi-vg/target0
## Allow access
[root@localhost network-scripts]# tgtadm --lld iscsi --op bind --mode target --tid 1 -I 192.168.50.2
[root@localhost network-scripts]# tgtadm --lld iscsi --op bind --mode target --tid 1 -I 192.168.51.2
[root@localhost network-scripts]# tgtadm --lld iscsi --op bind --mode target --tid 1 -I 192.168.52.2
- need to add the tgtadm commands to /etc/init.d/rc.local or the export will not be persistent
Check exports
[root@ractarget ~]# tgtadm --lld iscsi --op show --mode target | fgrep path
Backing store path: None
Backing store path: /dev/sda
Backing store path: None
Backing store path: /dev/sdb
Backing store path: None
Backing store path: /dev/sdc
Backing store path: None
Backing store path: /dev/sdd
Login to targets from initiator
[root@initiator network-scripts]# service iscsi start
iscsid is stopped
Turning off network shutdown. Starting iSCSI daemon: [ OK ]
[ OK ]
Setting up iSCSI targets: iscsiadm: No records found!
[ OK ]
[root@initiator network-scripts]# iscsiadm -m discovery -t sendtargets -p 192.168.50.1
192.168.50.1:3260,1 iqn.2010-28.com.example:disk1
[root@initiator network-scripts]# iscsiadm -m discovery -t sendtargets -p 192.168.50.1 -l
Logging in to [iface: default, target: iqn.2010-28.com.example:disk1, portal: 192.168.50.1,3260]
Login to [iface: default, target: iqn.2010-28.com.example:disk1, portal: 192.168.50.1,3260]: successful
[root@initiator network-scripts]# iscsiadm -m discovery -t sendtargets -p 192.168.51.1 -l
192.168.51.1:3260,1 iqn.2010-28.com.example:disk1
Logging in to [iface: default, target: iqn.2010-28.com.example:disk1, portal: 192.168.51.1,3260]
Login to [iface: default, target: iqn.2010-28.com.example:disk1, portal: 192.168.51.1,3260]: successful
[root@initiator network-scripts]# iscsiadm -m discovery -t sendtargets -p 192.168.52.1 -l
192.168.52.1:3260,1 iqn.2010-28.com.example:disk1
Logging in to [iface: default, target: iqn.2010-28.com.example:disk1, portal: 192.168.52.1,3260]
Login to [iface: default, target: iqn.2010-28.com.example:disk1, portal: 192.168.52.1,3260]: successful
[root@localhost iscsi-initiator-utils-6.2.0.871]# iscsiadm --mode node
192.168.52.1:3260,1 iqn.2010-28.com.example:disk1
192.168.50.1:3260,1 iqn.2010-28.com.example:disk1
192.168.51.1:3260,1 iqn.2010-28.com.example:disk1
[root@initiator network-scripts]# cat /proc/partitions | fgrep sd
8 0 1048576 sda
8 16 1048576 sdb
8 32 1048576 sdc
[root@initiator network-scripts]# cat /proc/scsi/scsi
Attached devices:
Host: scsi0 Channel: 00 Id: 00 Lun: 00
Vendor: IET Model: Controller Rev: 0001
Type: RAID ANSI SCSI revision: 05
Host: scsi0 Channel: 00 Id: 00 Lun: 01
Vendor: IET Model: VIRTUAL-DISK Rev: 0001
Type: Direct-Access ANSI SCSI revision: 05
Host: scsi1 Channel: 00 Id: 00 Lun: 00
Vendor: IET Model: Controller Rev: 0001
Type: RAID ANSI SCSI revision: 05
Host: scsi1 Channel: 00 Id: 00 Lun: 01
Vendor: IET Model: VIRTUAL-DISK Rev: 0001
Type: Direct-Access ANSI SCSI revision: 05
Host: scsi2 Channel: 00 Id: 00 Lun: 00
Vendor: IET Model: Controller Rev: 0001
Type: RAID ANSI SCSI revision: 05
Host: scsi2 Channel: 00 Id: 00 Lun: 01
Vendor: IET Model: VIRTUAL-DISK Rev: 0001
Type: Direct-Access ANSI SCSI revision: 05
[root@initiator iscsi]# find /var/lib/iscsi/ -type f
/var/lib/iscsi/send_targets/192.168.51.1,3260/st_config
/var/lib/iscsi/send_targets/192.168.50.1,3260/st_config
/var/lib/iscsi/send_targets/192.168.52.1,3260/st_config
/var/lib/iscsi/nodes/iqn.2010-28.com.example:disk1/192.168.52.1,3260,1/default
/var/lib/iscsi/nodes/iqn.2010-28.com.example:disk1/192.168.50.1,3260,1/default
/var/lib/iscsi/nodes/iqn.2010-28.com.example:disk1/192.168.51.1,3260,1/default
[root@initiator iscsi]# cat /var/lib/iscsi/nodes/iqn.2010-28.com.example:disk1/192.168.50.1,3260,1/default | fgrep address
node.discovery_address = 192.168.50.1
node.conn[0].address = 192.168.50.1
[root@initiator iscsi]# cat /var/lib/iscsi/nodes/iqn.2010-28.com.example:disk1/192.168.51.1,3260,1/default | fgrep address
node.discovery_address = 192.168.51.1
node.conn[0].address = 192.168.51.1
[root@initiator iscsi]# cat /var/lib/iscsi/nodes/iqn.2010-28.com.example:disk1/192.168.52.1,3260,1/default | fgrep address
node.discovery_address = 192.168.52.1
node.conn[0].address = 192.168.52.1
Setup multipath on initiator
- yum install device-mapper-multipath
- comment out blacklist
- multipath -v4
[root@initiator ~]# multipath -ll
mpath0 (1IET_00010001) dm-2 IET,VIRTUAL-DISK
[size=1.0G][features=0][hwhandler=0][rw]
\_ round-robin 0 [prio=1][active]
\_ 0:0:0:1 sda 8:0 [active][ready]
\_ round-robin 0 [prio=1][enabled]
\_ 1:0:0:1 sdb 8:16 [active][ready]
\_ round-robin 0 [prio=1][enabled]
\_ 2:0:0:1 sdc 8:32 [active][ready]
[root@localhost iscsi-initiator-utils-6.2.0.871]# scsi_id -g -u -s /block/sdc
1IET_00010001
dm-0: blacklisted
dm-1: blacklisted
dm-2: blacklisted
hda: blacklisted
hdc: blacklisted
md0: blacklisted
ram0: blacklisted
ram10: blacklisted
ram11: blacklisted
ram12: blacklisted
ram13: blacklisted
ram14: blacklisted
ram15: blacklisted
ram1: blacklisted
ram2: blacklisted
ram3: blacklisted
ram4: blacklisted
ram5: blacklisted
ram6: blacklisted
ram7: blacklisted
ram8: blacklisted
ram9: blacklisted
sda: not found in pathvec
sda: mask = 0x5
sda: bus = 1
sda: dev_t = 8:0
sda: size = 2097152
sda: vendor = IET
sda: product = VIRTUAL-DISK
sda: rev = 0001
sda: h:b:t:l = 0:0:0:1
sda: path checker = readsector0 (config file default)
sda: state = 2
sdb: not found in pathvec
sdb: mask = 0x5
sdb: bus = 1
sdb: dev_t = 8:16
sdb: size = 2097152
sdb: vendor = IET
sdb: product = VIRTUAL-DISK
sdb: rev = 0001
sdb: h:b:t:l = 1:0:0:1
sdb: path checker = readsector0 (config file default)
sdb: state = 2
sdc: not found in pathvec
sdc: mask = 0x5
sdc: bus = 1
sdc: dev_t = 8:32
sdc: size = 2097152
sdc: vendor = IET
sdc: product = VIRTUAL-DISK
sdc: rev = 0001
sdc: h:b:t:l = 2:0:0:1
sdc: path checker = readsector0 (config file default)
sdc: state = 2
===== paths list =====
uuid hcil dev dev_t pri dm_st chk_st vend/prod/rev
0:0:0:1 sda 8:0 0 [undef][ready] IET,VIRTUAL-DISK
1:0:0:1 sdb 8:16 0 [undef][ready] IET,VIRTUAL-DISK
2:0:0:1 sdc 8:32 0 [undef][ready] IET,VIRTUAL-DISK
params = 0 0 3 1 round-robin 0 1 1 8:0 1000 round-robin 0 1 1 8:16 1000 round-robin 0 1 1 8:32 1000
status = 2 0 0 0 3 1 A 0 1 0 8:0 A 0 E 0 1 0 8:16 A 0 E 0 1 0 8:32 A 0
*word = 0, len = 1
*word = 0, len = 1
*word = 3, len = 1
*word = 1, len = 1
*word = round-robin, len = 11
*word = 0, len = 1
*word = 1, len = 1
*word = 1, len = 1
*word = 8:0, len = 3
*word = 1000, len = 4
*word = 1, len = 1
*word = 1, len = 1
*word = 8:16, len = 4
*word = 1000, len = 4
*word = 1, len = 1
*word = 1, len = 1
*word = 8:32, len = 4
*word = 1000, len = 4
sda: mask = 0x8
sda: getprio = NULL (internal default)
sda: prio = 1
sdb: mask = 0x8
sdb: getprio = NULL (internal default)
sdb: prio = 1
sdc: mask = 0x8
sdc: getprio = NULL (internal default)
sdc: prio = 1
*word = 2, len = 1
*word = 0, len = 1
*word = 0, len = 1
*word = 3, len = 1
*word = A, len = 1
*word = 1, len = 1
*word = 0, len = 1
*word = A, len = 1
*word = 0, len = 1
*word = E, len = 1
*word = 1, len = 1
*word = 0, len = 1
*word = A, len = 1
*word = 0, len = 1
*word = E, len = 1
*word = 1, len = 1
*word = 0, len = 1
*word = A, len = 1
*word = 0, len = 1
mpath0 (1IET_00010001) dm-2 IET,VIRTUAL-DISK
[size=1.0G][features=0][hwhandler=0][rw]
\_ round-robin 0 [prio=1][active]
\_ 0:0:0:1 sda 8:0 [active][ready]
\_ round-robin 0 [prio=1][enabled]
\_ 1:0:0:1 sdb 8:16 [active][ready]
\_ round-robin 0 [prio=1][enabled]
\_ 2:0:0:1 sdc 8:32 [active][ready]
convert to mdadm raid
## Flush multipath
[root@initiator ~]# multipath -F
[root@initiator ~]# multipath -ll
## Create raid
[root@initiator ~]# mdadm -C /dev/md0 --level=multipath --raid-devices=3 /dev/sda /dev/sdb /dev/sdc
mdadm: array /dev/md0 started.
[root@initiator ~]#
[root@initiator ~]# cat /proc/mdstat
Personalities : [multipath]
md0 : active multipath sdc[0] sdb[1] sda[2]
1048512 blocks [3/3] [UUU]
unused devices: none
## Create filesystem
[root@initiator ~]# mke2fs -j /dev/md0
mke2fs 1.39 (29-May-2006)
Filesystem label=
OS type: Linux
Block size=4096 (log=2)
Fragment size=4096 (log=2)
131072 inodes, 262128 blocks
13106 blocks (5.00%) reserved for the super user
First data block=0
Maximum filesystem blocks=268435456
8 block groups
32768 blocks per group, 32768 fragments per group
16384 inodes per group
Superblock backups stored on blocks:
32768, 98304, 163840, 229376
Writing inode tables: done
Creating journal (4096 blocks): done
Writing superblocks and filesystem accounting information: done
This filesystem will be automatically checked every 20 mounts or
180 days, whichever comes first. Use tune2fs -c or -i to override.
## Mount
[root@initiator ~]# mkdir /mnt/test
[root@initiator ~]# mount /dev/md0 /mnt/test/
## Create testfile
[root@initiator ~]# echo test test test /mnt/test/testfile.txt
[root@initiator ~]# cat /mnt/test/testfile.txt
test test test
Convert back to multipath
[root@initiator ~]# umount /mnt/test
[root@initiator ~]# mdadm --stop /dev/md0
mdadm: stopped /dev/md0
[root@initiator ~]# mdadm --remove /dev/md0
## zero the superblock on -one- path, so that lvm won't skip over the device
[root@initiator ~]# mdadm --zero-superblock /dev/sda
[root@initiator ~]# multipath
create: mpath0 (1IET_00010001) IET,VIRTUAL-DISK
[size=1.0G][features=0][hwhandler=0][n/a]
\_ round-robin 0 [prio=1][undef]
\_ 0:0:0:1 sda 8:0 [undef][ready]
\_ round-robin 0 [prio=1][undef]
\_ 1:0:0:1 sdb 8:16 [undef][ready]
\_ round-robin 0 [prio=1][undef]
\_ 2:0:0:1 sdc 8:32 [undef][ready]
[root@initiator ~]# mount /dev/mapper/mpath0 /mnt/test/
[root@initiator ~]# cat /mnt/test/testfile.txt
test test test
Path fail test
[root@localhost network-scripts]# ifdown eth3
[root@localhost network-scripts]#
Feb 28 21:13:51 localhost kernel: ping timeout of 5 secs expired, last rx 804708, last ping 809709, now 814710
Feb 28 21:13:51 localhost kernel: connection1:0: iscsi: detected conn error (1011)
Feb 28 21:13:52 localhost iscsid: Kernel reported iSCSI connection 1:0 error (1011) state (3)
Feb 28 21:14:18 localhost iscsid: connect to 192.168.52.1:3260 failed (No route to host)
Feb 28 21:14:50 localhost last message repeated 10 times
Feb 28 21:14:53 localhost iscsid: connect to 192.168.52.1:3260 failed (No route to host)
Feb 28 21:14:54 localhost kernel: session1: iscsi: session recovery timed out after 120 secs
Feb 28 21:14:54 localhost kernel: iscsi: cmd 0x28 is not queued (8)
Feb 28 21:14:54 localhost kernel: iscsi: cmd 0x28 is not queued (8)
Feb 28 21:14:54 localhost kernel: device-mapper: multipath: Failing path 8:0.
Feb 28 21:14:54 localhost multipathd: sda: readsector0 checker reports path is down
Feb 28 21:14:54 localhost multipathd: checker failed path 8:0 in map mpath0
Feb 28 21:14:54 localhost multipathd: mpath0: remaining active paths: 2
Feb 28 21:14:54 localhost multipathd: dm-2: add map (uevent)
Feb 28 21:14:54 localhost multipathd: dm-2: devmap already registered
Feb 28 21:14:56 localhost iscsid: connect to 192.168.52.1:3260 failed (No route to host)
Feb 28 21:14:59 localhost iscsid: connect to 192.168.52.1:3260 failed (No route to host)
Feb 28 21:14:59 localhost kernel: iscsi: cmd 0x28 is not queued (8)
Feb 28 21:14:59 localhost multipathd: sda: readsector0 checker reports path is down
Feb 28 21:15:03 localhost iscsid: connect to 192.168.52.1:3260 failed (No route to host)
Feb 28 21:15:04 localhost kernel: iscsi: cmd 0x28 is not queued (8)
Feb 28 21:15:04 localhost multipathd: sda: readsector0 checker reports path is down
Feb 28 21:15:06 localhost iscsid: connect to 192.168.52.1:3260 failed (No route to host)
Feb 28 21:15:09 localhost iscsid: connect to 192.168.52.1:3260 failed (No route to host)
Feb 28 21:15:09 localhost kernel: iscsi: cmd 0x28 is not queued (8)
Feb 28 21:15:09 localhost multipathd: sda: readsector0 checker reports path is down
[root@localhost ~]# multipath -ll
sda: checker msg is readsector0 checker reports path is down
mpath0 (1IET_00010001) dm-2 IET,VIRTUAL-DISK
[size=1.0G][features=0][hwhandler=0][rw]
\_ round-robin 0 [prio=1][enabled]
\_ 1:0:0:1 sdb 8:16 [active][ready]
\_ round-robin 0 [prio=0][enabled]
\_ 0:0:0:1 sda 8:0 [failed][faulty]
\_ round-robin 0 [prio=1][enabled]
\_ 2:0:0:1 sdc 8:32 [active][ready]
[root@target ~]# ifup eth3
[root@target ~]#
Feb 28 21:19:07 localhost iscsid: connect to 192.168.52.1:3260 failed (No route to host)
Feb 28 21:19:09 localhost iscsid: connection1:0 is operational after recovery (96 attempts)
Feb 28 21:19:10 localhost multipathd: sda: readsector0 checker reports path is down
Feb 28 21:19:15 localhost multipathd: sda: readsector0 checker reports path is up
Feb 28 21:19:15 localhost multipathd: 8:0: reinstated
Feb 28 21:19:15 localhost multipathd: mpath0: remaining active paths: 3
Feb 28 21:19:15 localhost multipathd: dm-2: add map (uevent)
Feb 28 21:19:15 localhost multipathd: dm-2: devmap already registered