Fun With COMSTAR
I've been playing with a thumper for the past couple of weeks and a couple of fibre channel cards to build a simple storage array appliance .. thought I'd share the simple setup procedure for COMSTAR, that Sumit has nicely put together .. in the simple example below, I'm exporting zvols from the thumper killer to my initiator host tim (just a simple fibre channel connection FC private loop) .. observe:
killer:~ root# mdb -k
Loading modules: [ unix genunix specfs dtrace cpu.generic cpu_ms.AuthenticAMD.15
uppc pcplusmp scsi_vhci ufs ip hook neti sctp arp usba fctl nca lofs md cpc ran
dom crypto zfs fcip logindmux nsctl ptm sppp ]
> ::devbindings -q qlc
ffffff04de43a050 pci1077,2422, instance #0 (driver name: qlc)
ffffff04de437d48 pci1077,2422, instance #1 (driver name: qlc)
> ^D
killer:~ root# update_drv -a -i '"pci1077,2422"' qlt
<..reboot..>
killer:~ root# mdb -k
Loading modules: [ unix genunix specfs dtrace cpu.generic cpu_ms.AuthenticAMD.15
uppc pcplusmp scsi_vhci ufs ip hook neti sctp arp usba fctl nca lofs md cpc ran
dom crypto zfs smbsrv fcip fcp logindmux nsctl sdbc sv ptm ii sppp rdc ]
> ::devbindings -q qlt
ffffff04de43a050 pci1077,2422, instance #0 (driver name: qlt)
ffffff04de437d48 pci1077,2422, instance #1 (driver name: qlt)
> ^D
killer:~ root# svcadm enable stmf
killer:~ root# svcs stmf
STATE STIME FMRI
online 17:04:35 svc:/system/device/stmf:default
killer:~ root# stmfadm list-target -v
Target: wwn.210000E08B9E5134
Operational Status: Online
Provider Name : qlt
Alias : qlt0,0
Sessions : 1
Initiator: wwn.210000E08B9EE333
Alias: -
Logged in since: Thu Feb 21 17:50:40 2008
Target: wwn.210100E08BBE5134
Operational Status: Online
Provider Name : qlt
Alias : qlt1,0
Sessions : 1
Initiator: wwn.210100E08BBEE333
Alias: -
Logged in since: Thu Feb 21 17:50:40 2008
killer:~ root# zfs list
NAME USED AVAIL REFER MOUNTPOINT
bigpool 11.1T 3.15T 28.8K /bigpool
bigpool/vol1 100G 3.16T 98.1G -
bigpool/vol2 1T 3.18T 1002G -
bigpool/vol3 10T 3.36T 9.80T -
rootpool 55.0G 402G 23.5K /rootpool
rootpool/rootfs 5.04G 402G 5.04G legacy
rootpool/testvol 50G 403G 49.1G -
scratch 106K 457G 18K /scratch
killer:~ root# sbdadm create-lu /dev/zvol/rdsk/bigpool/vol1
Created the following LU:
GUID DATA SIZE SOURCE
-------------------------------- ------------------- ----------------
6000ae4080000000000047be01940001 107374116864 /dev/zvol/rdsk/bigpool/vol1
killer:~ root# sbdadm create-lu /dev/zvol/rdsk/bigpool/vol2
Created the following LU:
GUID DATA SIZE SOURCE
-------------------------------- ------------------- ----------------
6000ae4080000000000047be01ae0002 1099511562240 /dev/zvol/rdsk/bigpool/vol2
killer:~ root# sbdadm create-lu /dev/zvol/rdsk/bigpool/vol3
Created the following LU:
GUID DATA SIZE SOURCE
-------------------------------- ------------------- ----------------
6000ae4080000000000047be01b00003 10995116212224 /dev/zvol/rdsk/bigpool/vol3
killer:~ root# stmfadm list-lu -v
LU Name: 6000AE4080000000000047BE01940001
Operational Status: Online
Provider Name : sbd
Alias : /dev/zvol/rdsk/bigpool/vol1
View Entry Count : 0
LU Name: 6000AE4080000000000047BE01AE0002
Operational Status: Online
Provider Name : sbd
Alias : /dev/zvol/rdsk/bigpool/vol2
View Entry Count : 0
LU Name: 6000AE4080000000000047BE01B00003
Operational Status: Online
Provider Name : sbd
Alias : /dev/zvol/rdsk/bigpool/vol3
View Entry Count : 0
killer:~ root# stmfadm add-view -?
Usage: stmfadm add-view [OPTIONS]
OPTIONS:
-n, --lun
-t, --target-group
-h, --host-group
killer:~ root# stmfadm add-view -n 0 6000AE4080000000000047BE01940001
killer:~ root# stmfadm add-view -n 1 6000AE4080000000000047BE01AE0002
killer:~ root# stmfadm add-view -n 2 6000AE4080000000000047BE01B00003
killer:~ root# stmfadm list-lu -v
LU Name: 6000AE4080000000000047BE01940001
Operational Status: Online
Provider Name : sbd
Alias : /dev/zvol/rdsk/bigpool/vol1
View Entry Count : 1
LU Name: 6000AE4080000000000047BE01AE0002
Operational Status: Online
Provider Name : sbd
Alias : /dev/zvol/rdsk/bigpool/vol2
View Entry Count : 1
LU Name: 6000AE4080000000000047BE01B00003
Operational Status: Online
Provider Name : sbd
Alias : /dev/zvol/rdsk/bigpool/vol3
View Entry Count : 1
now back on tim
im:~ root# cfgadm -al -o show_SCSI_LUN c4 c5
Ap_Id Type Receptacle Occupant Condition
c4 fc-private connected configured unknown
c4::210000e08b9e5134,0 disk connected configured unknown
c4::210000e08b9e5134,1 disk connected configured unknown
c4::210000e08b9e5134,2 disk connected configured unknown
c5 fc-private connected configured unknown
c5::210100e08bbe5134,0 disk connected configured unknown
c5::210100e08bbe5134,1 disk connected configured unknown
c5::210100e08bbe5134,2 disk connected configured unknown
tim:~ root# echo | format
Searching for disks...done
AVAILABLE DISK SELECTIONS:
0. c3t0d0
/pci@0,0/pci1022,7458@11/pci1000,3060@4/sd@0,0
1. c3t1d0
/pci@0,0/pci1022,7458@11/pci1000,3060@4/sd@1,0
2. c3t2d0
/pci@0,0/pci1022,7458@11/pci1000,3060@4/sd@2,0
3. c3t3d0
/pci@0,0/pci1022,7458@11/pci1000,3060@4/sd@3,0
4. c6t6000AE4080000000000047BE01AE0002d0
/scsi_vhci/disk@g6000ae4080000000000047be01ae0002
5. c6t6000AE4080000000000047BE01B00003d0
/scsi_vhci/disk@g6000ae4080000000000047be01b00003
6. c6t6000AE4080000000000047BE01940001d0
/scsi_vhci/disk@g6000ae4080000000000047be01940001
Specify disk (enter its number): Specify disk (enter its number):
tim:~ root# zpool create z1 c6t6000AE4080000000000047BE01940001d0
tim:~ root# zpool create z2 c6t6000AE4080000000000047BE01AE0002d0
tim:~ root# zpool create z3 c6t6000AE4080000000000047BE01B00003d0
tim:~ root# df -h
Filesystem size used avail capacity Mounted on
/dev/dsk/c3t0d0s0 66G 5.1G 61G 8% /
/devices 0K 0K 0K 0% /devices
/dev 0K 0K 0K 0% /dev
ctfs 0K 0K 0K 0% /system/contract
proc 0K 0K 0K 0% /proc
mnttab 0K 0K 0K 0% /etc/mnttab
swap 54G 1.0M 54G 1% /etc/svc/volatile
objfs 0K 0K 0K 0% /system/object
sharefs 0K 0K 0K 0% /etc/dfs/sharetab
/usr/lib/libc/libc_hwcap2.so.1
66G 5.1G 61G 8% /lib/libc.so.1
fd 0K 0K 0K 0% /dev/fd
swap 54G 44K 54G 1% /tmp
swap 54G 40K 54G 1% /var/run
z1 98G 18K 98G 1% /z1
z2 1000G 18K 1000G 1% /z2
z3 9.8T 1K 9.8T 1% /z3
We'll talk about performance a little later, as well as some other details for building an easy appliance out of this setup in another entry .. stay tuned!
