Moving ASMLib disk to block devices (Non-ASMLib : UDEV) in 11g RAC Environment

# cat /etc/multipath.conf
# multipath.conf written by anaconda

defaults {
user_friendly_names yes
}
blacklist {
devnode “^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*”
devnode “^hd[a-z]”
devnode “^dcssblk[0-9]*”
device {
vendor “DGC”
product “LUNZ”
}
device {
vendor “IBM”
product “S/390.*”
}
# don’t count normal SATA devices as multipaths
device {
vendor “ATA”
}
# don’t count 3ware devices as multipaths
device {
vendor “3ware”
}
device {
vendor “AMCC”
}
# nor highpoint devices
device {
vendor “HPT”
}
wwid “3600508e0000000001e14fa4beee7fc03”
device {
vendor TSSTcorp
product CDDVDW_SE-208DB
}
wwid “*”
}
blacklist_exceptions {
wwid “36f84abf1005fc774000d6e8b00000000”
wwid “36f84abf1005fc774000d85dd00000001”
}
multipaths {
multipath {
uid 0
gid 0
wwid “36f84abf1005fc774000d6e8b00000000”
mode 0600
}

multipath {
wwid “36f84abf1005fc774000d85dd00000001″
alias ecdata
mode 0600
}
}

multipath -ll
ecdata (36f84abf1005fc774000d85dd00000001) dm-5 HUAWEI,S5500T
size=1.6T features=’0′ hwhandler=’0′ wp=rw
|-+- policy=’round-robin 0′ prio=1 status=active
| `- 0:0:1:0 sdb 8:16 active ready running
|-+- policy=’round-robin 0′ prio=1 status=enabled
| `- 1:0:0:2 sdf 8:80 active ready running
|-+- policy=’round-robin 0′ prio=1 status=enabled
| `- 1:0:1:2 sdh 8:112 active ready running
|-+- policy=’round-robin 0′ prio=1 status=enabled
| `- 0:0:0:0 sda 8:0 active ready running
|-+- policy=’round-robin 0′ prio=1 status=enabled
| `- 1:0:2:2 sdj 8:144 active ready running
|-+- policy=’round-robin 0′ prio=1 status=enabled
| `- 0:0:3:0 sdd 8:48 active ready running
|-+- policy=’round-robin 0′ prio=1 status=enabled
| `- 0:0:2:0 sdc 8:32 active ready running
`-+- policy=’round-robin 0′ prio=1 status=enabled
`- 1:0:3:2 sdl 8:176 active ready running

-bash-4.1$ cat /etc/scsi_id.config
options=-g

-bash-4.1$ cat /etc/udev/rules.d/99-oracle-asmdevices.rules
KERNEL==”fuse”, MODE=”0666″,OWNER=”root”,GROUP=”root”
KERNEL==”dm-*”, PROGRAM=”scsi_id –page=0x83 –whitelisted –device=/dev/%k”,RESULT==”36f84abf1005fc774000d85dd00000001″, OWNER:=”oracle”, GROUP:=”dba”

-bash-4.1$ ll /dev/dm*
brw-rw—- 1 root disk 253, 0 Sep 3 17:09 /dev/dm-0
brw-rw—- 1 root disk 253, 1 Sep 3 17:09 /dev/dm-1
brw-rw—- 1 root disk 253, 2 Sep 3 17:09 /dev/dm-2
brw-rw—- 1 root disk 253, 3 Sep 3 17:09 /dev/dm-3
brw-rw—- 1 root disk 253, 4 Sep 3 17:09 /dev/dm-4
brw-rw—- 1 oracle dba 253, 5 Sep 3 17:09 /dev/dm-5
brw-rw—- 1 root disk 253, 6 Sep 3 17:54 /dev/dm-6
brw-rw—- 1 root disk 253, 7 Sep 3 17:54 /dev/dm-7
brw-rw—- 1 root disk 253, 8 Sep 3 17:54 /dev/dm-8
brw-rw—- 1 root disk 253, 9 Sep 3 17:54 /dev/dm-9

Note:
we got only single lun of 1.6 tb

-bash-4.1$ ll /dev/mapper/
total 0
crw-rw—- 1 root root 10, 58 Sep 3 17:09 control
lrwxrwxrwx 1 root root 7 Sep 3 17:09 ecdata -> ../dm-5
lrwxrwxrwx 1 root root 7 Sep 3 17:09 ecdatap1 -> ../dm-6
lrwxrwxrwx 1 root root 7 Sep 3 17:09 ecdatap2 -> ../dm-7
lrwxrwxrwx 1 root root 7 Sep 3 17:09 ecdatap3 -> ../dm-8
lrwxrwxrwx 1 root root 7 Sep 3 17:09 ecdatap4 -> ../dm-9

fdisk -l
Disk /dev/mapper/ecdata: 1798.5 GB, 1798517555200 bytes
255 heads, 63 sectors/track, 218657 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x000bfd48

Device Boot Start End Blocks Id System
/dev/mapper/ecdatap1 * 1 654 5253223+ 83 Linux
/dev/mapper/ecdatap2 655 1308 5253255 83 Linux
/dev/mapper/ecdatap3 1309 1962 5253255 83 Linux
/dev/mapper/ecdatap4 1963 218657 1740602587+ 83 Linux

Disk /dev/mapper/ecdatap1: 5379 MB, 5379300864 bytes
255 heads, 63 sectors/track, 653 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000

Disk /dev/mapper/ecdatap2: 5379 MB, 5379333120 bytes
255 heads, 63 sectors/track, 654 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000

Disk /dev/mapper/ecdatap3: 5379 MB, 5379333120 bytes
255 heads, 63 sectors/track, 654 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000

Disk /dev/mapper/ecdatap4: 1782.4 GB, 1782377049600 bytes
255 heads, 63 sectors/track, 216695 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000

chown oracle:dba dm-6 dm-7 dm-8 dm-9

-bash-4.1$ ll /dev/dm*
brw-rw—- 1 root disk 253, 0 Sep 3 17:09 /dev/dm-0
brw-rw—- 1 root disk 253, 1 Sep 3 17:09 /dev/dm-1
brw-rw—- 1 root disk 253, 2 Sep 3 17:09 /dev/dm-2
brw-rw—- 1 root disk 253, 3 Sep 3 17:09 /dev/dm-3
brw-rw—- 1 root disk 253, 4 Sep 3 17:09 /dev/dm-4
brw-rw—- 1 oracle dba 253, 5 Sep 3 17:09 /dev/dm-5
brw-rw—- 1 oracle dba 253, 6 Sep 3 17:54 /dev/dm-6
brw-rw—- 1 oracle dba 253, 7 Sep 3 17:54 /dev/dm-7
brw-rw—- 1 oracle dba 253, 8 Sep 3 17:54 /dev/dm-8
brw-rw—- 1 oracle dba 253, 9 Sep 3 17:54 /dev/dm-9

# uname -a
Linux ecnode3 2.6.32-358.el6.x86_64 #1 SMP Tue Jan 29 11:47:41 EST 2013 x86_64 x86_64 x86_64 GNU/Linux
[root@ecnode3 ~]#
# rpm -qa | grep asm
oracleasmlib-2.0.4-1.el6.x86_64
oracleasm-support-2.1.8-1.el6.x86_64
kmod-oracleasm-2.0.6.rh1-2.el6.x86_64

-bash-4.1$ crsctl query css votedisk
## STATE File Universal Id File Name Disk group
— —– —————– ——— ———
1. ONLINE 785efe53af3c4f84bf14a53d7342dd6a (ORCL:ECOCRVOTE1) [ECLSTR]
2. ONLINE 0fc1af5f23b74fc5bf4f81d03accbead (ORCL:ECOCRVOTE2) [ECLSTR]
3. ONLINE 0d020c18947e4f8cbf54ef97dac7e353 (ORCL:ECOCRVOTE3) [ECLSTR]
Located 3 voting disk(s).

SQL> set pagesize 10000
SQL> set linesize 1000
SQL> select name,label,path from v$asm_disk;

NAME LABEL PATH
—————————— ——————————- —————————————————————————————————————————————————————————————————————————————————————-
ECDATA ECDATA ORCL:ECDATA
ECOCRVOTE1 ECOCRVOTE1 ORCL:ECOCRVOTE1
ECOCRVOTE2 ECOCRVOTE2 ORCL:ECOCRVOTE2
ECOCRVOTE3 ECOCRVOTE3 ORCL:ECOCRVOTE3

SQL> set pagesize 10000
SQL> set linesize 1000
SQL> select name,label,path from v$asm_disk;

NAME LABEL PATH
—————————— ——————————- —————————————————————————————————————————————————————————————————————————————————————-

/dev/mapper/ecdata
/dev/mapper/ecdatap1
/dev/mapper/ecdatap2
/dev/mapper/ecdatap3
/dev/mapper/ecdatap4
ECDATA ECDATA ORCL:ECDATA
ECOCRVOTE1 ECOCRVOTE1 ORCL:ECOCRVOTE1
ECOCRVOTE2 ECOCRVOTE2 ORCL:ECOCRVOTE2
ECOCRVOTE3 ECOCRVOTE3 ORCL:ECOCRVOTE3

SQL> show parameter asm

NAME TYPE VALUE
———————————— ———– ——————————
asm_diskgroups string ORADATA
asm_diskstring string
asm_power_limit integer 1
asm_preferred_read_failure_groups string
SQL>
SQL> alter system set asm_diskstring=’/dev/mapper/*’ scope=spfile;

System altered.

SQL> show parameter asm

NAME TYPE VALUE
———————————— ———– ——————————
asm_diskgroups string ORADATA
asm_diskstring string ORCL:*, /dev/mapper/*
asm_power_limit integer 1
asm_preferred_read_failure_groups string
SQL>
SQL> alter system set asm_diskstring=’/dev/mapper/*’ scope=spfile;

System altered.

SQL> show parameter asm

NAME TYPE VALUE
———————————— ———– ——————————
asm_diskgroups string ORADATA
asm_diskstring string ORCL:*, /dev/mapper/*
asm_power_limit integer 1
asm_preferred_read_failure_groups string
SQL>

SQL> set pagesize 10000
SQL> set linesize 1000
SQL> select name,label,path from v$asm_disk;

NAME LABEL PATH
—————————— ——————————- —————————————————————————————————————————————————————————————————————————————————————-
/dev/mapper/ecdata
ECOCRVOTE1 /dev/mapper/ecdatap1
ECOCRVOTE2 /dev/mapper/ecdatap2
ECOCRVOTE3 /dev/mapper/ecdatap3
ECDATA /dev/mapper/ecdatap4

SQL>

-bash-4.1$ crsctl query css votedisk
## STATE File Universal Id File Name Disk group
— —– —————– ——— ———
1. ONLINE 785efe53af3c4f84bf14a53d7342dd6a (/dev/mapper/ecdatap1) [ECLSTR]
2. ONLINE 0fc1af5f23b74fc5bf4f81d03accbead (/dev/mapper/ecdatap2) [ECLSTR]
3. ONLINE 0d020c18947e4f8cbf54ef97dac7e353 (/dev/mapper/ecdatap3) [ECLSTR]
Located 3 voting disk(s).

-bash-4.1$ crsctl stat res -t
——————————————————————————–
NAME TARGET STATE SERVER STATE_DETAILS
——————————————————————————–
Local Resources
——————————————————————————–
ora.ECLSTR.dg
ONLINE ONLINE ecnode1
ONLINE ONLINE ecnode2
ora.LISTENER.lsnr
ONLINE ONLINE ecnode1
ONLINE ONLINE ecnode2
ora.ORADATA.dg
ONLINE ONLINE ecnode1
ONLINE ONLINE ecnode2
ora.asm
ONLINE ONLINE ecnode1 Started
ONLINE ONLINE ecnode2 Started
ora.gsd
OFFLINE OFFLINE ecnode1
OFFLINE OFFLINE ecnode2
ora.net1.network
ONLINE ONLINE ecnode1
ONLINE ONLINE ecnode2
ora.ons
ONLINE ONLINE ecnode1
ONLINE ONLINE ecnode2
——————————————————————————–
Cluster Resources
——————————————————————————–
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE ecnode2
ora.cvu
1 ONLINE ONLINE ecnode2
ora.ecdb.db
1 ONLINE ONLINE ecnode1 Open
2 ONLINE ONLINE ecnode2 Open
3 ONLINE OFFLINE
ora.ecdb.spr.svc
1 ONLINE ONLINE ecnode1
ora.ecnode1.vip
1 ONLINE ONLINE ecnode1
ora.ecnode2.vip
1 ONLINE ONLINE ecnode2
ora.ecnode3.vip
1 ONLINE INTERMEDIATE ecnode2 FAILED OVER
ora.nvtxuat.db
1 ONLINE OFFLINE
ora.oc4j
1 ONLINE ONLINE ecnode2
ora.scan1.vip
1 ONLINE ONLINE ecnode2
-bash-4.1$

Ref
http://rhsummit.files.wordpress.com/2012/03/miller_oracle11gr2_rhel6.pdf
http://asanga-pradeep.blogspot.in/2012/06/moving-asmlib-disk-to-block-devices.html
http://gemsofprogramming.wordpress.com/2013/04/18/how-to-create-an-asm-disk-or-diskgroup-when-using-udev/

Advertisements

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s