赞
踩
服务器挂载共享磁盘。找出新挂载的磁盘,或者由存储工程师提供新挂载磁盘的uuid。
[root@rac01 ~]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sdf 8:80 0 5G 0 disk
└─ol-root 252:0 0 36.5G 0 lvm /
sdd 8:48 0 10G 0 disk
└─asm_ocr_3 252:2 0 10G 0 mpath
sdb 8:16 0 10G 0 disk
└─asm_ocr_1 252:3 0 10G 0 mpath
sr0 11:0 1 4.3G 0 rom
sdg 8:96 0 10G 0 disk
└─mpathc 252:6 0 10G 0 mpath
sde 8:64 0 10G 0 disk
└─asm_data_1 252:5 0 10G 0 mpath
sdc 8:32 0 10G 0 disk
└─asm_ocr_2 252:4 0 10G 0 mpath
sda 8:0 0 40G 0 disk
├─sda2 8:2 0 39.5G 0 part
│ ├─ol-swap 252:1 0 8G 0 lvm [SWAP]
│ └─ol-root 252:0 0 36.5G 0 lvm /
└─sda1 8:1 0 500M 0 part /boot
由lsblk命令分析,本次新挂载的磁盘为sdg。
以本次新增sdg盘为例:
[root@rac01 ~]# /usr/lib/udev/scsi_id -g -u /dev/sdg
36000c299b5e68b5b56df4dab0ff2904a
vim /etc/multipath.conf
#在multipaths中增加以下内容
multipath {
wwid "36000c299b5e68b5b56df4dab0ff2904a"
alias asm_data_2
}
[root@rac01 ~]# multipath -r
reload: asm_ocr_1 (36000c29a08f7e7a3698484f9f061e9e5) undef VMware, ,VMware Virtual S
size=10G features='0' hwhandler='0' wp=undef
`-+- policy='service-time 0' prio=1 status=undef
`- 3:0:0:0 sdb 8:16 active ready running
reload: asm_ocr_2 (36000c29bbfb107720df22a8ac6dab972) undef VMware, ,VMware Virtual S
size=10G features='0' hwhandler='0' wp=undef
`-+- policy='service-time 0' prio=1 status=undef
`- 3:0:1:0 sdc 8:32 active ready running
reload: asm_ocr_3 (36000c29909d9000e7cead142ba3c4bd4) undef VMware, ,VMware Virtual S
size=10G features='0' hwhandler='0' wp=undef
`-+- policy='service-time 0' prio=1 status=undef
`- 3:0:2:0 sdd 8:48 active ready running
reload: asm_data_1 (36000c290de923077f73fb939c302228c) undef VMware, ,VMware Virtual S
size=10G features='0' hwhandler='0' wp=undef
`-+- policy='service-time 0' prio=1 status=undef
`- 3:0:3:0 sde 8:64 active ready running
Dec 21 11:20:45 | 36000c299b5e68b5b56df4dab0ff2904a: rename mpathc to asm_data_2
: asm_data_2 (36000c299b5e68b5b56df4dab0ff2904a) undef VMware, ,VMware Virtual S
size=10G features='0' hwhandler='0' wp=undef
`-+- policy='service-time 0' prio=1 status=undef
`- 3:0:5:0 sdg 8:96 active ready running
vim /etc/udev/rules.d/99-oracle-asmdevices.rules
#在末尾增加
KERNEL=="dm-*",ENV{DM_UUID}=="mpath-36000c299b5e68b5b56df4dab0ff2904a",SYMLINK+="asm_asm_data_2",OWNER="grid",GROUP="asmadmin",MODE="0660"
udevadm control --reload-rules
udevadm trigger --type=devices
至此,多路径共享存储配置完成。
[root@rac02 mapper]# ll /dev/asm_asm_*
lrwxrwxrwx 1 root root 4 12月 21 11:38 /dev/asm_asm_data_1 -> dm-4
lrwxrwxrwx 1 root root 4 12月 21 11:37 /dev/asm_asm_data_2 -> dm-7
lrwxrwxrwx 1 root root 4 12月 21 11:37 /dev/asm_asm_ocr_1 -> dm-3
lrwxrwxrwx 1 root root 4 12月 21 11:37 /dev/asm_asm_ocr_2 -> dm-2
lrwxrwxrwx 1 root root 4 12月 21 11:38 /dev/asm_asm_ocr_3 -> dm-6
在grid用户下,登录sysdba,查询刚才新增的磁盘状态
su - grid
sqlplus / as sysasm
查看刚才创建的硬盘
SQL> set lines 200 pages 50000
SQL> col name for a25
SQL> col path for a55
SQL> select group_number,disk_number,name,path,total_mb/1024 total_G,mode_status,repair_timer from v$asm_disk order by group_number,disk_number;
GROUP_NUMBER DISK_NUMBER NAME PATH TOTAL_G MODE_STATUS REPAIR_TIMER
------------ ----------- ------------------------- ------------------------------------------------------- ---------- --------------------- ------------
0 0 /dev/asm_asm_data_2 0 ONLINE 0
1 0 DATA_0000 /dev/asm_asm_data_1 10 ONLINE 0
2 0 OCR_0000 /dev/asm_asm_ocr_1 10 ONLINE 0
2 1 OCR_0001 /dev/asm_asm_ocr_2 10 ONLINE 0
2 2 OCR_0002 /dev/asm_asm_ocr_3 10 ONLINE 0
查看原磁盘组的情况
SQL> select name,state,type,total_mb,free_mb from v$asm_diskgroup;
NAME STATE TYPE TOTAL_MB FREE_MB
------------------------- --------------------------------- ------------------ ---------- ----------
DATA MOUNTED EXTERN 10240 2696
OCR MOUNTED NORMAL 30720 29708
将查询到的硬盘添加到DATA磁盘组
SQL> alter diskgroup DATA add disk '/dev/asm_asm_data_2' rebalance power 6;
Diskgroup altered.
注:rebalance power的级别从1到11中选择一个数值;数值越大,rebalance速度越快,对现有运行系统影响也越大。需要根据当时业务权衡选择适合的级别;
查看磁盘平衡进度
SQL> select group_number,operation,state,est_work,est_rate,est_minutes,error_code from v$asm_operation;
##直到查询结果是no rows selected,表示rebalance已经完成。
重新查询磁盘状态
SQL> set lines 200 pages 50000
SQL> col name for a25
SQL> col path for a55
SQL> select group_number,disk_number,name,path,total_mb/1024 total_G,mode_status,repair_timer from v$asm_disk order by group_number,disk_number;
GROUP_NUMBER DISK_NUMBER NAME PATH TOTAL_G MODE_STATUS REPAIR_TIMER
------------ ----------- ------------------------- ------------------------------------------------------- ---------- --------------------- ------------
1 0 DATA_0000 /dev/asm_asm_data_1 10 ONLINE 0
1 1 DATA_0001 /dev/asm_asm_data_2 10 ONLINE 0
2 0 OCR_0000 /dev/asm_asm_ocr_1 10 ONLINE 0
2 1 OCR_0001 /dev/asm_asm_ocr_2 10 ONLINE 0
2 2 OCR_0002 /dev/asm_asm_ocr_3 10 ONLINE 0
重新查询磁盘组状态
SQL> select group_number,name,state,type,total_mb,free_mb from v$asm_diskgroup;
GROUP_NUMBER NAME STATE TYPE TOTAL_MB FREE_MB
------------ ------------------------- --------------------------------- ------------------ ---------- ----------
1 DATA MOUNTED EXTERN 20480 12840
2 OCR MOUNTED NORMAL 30720 29708
至此,ASM磁盘组扩容完成。
查询磁盘状态
SQL> set lines 200 pages 50000
SQL> col name for a25
SQL> col path for a55
SQL> select group_number,disk_number,name,path,total_mb/1024 total_G,mode_status,repair_timer from v$asm_disk order by group_number,disk_number;
GROUP_NUMBER DISK_NUMBER NAME PATH TOTAL_G MODE_STATUS REPAIR_TIMER
------------ ----------- ------------------------- ------------------------------------------------------- ---------- --------------------- ------------
1 0 DATA_0000 /dev/asm_asm_data_1 10 ONLINE 0
1 1 DATA_0001 /dev/asm_asm_data_2 10 ONLINE 0
2 0 OCR_0000 /dev/asm_asm_ocr_1 10 ONLINE 0
2 1 OCR_0001 /dev/asm_asm_ocr_2 10 ONLINE 0
2 2 OCR_0002 /dev/asm_asm_ocr_3 10 ONLINE 0
查询磁盘组状态
SQL> select group_number,name,state,type,total_mb,free_mb from v$asm_diskgroup;
GROUP_NUMBER NAME STATE TYPE TOTAL_MB FREE_MB
------------ ------------------------- --------------------------------- ------------------ ---------- ----------
1 DATA MOUNTED EXTERN 20480 12840
2 OCR MOUNTED NORMAL 30720 29708
确定要删除的磁盘,本次以移除/dev/sdg为例:
[root@rac01 ~]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sdf 8:80 0 5G 0 disk
└─ol-root 252:0 0 36.5G 0 lvm /
sdd 8:48 0 10G 0 disk
└─asm_ocr_3 252:5 0 10G 0 mpath
sdb 8:16 0 10G 0 disk
└─asm_ocr_1 252:4 0 10G 0 mpath
sr0 11:0 1 4.3G 0 rom
sdg 8:96 0 10G 0 disk
└─asm_data_2 252:6 0 10G 0 mpath
sde 8:64 0 10G 0 disk
└─asm_data_1 252:2 0 10G 0 mpath
sdc 8:32 0 10G 0 disk
└─asm_ocr_2 252:3 0 10G 0 mpath
sda 8:0 0 40G 0 disk
├─sda2 8:2 0 39.5G 0 part
│ ├─ol-swap 252:1 0 8G 0 lvm [SWAP]
│ └─ol-root 252:0 0 36.5G 0 lvm /
└─sda1 8:1 0 500M 0 part /boot
[root@rac01 ~]# /usr/lib/udev/scsi_id -g -u /dev/sdg
36000c299b5e68b5b56df4dab0ff2904a
找到sdg的多路径和udev配置信息,结合asm磁盘中的path信息,确定本次要移除的/dev/sdg磁盘的多路径配置信息为wwid为“36000c299b5e68b5b56df4dab0ff2904a”相关的配置,udev配置为“36000c299b5e68b5b56df4dab0ff2904a”和“asm_asm_data_2”相关的配置,asm磁盘path为“/dev/asm_asm_data_2”。
在grid用户下,登录sysdba
su - grid
sqlplus / as sysasm
移除磁盘
SQL> alter diskgroup DATA drop disk 'DATA_0001';
查看磁盘重新平衡进度
SQL> select group_number,operation,state,est_work,est_rate,est_minutes,error_code from v$asm_operation;
GROUP_NUMBER OPERATION STATE EST_WORK EST_RATE EST_MINUTES
------------ --------------- ------------ ---------- ---------- -----------
ERROR_CODE
------------------------------------------------------------------------------------------------------------------------------------
1 REBAL REAP 0 0 0
1 REBAL DONE 949 0 0
1 REBAL DONE 0 0 0
SQL> select group_number,operation,state,est_work,est_rate,est_minutes,error_code from v$asm_operation;
no rows selected
vim /etc/udev/rules.d/99-oracle-asmdevices.rules
#删除2.1中确认的配置内容
KERNEL=="dm-*",ENV{DM_UUID}=="mpath-36000c299b5e68b5b56df4dab0ff2904a",SYMLINK+="asm_asm_data_2",OWNER="grid",GROUP="asmadmin",MODE="0660"
udevadm control --reload-rules
udevadm trigger --type=devices
至此,多路径共享存储配置完成。
[root@rac01 ~]# ll /dev/asm_asm_*
lrwxrwxrwx 1 root root 4 12月 21 14:02 /dev/asm_asm_data_1 -> dm-2
lrwxrwxrwx 1 root root 4 12月 21 14:02 /dev/asm_asm_ocr_1 -> dm-4
lrwxrwxrwx 1 root root 4 12月 21 14:02 /dev/asm_asm_ocr_2 -> dm-3
lrwxrwxrwx 1 root root 4 12月 21 14:02 /dev/asm_asm_ocr_3 -> dm-5
vim /etc/multipath.conf
#在multipaths中删除2.1中确认的以下内容
multipath {
wwid "36000c299b5e68b5b56df4dab0ff2904a"
alias asm_data_2
}
[root@rac02 ~]# multipath -r
reload: asm_ocr_1 (36000c29a08f7e7a3698484f9f061e9e5) undef VMware, ,VMware Virtual S
size=10G features='0' hwhandler='0' wp=undef
`-+- policy='service-time 0' prio=1 status=undef
`- 3:0:0:0 sdb 8:16 active ready running
reload: asm_ocr_2 (36000c29bbfb107720df22a8ac6dab972) undef VMware, ,VMware Virtual S
size=10G features='0' hwhandler='0' wp=undef
`-+- policy='service-time 0' prio=1 status=undef
`- 3:0:1:0 sdc 8:32 active ready running
reload: asm_ocr_3 (36000c29909d9000e7cead142ba3c4bd4) undef VMware, ,VMware Virtual S
size=10G features='0' hwhandler='0' wp=undef
`-+- policy='service-time 0' prio=1 status=undef
`- 3:0:2:0 sdd 8:48 active ready running
reload: asm_data_1 (36000c290de923077f73fb939c302228c) undef VMware, ,VMware Virtual S
size=10G features='0' hwhandler='0' wp=undef
`-+- policy='service-time 0' prio=1 status=undef
`- 3:0:3:0 sde 8:64 active ready running
reload: mpathb (36000c2962c475885d64356eadf8b9757) undef VMware, ,VMware Virtual S
size=5.0G features='0' hwhandler='0' wp=undef
`-+- policy='service-time 0' prio=1 status=undef
`- 3:0:4:0 sdf 8:80 active ready running
Dec 21 14:04:44 | 36000c299b5e68b5b56df4dab0ff2904a: rename asm_data_2 to mpatha
: mpatha (36000c299b5e68b5b56df4dab0ff2904a) undef VMware, ,VMware Virtual S
size=10G features='0' hwhandler='0' wp=undef
`-+- policy='service-time 0' prio=1 status=undef
`- 3:0:5:0 sdg 8:96 active ready running
至此磁盘移除配置完成。/dev/sdg磁盘可以从操作系统中移除,或者改为其他用途。
在实际生产中,还会遇到另外一种情况,就是需要对正在使用的磁盘进行替换。那么我们可以按照前面的内容进行增加和删除磁盘,但是在实际应用中我发现先增加磁盘,再删除磁盘的方案虽然可行,但是在数据量很大的情况下实际操作很慢。故查阅文档,发现了文档中提到了一个将删除和添加磁盘放到一个sql语句中执行的方案,并且经过验证,该方法可以大幅提升磁盘数据迁移的效率。以下为sql案例:
SQL> alter diskgroup DATA drop disk 'DATA_0001' add disk '/dev/asm_asm_data_2' rebalance power 6;
以下命令用于对ASM实例进行管理:
srvctl status asm
srvctl stop asm
srvctl start asm
以下命令用于对ASM磁盘组进行管理:
srvctl status diskgroup -g data
srvctl stop diskgroup -g data
srvctl start diskgroup -g data
创建磁盘组(一般建议用asmca图形化工具进行创建):
$ sqlplus / as sysasm
SQL> CREATE DISKGROUP dg1 external REDUNDANCY
DISK
'/dev/raw/raw1',
'/dev/raw/raw2'
磁盘组通过mirror方式保证数据的安全
普通冗余:每个磁盘组有两个FAILGROUP
高冗余:每个磁盘组有三个FAILGROUP
外部冗余:通过存储设备的冗余来保证数据安全
SQL> CREATE DISKGROUP dg1 NORMAL REDUNDANCY
FAILGROUP fg1 DISK
‘/dev/raw/raw1, '/dev/raw/raw3'
FAILGROUP fg2 DISK
‘/dev/raw/raw2’, '/dev/raw/raw4';
在创建磁盘组的节点上,磁盘组被自动挂接。在其他节点需要手动挂载:
SQL> ALTER DISKGROUP dg1 MOUNT;
磁盘组卸载:
SQL> ALTER DISKGROUP dg1 DISMOUNT;
向磁盘组中添加磁盘:
SQL>ALTER DISKGROUP dg1
ADD DISK
'/dev/raw/raw3' NAME disk3,
'/dev/raw/raw4' NAME disk4;
从磁盘组中剔除磁盘:
SQL> ALTER DISKGROUP dg1 DROP DISK disk3;
SQL> alter diskgroup dg1 drop disk3 in failgroup fg2;
SQL> ALTER DISKGROUP DG1 UNDROP DISKS;
当磁盘组中的磁盘发生变化时,磁盘组将自动进行重新平衡。平衡级别为0-11,当手工进行重新平衡时,可以指定平衡级别。可用的最高级别通过初始化参数ASM_POWER_LIMIT指定。
SQL> ALTER DISKGROUP dg1 REBALANCE POWER 3;
SQL> ALTER DISKGROUP dg1 REBALANCE POWER 3 WAIT;
在其中一个选定的节点上保持磁盘组处于mounted状态,**在其他节点上将磁盘组卸载**,在选定的节点上执行:
SQL> DROP DISKGROUP dg1;
SQL> DROP DISKGROUP dg1 INCLUDING CONTENTS;
#清理磁盘头信息方法。
dd if=/dev/null of=/dev/sdb1 bs=1m count=1
磁盘组的查询:
select name,state,type,total_mb,free_mb from v$asm_diskgroup;
select group_num,path,total_mb,free_mb from v$asm_disk;
select group_number,instance_name,db_name,status from v$asm_client;
磁盘组中目录的管理:
#目录的创建:
SQL>ALTER DISKGROUP dg1 ADD DIRECTORY '+dg1/dir1';
#目录的重新命名:
SQL>ALTER DISKGROUP dg1 RENAME DIRECTORY ‘+dg1/dir1’ TO ‘+dg1/dir2‘
#目录的删除:
SQL>ALTER DISKGROUP dg1 DROP DIRECTORY '+dg1/dir1' FORCE;
#删除磁盘组中不再需要的文件,例如:
SQL> ALTER DISKGROUP dg1 DROP FILE '+dg1/myfiles/my_undo_ts';
Copyright © 2003-2013 www.wpsshop.cn 版权所有,并保留所有权利。