第一章 基础环境部署
1.1、环境
操作系统：SuSE Linux Enterprise Server 11SP3
数据库：Oracle 11gR2(112040)
节点1
主机名：jason1
ip地址：
10.18.199.209 jason1
10.0.0.1 jason1-priv
10.18.199.221 jason1-vip
ORACLE_SID：ORACLE1
SCANIP: 10.18.199.223
节点2
主机名：jason2
ip地址：
10.18.199.210 jason2
10.0.0.2 jason2-priv
10.18.199.222 jason2-vip
ORACLE_SID：ORACLE2
SCANIP: 10.18.199.223
共享存储：
设备名 | 大小 | 作用 |
/dev/sdb | 2g | OCR_VOTE |
/dev/sdc | 5g | 数据库归档日志 |
/dev/sdd | 15g | 数据库文件 |
1.2、创建账户
分别在两台服务器上创建如下账户。
groupadd-g 1000 oinstall
groupadd -g 1200asmadmin
groupadd -g 1201asmdba
groupadd -g 1202asmoper
useradd -m -u 1100 -goinstall -G asmadmin,asmdba,asmoper -d /home/grid -s /bin/bash grid
groupadd -g 1300 dba
groupadd -g 1301oper
useradd -m -u 1101 -goinstall -G dba,oper,asmdba -d /home/oracle -s /bin/bash oracle
mkdir -p /u01/app/grid
mkdir -p /u01/app/11.2.0/grid
chown -R grid:oinstall/u01
mkdir -p /u01/app/oracle
chown -R oracle:oinstall/u01/app/oracle
chmod -R 775 /u01
passwd oracle
1.3、配置hosts文件
配置两台服务器hosts文件。
#vi /etc/hosts
10.18.199.209 jason1
10.0.0.1 jason1-priv
10.18.199.221 jason1-vip
10.18.199.210 jason2
10.0.0.2 jason2-priv
10.18.199.222 jason2-vip
10.18.199.223 jason-cluster jason-cluster-scan
1.4、安装所需rpm包
两节点均查看所需rpm包，安装所需rpm包。
#rpm -q binutils gcc gcc-32bit gcc-c++ glibc glibc-32bitglibc-devel glibc-devel-32bit ksh libaio libaio-32bit libaio-devellibaio-devel-32bit libstdc++33 libstdc++33-32bit libstdc++43 libstdc++43-32bitlibstdc++43-devel libstdc++43-devel-32bit libgcc43 libstdc++-devel make sysstatunixODBC unixODBC-devel unixODBC-32bit unixODBC-devel-32bit libcap1
binutils-2.23.1-0.17.18
gcc-4.3-62.198
gcc-32bit-4.3-62.198
gcc-c++-4.3-62.198
glibc-2.11.3-17.54.1
glibc-32bit-2.11.3-17.54.1
glibc-devel-2.11.3-17.54.1
glibc-devel-32bit-2.11.3-17.54.1
ksh-93u-0.18.1
libaio-0.3.109-0.1.46
libaio-32bit-0.3.109-0.1.46
libaio-devel-0.3.109-0.1.46
libaio-devel-32bit-0.3.109-0.1.46
libstdc++33-3.3.3-11.9
libstdc++33-32bit-3.3.3-11.9
libstdc++43-4.6.9-0.11.38
libstdc++43-32bit-4.6.9-0.11.38
libstdc++43-devel-4.3.4_20091019-0.37.30
libstdc++43-devel-32bit-4.3.4_20091019-0.37.30
libgcc43-4.6.9-0.11.38
libstdc++-devel-4.3-62.198
make-3.81-128.20
sysstat-8.1.5-7.45.24
unixODBC-2.2.12-198.17
unixODBC-devel-2.2.12-198.17
unixODBC-32bit-2.2.12-198.17
unixODBC-devel-32bit-2.2.12-198.17
libcap1-1.10-6.10
1.5、修改内核参数
设置两台服务器的内核参数
#vi /etc/sysctl.conf
net.ipv4.icmp_echo_ignore_broadcasts = 1
net.ipv4.conf.all.rp_filter = 1
fs.inotify.max_user_watches = 65536
net.ipv4.conf.default.promote_secondaries = 1
net.ipv4.conf.all.promote_secondaries = 1
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmall = 2097152
kernel.shmmax = 978288640
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 9000 65500
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
vm.hugetlb_shm_group= 1000 ###此处为oinstall group number
1.6、设置Shell对Oracle与grid用户的限制
#vi /etc/security/limits.conf
oracle soft nproc 2047
oracle hard nproc 16384
oracle soft nofile 1024
oracle hard nofile 65536
grid soft nproc 2047
grid hard nproc 16384
grid soft nofile 1024
grid hard nofile 65536
编辑/etc/pam.d/login，添加如下内容：
vi /etc/pam.d/login
session required pam_limits.so
1.7、禁用NTP服务
在两台服务器上均禁止NTP服务
service ntp stop
chkconfig ntp off
mv /etc/ntp.conf /etc/ntp.conf.bak
1.8、grid用户环境变量
节点1
export ORACLE_SID=+ASM1
export ORACLE_BASE=/u01/app/grid
export ORACLE_HOME=/u01/app/11.2.0/grid
export PATH=$ORACLE_HOME/bin:$ORACLE_HOME/OPatch:$PATH
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib
export CLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib
export ORACLE_TERM=xterm
export NLS_DATE_FORMAT="YYYY-MM-DD HH24:MI:SS"
export NLS_LANG=AMERICAN_AMERICA.ZHS16GBK
export TEMP=/tmp
export TMPDIR=/tmp
umask 022
if [ $USER = "oracle" ] || [ $USER = "grid"]; then
if [ $SHELL ="/bin/ksh" ]; then
ulimit -p16384
ulimit -n65536
else
ulimit -u 16384 -n65536
fi
umask 022
fi
节点2
export ORACLE_SID=+ASM2
export ORACLE_BASE=/u01/app/grid
export ORACLE_HOME=/u01/app/11.2.0/grid
export PATH=$ORACLE_HOME/bin:$ORACLE_HOME/OPatch:$PATH
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib
exportCLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib
export ORACLE_TERM=xterm
export NLS_DATE_FORMAT="YYYY-MM-DD HH24:MI:SS"
export NLS_LANG=AMERICAN_AMERICA.ZHS16GBK
export TEMP=/tmp
export TMPDIR=/tmp
umask 022
if [ $USER = "oracle" ] || [ $USER = "grid"]; then
if [ $SHELL ="/bin/ksh" ]; then
ulimit -p16384
ulimit -n65536
else
ulimit -u 16384 -n 65536
fi
umask 022
fi
1.9、oracle用户环境变量
节点1
export ORACLE_SID=ORACLE1
export ORACLE_BASE=/u01/app/oracle
export ORACLE_HOME=$ORACLE_BASE/product/11.2.0/dbhome_1
export PATH=$ORACLE_HOME/bin:$ORACLE_HOME/OPatch:$PATH
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib
exportCLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib
export ORACLE_TERM=xterm
export NLS_DATE_FORMAT="YYYY-MM-DD HH24:MI:SS"
export NLS_LANG=AMERICAN_AMERICA.ZHS16GBK
export TEMP=/tmp
export TMPDIR=/tmp
umask 022
if [ $USER = "oracle" ] || [ $USER = "grid"]; then
if [ $SHELL ="/bin/ksh" ]; then
ulimit -p16384
ulimit -n65536
else
ulimit -u 16384 -n65536
fi
umask 022
fi
节点2
export ORACLE_SID=ORACLE2
export ORACLE_BASE=/u01/app/oracle
export ORACLE_HOME=$ORACLE_BASE/product/11.2.0/dbhome_1
export PATH=$ORACLE_HOME/bin:$ORACLE_HOME/OPatch:$PATH
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib
exportCLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib
export ORACLE_TERM=xterm
export NLS_DATE_FORMAT="YYYY-MM-DD HH24:MI:SS"
export NLS_LANG=AMERICAN_AMERICA.ZHS16GBK
export TEMP=/tmp
export TMPDIR=/tmp
umask 022
if [ $USER = "oracle" ] || [ $USER = "grid"]; then
if [ $SHELL ="/bin/ksh" ]; then
ulimit -p16384
ulimit -n65536
else
ulimit -u 16384 -n65536
fi
umask 022
fi
1.10、使用multipath配置asm磁盘
节点1与节点2均进行配置。
jason1:~# fdisk -l
Disk/dev/sda: 42.9 GB, 42949672960 bytes
255heads, 63 sectors/track, 5221 cylinders, total 83886080 sectors
Units= sectors of 1 * 512 = 512 bytes
Sectorsize (logical/physical): 512 bytes / 512 bytes
I/Osize (minimum/optimal): 512 bytes / 512 bytes
Diskidentifier: 0x0005e0f3
Device Boot Start End Blocks Id System
/dev/sda1 * 2048 208895 103424 83 Linux
/dev/sda2 208896 8595455 4193280 82 Linux swap / Solaris
/dev/sda3 8595456 83886079 37645312 83 Linux
Disk/dev/sdb: 5242 MB, 5242880000 bytes
162heads, 62 sectors/track, 1019 cylinders, total 10240000 sectors
Units= sectors of 1 * 512 = 512 bytes
Sectorsize (logical/physical): 512 bytes / 512 bytes
I/Osize (minimum/optimal): 512 bytes / 512 bytes
Diskidentifier: 0x00000000
Disk/dev/sdb doesn't contain a valid partition table
Disk/dev/sdc: 2147 MB, 2147483648 bytes
67heads, 62 sectors/track, 1009 cylinders, total 4194304 sectors
Units= sectors of 1 * 512 = 512 bytes
Sectorsize (logical/physical): 512 bytes / 512 bytes
I/Osize (minimum/optimal): 512 bytes / 512 bytes
Diskidentifier: 0x00000000
Disk/dev/sdc doesn't contain a valid partition table
Disk/dev/sdd: 15.7 GB, 15728640000 bytes
64heads, 32 sectors/track, 15000 cylinders, total 30720000 sectors
Units= sectors of 1 * 512 = 512 bytes
Sectorsize (logical/physical): 512 bytes / 512 bytes
I/Osize (minimum/optimal): 512 bytes / 512 bytes
Diskidentifier: 0x00000000
Disk/dev/sdd doesn't contain a valid partition table
jason1:~#
编辑/etc/multipath.conf文件,配置多路径软件。
defaults {
polling_interval 5
path_selector "round-robin 0"
path_grouping_policy failover
uid_attribute "ID_SERIAL"
getuid_callout "/lib/udev/scsi_id --whitelisted --device=/dev/%n"
path_checker directio
rr_weight uniform
failback immediate
queue_without_daemon yes
mode 0644
uid 1100
gid 1200
checker_timeout 30
}
blacklist{
devnode "^sda"
devnode "ofsctl"
}
multipaths{
multipath {
wwid 24d3963ec66158aac
alias ASMOCR
path_grouping_policy failover
user_friendly_names yes
path_selector "round-robin 0"
failback immediate
rr_weight uniform
}
multipath {
wwid 2a188df0f367ec9ad
alias ASMFRA
path_grouping_policy failover
user_friendly_names yes
path_selector "round-robin 0"
failback immediate
rr_weight uniform
}
multipath{
wwid 291bbf9b896a0f503
alias ASMDATA
path_grouping_policy failover
user_friendly_names yes
path_selector "round-robin 0"
failback immediate
rr_weight uniform
}
}
添加/etc/udev/rules.d/99-oracle-asmdevices.rules文件，修改设备文件权限。
使用如下命令查询
for i in 1 2 3
do
echo "KERNEL==\"dm-*\", SUBSYSTEM==\"block\",PROGRAM==\"/sbin/scsi_id --whitelisted --replace-whitespace --device=/dev/\$name\",RESULT==\"` /sbin/scsi_id --whitelisted --replace-whitespace --device=/dev/dm-$i`\", OWNER=\"grid\",GROUP=\"asmadmin\", MODE=\"0660\""
done
jason1:~ # cd /etc/udev/rules.d/
jason1:/etc/udev/rules.d # vi 99-oracle-asmdevices.rules
KERNEL=="dm-*", SUBSYSTEM=="block",PROGRAM=="/sbin/scsi_id --whitelisted --replace-whitespace --device=/dev/$name",RESULT=="24d3963ec66158aac", OWNER="grid",GROUP="asmadmin", MODE="0660"
KERNEL=="dm-*", SUBSYSTEM=="block",PROGRAM=="/sbin/scsi_id --whitelisted --replace-whitespace --device=/dev/$name",RESULT=="2a188df0f367ec9ad", OWNER="grid",GROUP="asmadmin", MODE="0660"
KERNEL=="dm-*", SUBSYSTEM=="block",PROGRAM=="/sbin/scsi_id --whitelisted --replace-whitespace --device=/dev/$name",RESULT=="291bbf9b896a0f503", OWNER="grid",GROUP="asmadmin", MODE="0660"
jason1:/etc/udev/rules.d #
查看生成的设备名称
jason1:/dev/mapper# ll
total0
lrwxrwxrwx1 root root 7 Jan 20 12:37 ASMDATA-> ../dm-2
lrwxrwxrwx1 root root 7 Jan 20 12:33 ASMFRA-> ../dm-0
lrwxrwxrwx1 root root 7 Jan 20 12:37 ASMOCR-> ../dm-1
crw-rw----1 root root 10, 236 Jan 20 12:27 control
jason1:/dev/mapper#
jason1:/dev# ll dm-*
brw-rw----1 grid asmadmin 253, 0 Jan 20 12:40 dm-0
brw-rw----1 grid asmadmin 253, 1 Jan 20 12:40 dm-1
brw-rw----1 grid asmadmin 253, 2 Jan 20 12:40 dm-2
jason1:/dev#
1.11、开启mutilipath服务
chkconfigboot.multipath on
chkconfigmultipathd on
至此，RAC初始化环境搭建完毕，接下来安装grid软件。
