linux下搭建HA高可用集群

浏览数：26 / 时间：2015年06月20日

HA 即（high available）高可用，又被叫做双机热备，用于关键性业务。简单理解就是，有两台机器A和B，正常是A提供服务，B待命闲置，当A宕机或服务宕掉，会切换至B机器继续提供服务。常用实现高可用的开源软件有heartbeat和keepalived，其中keepalived有负载均衡的功能。

实验前需准备的环境：

系统：两个CentOS 6.6系统

网卡：eth0、eth1 （本实验是在虚拟机上搭建需要提前给两个虚拟机系统各添加一块网卡）

给两个系统配置好网卡和IP后：

root@webserver ~]# ifconfig   //此为主机
eth0      Link encap:Ethernet HWaddr 00:0C:29:E6:9E:DF
          inet addr:192.168.1.111 Bcast:192.168.1.255 Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fee6:9edf/64 Scope:Link

eth1      Link encap:Ethernet HWaddr 00:0C:29:E6:9E:E9
          inet addr:192.168.217.111 Bcast:192.168.217.255 Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fee6:9ee9/64 Scope:Link

[root@localhost ~]# ifconfig    //此为从机
eth0      Link encap:Ethernet HWaddr 00:0C:29:77:83:2A
          inet addr:192.168.1.119 Bcast:192.168.1.255 Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fe77:832a/64 Scope:Link

eth1      Link encap:Ethernet HWaddr 00:0C:29:77:83:34
          inet addr:192.168.217.119 Bcast:192.168.217.255 Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fe77:8334/64 Scope:Link

搭建步骤：

关闭防火墙、selinux
[root@webserver ~]# iptables -F
[root@webserver ~]# setenforce 0
setenforce: SELinux is disabled
[root@localhost ~]# setenforce 0
setenforce: SELinux is disabled
[root@localhost ~]# iptables -F
添加hosts
[root@webserver ~]# vi /etc/hosts //添加以下内容
192.168.1.111 webserver
192.168.1.119 localhost
[root@localhost ~]# vi /etc/hosts //添加以下内容
192.168.1.111 webserver
192.168.1.119 localhost
两台机同时安装epel扩展源
[root@webserver ~]# yum install -y epel-release
[root@localhost ~]# yum install -y epel-release
两个机器都安装heartbeat 、libnet //本机代理为nginx服务，由于做LNMP时已安装过
[root@webserver ~]# yum install -y heartbeat* libnet
[root@localhost ~]# yum install -y heartbeat* libnet

主上配置

[root@webserver ~]# cd /usr/share/doc/heartbeat-3.0.4/
[root@webserver heartbeat-3.0.4]# cp authkeys ha.cf haresources /etc/ha.d/
[root@webserver heartbeat-3.0.4]# cd /etc/ha.d/
[root@webserver ha.d]# vi authkeys //按以下更改配置文件

auth 3

#1 crc

#2 sha1 HI!

3 md5 Hello!

[root@webserver ha.d]# chmod 600 authkeys

[root@webserver ha.d]# vi haresources //添加以下内容，192.168.1.110 ip地址是流动ip，虚拟ip(vip)；用来绑定服务的；子网掩码为24位；使用eth0:0绑定虚拟ip，并运行nginx进行测试；

webserver 192.168.1.110/24/eth0:0 nginx

[root@webserver ha.d]# vi ha.cf //添加以下内容

debugfile /var/log/ha-debug

logfile /var/log/ha-log

logfacility local0

keepalive 2

deadtime 30

warntime 10

initdead 60

udpport 694

ucast eth1 192.168.217.119

auto_failback on

node webserver

node localhost

ping192.168.217.1

respawn hacluster /usr/lib/heartbeat/ipfail

从上配置

[root@webserver ha.d]# scp ha.cf haresources authkeys localhost:/etc/ha.d/ //将主上配置文件复制到从

[root@localhost ha.d]# vi ha.cf //另外两个文件不用更改

debugfile /var/log/ha-debug

logfile /var/log/ha-log

logfacility local0

keepalive 2

deadtime 30

warntime 10

initdead 60

udpport 694

ucast eth1 192.168.217.111

auto_failback on

node webserver

node localhost

ping 192.168.217.1

respawn hacluster /usr/lib/heartbeat/ipfail

启动heartbeat，现主后从
[root@webserver ha.d]# /etc/init.d/heartbeat start
Starting High-Availability services: INFO: Resource is stopped
Done.
[root@localhost ha.d]# /etc/init.d/heartbeat start
Starting High-Availability services: INFO: Resource is stopped
Done.
检查测试
[root@webserver ha.d]# ifconfig
eth0      Link encap:Ethernet  HWaddr 00:0C:29:E6:9E:DF
          inet addr:192.168.1.111  Bcast:192.168.1.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fee6:9edf/64 Scope:Link
eth0:0     Link encap:Ethernet HWaddr 00:0C:29:E6:9E:E9
        inet addr:192.168.1.110 Bcast:0.0.0.0 Mask:255.255.255.0
         UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
         Interrupt:19 Base address:0x2080

eth1      Link encap:Ethernet  HWaddr 00:0C:29:E6:9E:E9
          inet addr:192.168.217.111  Bcast:192.168.217.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fee6:9ee9/64 Scope:Link
[root@webserver ha.d]# ps aux |grep nginx
root     15707  0.0  0.1  15668  1216 ?        Ss   23:16   0:00 nginx: master process /usr/sbin/nginx -c /etc/nginx/nginx.conf
nginx    15709  0.0  0.1  15824  1796 ?        S    23:16   0:00 nginx: worker process
root     15711  0.0  0.0   5980   748 pts/1    S+   23:16   0:00 grep nginx
测试1：故意禁ping
[root@webserver ha.d]# iptables -I INPUT -p icmp -j DROP
[root@webserver ha.d]# ifconfig
eth0      Link encap:Ethernet  HWaddr 00:0C:29:E6:9E:DF
          inet addr:192.168.1.111  Bcast:192.168.1.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fee6:9edf/64 Scope:Link

eth1      Link encap:Ethernet  HWaddr 00:0C:29:E6:9E:E9
          inet addr:192.168.217.111  Bcast:192.168.217.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fee6:9ee9/64 Scope:Link

[root@localhost ha.d]# ifconfig
eth0      Link encap:Ethernet  HWaddr 00:0C:29:77:83:2A
          inet addr:192.168.1.119  Bcast:192.168.1.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fe77:832a/64 Scope:Link
eth0:0   Link encap:Ethernet HWaddr 00:0C:29:77:83:34
          inet addr:192.168.1.110 Bcast:0.0.0.0 Mask:255.255.255.0
          UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
          Interrupt:19 Base address:0x2080

eth1      Link encap:Ethernet  HWaddr 00:0C:29:77:83:34
          inet addr:192.168.217.119  Bcast:192.168.217.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fe77:8334/64 Scope:Link

[root@localhost ha.d]# ps aux |grep nginx
root      5037  0.0  0.1  15668  1220 ?        Ss   23:48   0:00 nginx: master process /usr/sbin/nginx -c /etc/nginx/nginx.conf
nginx     5039  0.0  0.1  15824  1800 ?        S    23:48   0:00 nginx: worker process
root      5042  0.0  0.0   5980   748 pts/0    S+   23:49   0:00 grep nginx
测试2：主上停止heartbeat服务
[root@webserver ha.d]# service heartbeat stop
Stopping High-Availability services: Done.

[root@webserver ha.d]# ifconfig
eth0      Link encap:Ethernet  HWaddr 00:0C:29:E6:9E:DF
          inet addr:192.168.1.111  Bcast:192.168.1.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fee6:9edf/64 Scope:Link

eth1      Link encap:Ethernet  HWaddr 00:0C:29:E6:9E:E9
          inet addr:192.168.217.111  Bcast:192.168.217.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fee6:9ee9/64 Scope:Link

[root@localhost ha.d]# ifconfig
eth0      Link encap:Ethernet  HWaddr 00:0C:29:77:83:2A
          inet addr:192.168.1.119  Bcast:192.168.1.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fe77:832a/64 Scope:Link
eth1:0   Link encap:Ethernet HWaddr 00:0C:29:77:83:34
          inet addr:192.168.1.110 Bcast:0.0.0.0 Mask:255.255.255.0
          UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
          Interrupt:19 Base address:0x2080

eth1      Link encap:Ethernet  HWaddr 00:0C:29:77:83:34
          inet addr:192.168.217.119  Bcast:192.168.217.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fe77:8334/64 Scope:Link

[root@localhost ha.d]# ps aux |grep nginx
root      6044  0.0  0.1  15668  1460 ?        Ss   00:43   0:00 nginx: master process /usr/sbin/nginx -c /etc/nginx/nginx.conf
nginx     6046  0.0  0.1  15824  1908 ?        S    00:43   0:00 nginx: worker process
root      6064  0.0  0.0   5980   744 pts/0    S+   00:44   0:00 grep nginx
在主停掉heartbeat服务后从会接管服务。
测试脑裂：主和从上都down掉eth1网卡
[root@webserver ha.d]# ifdown eth1
[root@localhost ha.d]# ifdown eth1
[root@webserver ha.d]# ifconfig
eth0      Link encap:Ethernet  HWaddr 00:0C:29:E6:9E:DF
          inet addr:192.168.1.111  Bcast:192.168.1.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fee6:9edf/64 Scope:Link

eth0:0    Link encap:Ethernet  HWaddr 00:0C:29:E6:9E:DF
          inet addr:192.168.1.110  Bcast:192.168.1.255  Mask:255.255.255.0
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          Interrupt:19 Base address:0x2000

[root@webserver ha.d]# ps aux |grep nginx
root     17665  0.0  0.1  15668  1452 ?        Ss   00:28   0:00 nginx: master process /usr/sbin/nginx -c /etc/nginx/nginx.conf
nginx    17667  0.0  0.1  15824  1900 ?        S    00:28   0:00 nginx: worker process
root     18146  0.0  0.0   5980   744 pts/1    S+   00:30   0:00 grep nginx
[root@localhost ha.d]# ifconfig
eth0      Link encap:Ethernet  HWaddr 00:0C:29:77:83:2A
          inet addr:192.168.1.119  Bcast:192.168.1.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fe77:832a/64 Scope:Link

eth0:0    Link encap:Ethernet  HWaddr 00:0C:29:77:83:2A
          inet addr:192.168.1.110  Bcast:192.168.1.255  Mask:255.255.255.0
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          Interrupt:19 Base address:0x2000

[root@localhost ha.d]# ps aux |grep nginx
root      7095  0.0  0.1  15668  1216 ?        Ss   00:53   0:00 nginx: master process /usr/sbin/nginx -c /etc/nginx/nginx.conf
nginx     7097  0.0  0.1  15824  1796 ?        S    00:53   0:00 nginx: worker process
root      7141  0.0  0.0   5980   744 pts/0    S+   00:54   0:00 grep nginx
down掉eth1网卡后主和从的心跳线断掉，会导致脑裂现象，而从会认为主已死掉从而接管服务。
[root@webserver ha.d]# ifup eth1
Determining if ip address 192.168.217.111 is already in use for device eth1..
[root@localhost ha.d]# ifup eth1
Determining if ip address 192.168.217.119 is already in use for device eth1...
[root@webserver ha.d]# ifconfig
eth0      Link encap:Ethernet  HWaddr 00:0C:29:E6:9E:DF
          inet addr:192.168.1.111  Bcast:192.168.1.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fee6:9edf/64 Scope:Link

eth1      Link encap:Ethernet  HWaddr 00:0C:29:E6:9E:E9
          inet addr:192.168.217.111  Bcast:192.168.217.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fee6:9ee9/64 Scope:Link
[root@localhost ha.d]# ifconfig
eth0      Link encap:Ethernet  HWaddr 00:0C:29:77:83:2A
          inet addr:192.168.1.119  Bcast:192.168.1.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fe77:832a/64 Scope:Link

eth1      Link encap:Ethernet  HWaddr 00:0C:29:77:83:34
          inet addr:192.168.217.119  Bcast:192.168.217.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fe77:8334/64 Scope:Link
[root@webserver ha.d]# /etc/init.d/heartbeat restart
Stopping High-Availability services: Done.

Waiting to allow resource takeover to complete:Done.

Starting High-Availability services: INFO:  Resource is stopped
Done.

[root@webserver ha.d]# ifconfig
eth0      Link encap:Ethernet  HWaddr 00:0C:29:E6:9E:DF
          inet addr:192.168.1.111  Bcast:192.168.1.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fee6:9edf/64 Scope:Link

eth0:0    Link encap:Ethernet  HWaddr 00:0C:29:E6:9E:DF
          inet addr:192.168.1.110  Bcast:192.168.1.255  Mask:255.255.255.0
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          Interrupt:19 Base address:0x2000

eth1      Link encap:Ethernet  HWaddr 00:0C:29:E6:9E:E9
          inet addr:192.168.217.111  Bcast:192.168.217.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fee6:9ee9/64 Scope:Link
[root@localhost ha.d]# ifconfig
eth0      Link encap:Ethernet HWaddr 00:0C:29:77:83:2A
          inet addr:192.168.1.119 Bcast:192.168.1.255 Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fe77:832a/64 Scope:Link
eth1      Link encap:Ethernet HWaddr 00:0C:29:77:83:34
          inet addr:192.168.217.119 Bcast:192.168.217.255 Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fe77:8334/64 Scope:Link
打开两网卡后不能立即恢复服务，还需要重启heartbeat服务才能恢复正常运行。