一、环境
1、基础环境
A、系统：Centos6.3
B、iptables、selinux关闭
C、系统内核参数（负载均衡器、节点）
fs.file-max = 1000000
kernel.core_uses_pid = 1 kernel.msgmax = 1048560 kernel.msgmnb = 1073741824 kernel.shmall = 4294967296 kernel.shmmax = 68719476736 kernel.sysrq = 0 net.core.netdev_max_backlog = 8192 net.core.rmem_default = 2097152 net.core.rmem_max = 16777216 net.core.somaxconn = 8192 net.core.wmem_default = 2097152 net.core.wmem_max = 16777216 net.ipv4.conf.default.accept_source_route = 0 net.ipv4.conf.default.rp_filter = 1 net.ipv4.ip_forward = 1 net.ipv4.ip_local_port_range = 1024 65000 net.ipv4.neigh.default.gc_thresh1 = 10240 net.ipv4.neigh.default.gc_thresh2 = 40960 net.ipv4.neigh.default.gc_thresh3 = 81920 net.ipv4.tcp_fin_timeout = 1 net.ipv4.tcp_keepalive_intvl = 15 net.ipv4.tcp_keepalive_probes = 5 net.ipv4.tcp_keepalive_time = 30 net.ipv4.tcp_max_orphans = 3276800 net.ipv4.tcp_max_syn_backlog = 8192 net.ipv4.tcp_max_tw_buckets = 51200 net.ipv4.tcp_mem = 94500000 915000000 927000000 net.ipv4.tcp_orphan_retries = 3 net.ipv4.tcp_reordering = 5 net.ipv4.tcp_retrans_collapse = 0 net.ipv4.tcp_retries2 = 5 net.ipv4.tcp_rmem = 4096 87380 4194304 net.ipv4.tcp_sack = 1 net.ipv4.tcp_synack_retries = 1 net.ipv4.tcp_syncookies = 0 net.ipv4.tcp_syn_retries = 1 net.ipv4.tcp_timestamps = 1 net.ipv4.tcp_tw_recycle = 1 net.ipv4.tcp_tw_reuse = 1 net.ipv4.tcp_window_scaling = 1 net.ipv4.tcp_wmem = 4096 16384 4194304 net.ipv6.conf.all.disable_ipv6 = 1 net.ipv6.conf.default.disable_ipv6 = 1二、服务分布
负载均衡器master：
eth0：10.10.10.100（外网）
eth1:172.28.26.119（内网）
负载均衡器backup：
eth0：10.10.10.200（外网）
eth1:172.28.26.120（内网）
VIP：10.10.10.250
node1：
eth0：10.10.10.10（外网）
eth1:172.28.26.121（内网）
node2：
eth0：10.10.10.11（外网）
eth1:172.28.26.122（内网
三、负载均衡器keepalived的配置
[root@zh_lvs_backup ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalivedglobal_defs {
#报警功能 notification_email { } notification_email_from smtp_server 127.0.0.1 smtp_connect_timeout 30 router_id LVS_B_proxy_A }vrrp_instance VI_1 {
state MASTER (10.10.10.200为BACKUP) interface eth0 virtual_router_id 30 priority 100 (10.10.10.200为99) advert_int 1 authentication { auth_type PASS auth_pass 1111 } virtual_ipaddress { 10.10.10.250}
}#LVS功能
virtual_server 10.10.10.250 80 {
delay_loop 6 # lb_algo wrr lb_algo wlc lb_kind DR nat_mask 255.255.255.0 persistence_timeout 60 protocol TCPreal_server 172.28.26.121 80 {
weight 1 TCP_CHECK { connect_timeout 5 nb_get_retry 3 delay_before_retry 3 connect_port 80 } } real_server 172.28.26.122 80 { weight 1 TCP_CHECK { connect_timeout 5 nb_get_retry 3 delay_before_retry 3 connect_port 80 } }}
四、节点脚本
cat /sbin/lvs-client.sh
#!/bin/sh
#-------------------------------------------------------- # by Cooper 2012-06-16 for LVS (real server) #--------------------------------------------------------VirtualIP="10.10.10.250"
case "$1" in
start) ifconfig lo:0 $VirtualIP netmask 255.255.255.255 broadcast $VirtualIP up /sbin/route add -host $VirtualIP dev lo:0 echo "1" > /proc/sys/net/ipv4/ip_forward echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce sysctl -p >/dev/null 2>&1 echo "RealServer Start OK" ;; stop) ifconfig lo:0 down route del $VirtualIP echo "0" > /proc/sys/net/ipv4/ip_forward echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce sysctl -p >/dev/null 2>&1 echo "Realserver Stoped" ;; restart) sh $0 stop && sleep 2 && sh $0 start ;; *) echo "Usage: $0 {start|stop|restart}" exit 1 esac五、问题
客户端请求，负载均衡器也转发了，后端节点返回不了。
六、解决
节点的内核参数加入下面两条：
net.ipv4.conf.eth1.rp_filter = 0
net.ipv4.conf.all.rp_filter = 0eth1是内网网卡，，一般情况下，内网网卡来的数据，默认是内网回包，所以返回不了，这两个参数就是取消这个限制，内网网卡来的数据包外网网卡一可以回。