使用DRBD和keepalived实现文件实时同步和双机热备

DRBD + keepalived实现文件实时同步和双机热备

安装DRBD

系统初始化设置

注意： 需要有数据盘或者多的分区

yum update -y

关闭防火墙

	systemctl stop firewalld
	systemctl disable firewalld

修改host文件

	vim /etc/hosts
	192.168.1.240 Primary kylin-01
	192.168.1.241 Secondary kylin-02

关闭SELINUX

	vim /etc/sysconfig/selinux
	SELINUX=disabled

安装依赖

yum install gcc libxslt-devel libxslt perl keyutils-libs-devel net-tools -y

下载源码编译安装

drbd需要两个安装报：drbd drbd-utils

drbd 安装

	wget https://pkg.linbit.com//downloads/drbd/9/drbd-9.2.8.tar.gz
	tar -zxvf drbd-9.2.8.tar.gz
	cd drbd-9.2.8
	make && make install

drbd-utils 安装

	wget https://pkg.linbit.com//downloads/drbd/utils/drbd-utils-9.27.0.tar.gz
	tar -zxvf drbd-utils-9.27.0.tar.gz
	cd drbd-utils-9.27.0
	./configure --prefix=/usr/local/drbd --without-83support --with-udev --with-initscripttype=systemd --without-manual
	make && make install

安装配置路径：/usr/local/drbd/etc/drbd.d 安装路径： /usr/sbin/drbdsetup /usr/sbin/drbdmeta /usr/sbin/drbdadm

配置drbd

磁盘分区 此处不要格式化磁盘 fdisk /dev/sdb

全局配置

global_common.conf内容如下：

	# DRBD is the result of over a decade of development by LINBIT.
	# In case you need professional services for DRBD or have
	# feature requests visit http://www.linbit.com
	global {
	usage-count yes;
	# Decide what kind of udev symlinks you want for "implicit" volumes
	# (those without explicit volume {} block, implied vnr=0):
	# /dev/drbd/by-resource// (explicit volumes)
	# /dev/drbd/by-resource/ (default for implict)
	udev-always-use-vnr; # treat implicit the same as explicit volumes
	# minor-count dialog-refresh disable-ip-verification
	# cmd-timeout-short 5; cmd-timeout-medium 121; cmd-timeout-long 600;
	}
	common {
	handlers {
	# These are EXAMPLE handlers only.
	# They may have severe implications,
	# like hard resetting the node under certain circumstances.
	# Be careful when choosing your poison.
	# IMPORTANT: most of the following scripts symlink to "notify.sh" which tries to send mail via "mail".
	# If you intend to use this notify.sh script make sure that "mail" is installed.
	#
	pri-on-incon-degr "/usr/lib/drbd/notify-pri-on-incon-degr.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
	pri-lost-after-sb "/usr/lib/drbd/notify-pri-lost-after-sb.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
	local-io-error "/usr/lib/drbd/notify-io-error.sh; /usr/lib/drbd/notify-emergency-shutdown.sh; echo o > /proc/sysrq-trigger ; halt -f";
	# fence-peer "/usr/lib/drbd/crm-fence-peer.sh";
	# split-brain "/usr/lib/drbd/notify-split-brain.sh root";
	# out-of-sync "/usr/lib/drbd/notify-out-of-sync.sh root";
	# before-resync-target "/usr/lib/drbd/snapshot-resync-target-lvm.sh -p 15 -- -c 16k";
	# after-resync-target /usr/lib/drbd/unsnapshot-resync-target-lvm.sh;
	# quorum-lost "/usr/lib/drbd/notify-quorum-lost.sh root";
	# disconnected /bin/true;
	}
	startup {
	# wfc-timeout degr-wfc-timeout outdated-wfc-timeout wait-after-sb
	}
	options {
	# cpu-mask on-no-data-accessible
	# RECOMMENDED for three or more storage nodes with DRBD 9:
	# quorum majority;
	# on-no-quorum suspend-io | io-error;
	}
	disk {
	on-io-error detach; #配置I/O错误处理策略为分离
	# size on-io-error fencing disk-barrier disk-flushes
	# disk-drain md-flushes resync-rate resync-after al-extents
	# c-plan-ahead c-delay-target c-fill-target c-max-rate
	# c-min-rate disk-timeout
	}
	net {
	# protocol timeout max-epoch-size max-buffers
	# connect-int ping-int sndbuf-size rcvbuf-size ko-count
	# allow-two-primaries cram-hmac-alg shared-secret after-sb-0pri
	# after-sb-1pri after-sb-2pri always-asbp rr-conflict
	# ping-timeout data-integrity-alg tcp-cork on-congestion
	# congestion-fill congestion-extents csums-alg verify-alg
	# use-rle
	}
	}

资源配置

在node1、node2 上分别建立drbd.res： drbd.d目录下新建资源文件（drbd.res） vim /usr/local/drbd/etc/drbd.d/drbd.res

	resource r1 { #这个r1是定义资源的名字
	protocol C;
	on kylin-01 { #on开头，后面是主机名称
	device /dev/drbd0; #drbd设备名称
	disk /dev/sdb1; #drbd0使用的磁盘分区为sdb1
	address 192.168.1.240:7789; #设置drbd监听地址与端口
	meta-disk internal;
	}
	on kylin-02 { #on开头，后面是主机名称
	device /dev/drbd0; #drbd设备名称
	disk /dev/sdb1; #drbd0使用的磁盘分区为sdb1
	address 192.168.1.241:7789; #设置drbd监听地址与端口
	meta-disk internal;
	}
	}

在node1、node2 上初始化资源、启动drbd：

	[root@kylin-01 drbd.d]# drbdadm create-md r1
	initializing activity log
	initializing bitmap (640 KB) to all zero
	Writing meta data...
	New drbd meta data block successfully created.
	#启动 两个节点需要同时启动才生效
	[root@kylin-01 drbd.d]# systemctl start drbd
	# 开机启动
	[root@kylin-01 drbd.d]# systemctl enable drbd
	# 查看状态
	[root@kylin-01 drbd.d]# systemctl status drbd
	# 查看状态
	netstat -anput|grep 7789
	tcp 0 0 192.168.1.240:33015 192.168.1.241:7789 ESTABLISHED -
	tcp 0 0 192.168.1.240:40897 192.168.1.241:7789 ESTABLISHED -

检查资源状态：

	#查看节点角色状态：
	drbdadm role r1
	注：第一次启动drbd时，两个drbd节点默认都处于Secondary状态

主节点执行设置主节点：

	#初始化资源
	drbdadm primary --force r1
	#查看资源状态
	drbdadm status r1
	r1 role:Primary
	disk:UpToDate
	kylin-02 role:Secondary
	replication:SyncSource peer-disk:Inconsistent done:6.16
	#查看同步状态
	cat /proc/drbd
	version: 9.2.8 (api:2/proto:86-122)
	GIT-hash: e163b05a76254c0f51f999970e861d72bb16409a build by root@kylin-01, 2024-04-08 1556
	Transports (api tcp (9.2.8)
	会把主机上的数据传到备机，开始会显示同步进度，过一会显示状态都是”实时”,表示数据同步完成了。接下来就可以使用DRBD了。

DRBD使用（测试）

你现在可以把主机上的DRBD设备挂载到一个目录上进行使用，备机的DRBD设备无法被挂载,因为它是用来接收主机数据的,由DRBD负责操作.

格式化文件系统（文件格式根据自己的系统环境选择）

mkfs.ext4 /dev/drbd0

挂载此文件系统

	mkdir /data
	mount /dev/drbd0 /data/

在挂载data目录中创建一个测试文件，然后卸载挂载目录，然后切换主备节点，在备用节点上查看刚刚建立的测试文件还是否存在

node1:

	[root@node1 ~]# mkdir /data/test
	将node1变为备用节点
	[root@node1 ~]# umount /data/
	[root@node1 ~]# drbdadm secondary r1
	[root@node1 ~]# drbdadm role r1
	Secondary

node2:

	将node2变为主节点
	[root@node2 ~]# drbdadm primary r1
	[root@node2 ~]# drbdadm role r1
	Primary/Secondary
	挂载设备，然后看文件是否存在
	[root@node2 ~]# mount /dev/drbd0 /data
	[root@node2 ~]# cd /data/
	[root@node2 mnt]# ls
	test
	OK! 到这里已经算是完成了！

同样，在Node2上建立文件，然后

卸载/mnt/：umount /mnt/

将Node2降级成备用节点：drbdadm secondary r1

在Node1上升级为主机节点：drbdadm primary r1

在Node1上挂载：mount /dev/drbd0 /data

会发现Node2上的文件也同步到了Node1上。

问题

1、umount时，如果提示device is busy，使用下面方法解决： fuser -m /data 显示：/data: 25023c 然后kill -9 25023 即可

2、"Split-Brain"（脑裂）的情况：

假设把Primary主机的的eth0设备宕掉，然后直接在Secondary主机上进行提权升级为DRBD的主节点，并且mount挂载DRBD，这时会发现之前在Primary主机上写入的数据文件确实同步过来了。 接着再把Primary主机的eth0设备恢复，看看有没有自动恢复 主从关系。经过查看，发现DRBD检测出了Split-Brain的状况，也就是两个节点都处于standalone状态， 故障描述如下：Split-Brain detected,dropping connection! 这就是传说中的“脑裂”。

DRBD官方推荐的手动恢复方案：

Secondary主机上的操作

	drbdadm secondary r0
	drbdadm disconnect all
	drbdadm --discard-my-data connect r0 //或者"drbdadm -- --discard-my-data connect r0"

Primary主机上的操作

	drbdadm disconnect all
	drbdadm connect r0
	drbdsetup /dev/drbd0 primary

检查drdb状态

	[root@kylin-01 ~]# drbdadm status r1
	r1 role:Primary
	disk:UpToDate
	kylin-02 role:Secondary
	peer-disk:UpToDate

安装keepalived

直接采用yum安装

yum install -y keepalived

查看keepalived版本

	[root@kylin-02 keepalived]# keepalived -v
	Keepalived v2.0.20 (01/22,2020)
	Copyright(C) 2001-2020 Alexandre Cassen,
	Built with kernel headers for Linux 4.19.90
	Running on Linux 4.19.90-52.25.v2207.ky10.x86_64 #1 SMP Fri Jun 2 1228 CST 2023
	..........

更改keepalived配置

注：本处采用pgsql测试两个机器数据同步

pg docker-compose的文件(docker的配置安装请自行百度)

	version: "3"
	services:
	postgresql:
	image: postgres:11.8
	container_name: postgres
	hostname: postgres
	ports:
	- "5432:5432"
	volumes:
	- "/data/pgsql:/var/lib/postgresql/data"
	- "/etc/localtime:/etc/localtime"
	restart: on-failure
	logging:
	driver: "json-file"
	options:
	tag: postgres
	cap_add:
	- ALL
	environment:
	POSTGRES_USER: "root"
	POSTGRES_PASSWORD: "123456"
	ALLOW_IP_RANGE: "0.0.0.0/0"

	cd /etc/keepalived
	vim keepalived.conf

主节点keepalived.conf 文件内容如下：

注意：再执行stop脚本时不能直接执行，需要采用脚本调用脚本的方式执行，否则执行不完就会被kill掉（原因暂时没弄清楚）

	! Configuration File for keepalived
	global_defs {
	#notification_email {
	# acassen@firewall.loc
	# failover@firewall.loc
	# sysadmin@firewall.loc
	#}
	#notification_email_from Alexandre.Cassen@firewall.loc
	#smtp_server 192.168.200.1
	#smtp_connect_timeout 30
	router_id kylin-02 # 节点标识，主机名
	vrrp_skip_check_adv_addr
	vrrp_strict
	vrrp_garp_interval 0
	vrrp_gna_interval 0
	}
	vrrp_instance VI_1 {
	state BACKUP
	interface ens32 # 网卡
	virtual_router_id 51
	priority 100 # 节点权重，主节点100 备节点小于100，数字越大优先级越高
	mcast_src_ip 192.168.1.240 # 本机IP
	advert_int 1
	authentication {
	auth_type PASS
	auth_pass 1111
	}
	virtual_ipaddress {
	192.168.1.239 # 绑定的虚拟IP
	}
	notify_master "/etc/keepalived/notify.sh" # 节点为master时执行脚本
	notify_backup "/etc/keepalived/notify_back.sh" # 切换为备节点时执行脚本
	notify_stop "/etc/keepalived/notify_back.sh" # stop keepalived时执行的脚本
	}

备节点keepalived.conf 文件内容如下：

	! Configuration File for keepalived
	global_defs {
	#notification_email {
	# acassen@firewall.loc
	# failover@firewall.loc
	# sysadmin@firewall.loc
	#}
	#notification_email_from Alexandre.Cassen@firewall.loc
	#smtp_server 192.168.200.1
	#smtp_connect_timeout 30
	router_id kylin-02 # 节点标识，主机名
	vrrp_skip_check_adv_addr
	vrrp_strict
	vrrp_garp_interval 0
	vrrp_gna_interval 0
	}
	vrrp_instance VI_1 {
	state BACKUP
	interface ens32
	virtual_router_id 51
	priority 99 # 节点权重，主节点100 备节点小于100，数字越大优先级越高
	mcast_src_ip 192.168.1.241 # 本机IP
	advert_int 1
	authentication {
	auth_type PASS
	auth_pass 1111
	}
	virtual_ipaddress {
	192.168.1.239
	}
	notify_master "/etc/keepalived/notify.sh"
	notify_backup "/etc/keepalived/notify_back.sh"
	notify_stop "/etc/keepalived/notify_back.sh"
	}

notify.sh

	#!/bin/bash
	drbdadm primary r1
	while true
	do
	drdbs=$(drbdadm role r1)
	echo "drbd status is $drdbs"
	if [[ "$drdbs" == "Primary" ]];then
	break
	else
	drbdadm primary r1
	sleep 3
	fi
	done
	mount /dev/drbd0 /data
	docker-compose -f /opt/pgsql/docker-compose.yml up -d

stop.sh

	#!/bin/bash
	docker stop postgres
	umount /data/
	drbdadm secondary r1
	while true
	do
	drdbs=$(drbdadm role r1)
	echo "drbd status is $drdbs"
	if [[ "$drdbs"=="Secondary" ]];then
	break
	else
	drbdadm secondary r1
	sleep 3
	fi
	done

notify_back.sh

	#!/bin/bash
	/etc/keepalived/stop.sh

启动keepalived

systemctl start keepalived

开机启动

systemctl enable keepalived

验证

在数据库里面执行创建删除，切换keepalived节点后查看数据是否同步

链接：https://www.cnblogs.com/pgyLang/p/18124303