[ClusterLabs] Antw: [EXT] DRBD split‑brain investigations, automatic fixes and manual intervention...

[Ian Diddams]

>>So you drive without safety-belt and airbag (read: fencing)?

possibly?  probably?

As I said Im flying blind with this all - I was asked to try and implement it, Ive tried the best I can to implement it but for all I know the how-tos and advice I found have nmissed what may be needed.

Ive looked for online tutorials  but have failed to come up with anything much aside from "do these commands and there you have it".  Which may not uinclude more belts and braces.

>>
you're asking the worng operson.  Im given two systems, whose disks are on a SAN.  Minbe is not to reason why etc.
>> I wondered where the cluster is in those logs.
sorry - Ive not understood the questin here.


happy to provide extracts form logs etc.  Below Ive appended the "set up" commands/steps used to implement drbd_pcs+corosync on the systems if that helps outloine any more.

I'm just the guy that fires the bullets in effect, trying to aim as best he can...

ian
====
# prep
umount /var/lib/mysql
  - and remove /var/lib/mysql from /etc/fstab
yum remove mysql-community-server
cd /var/lib/mysql; rm -rf *
mkdir /var/lib/mysql
chown mysql:mysql /var/lib/mysql
chmod 755 /var/lib/mysql
reboot

yum makecache fast
yum -y install wget mlocate telnet lsof
updatedb


# Install Pacemaker and Corosync
yum install -y pcs
yum install -y policycoreutils-python
echo "passwd" | passwd hacluster --stdin
systemctl start pcsd.service
systemctl enable pcsd.service

#Configure Corosync
#[estrela]]
pcs cluster auth estrela rafeiro -u hacluster -p passwd
pcs cluster setup --name mysql_cluster estrela rafeiro
pcs cluster start --all

## Install DRBD
## BOTH

yum install -y kmod-drbd90
yum install -y drbd90-utils
systemctl enable corosync
systemctl enable pacemaker
reboot

# when back up
modprobe drbd
systemctl status pcsd
systemctl status corosync
systemctl status pacemaker

cat << EOL >/etc/drbd.d/mysql01.res
resource mysql01 {
 protocol C;
 meta-disk internal;
 device /dev/drbd0;
 disk   /dev/vg_mysql/lv_mysql;
 handlers {
  split-brain "/usr/lib/drbd/notify-split-brain.sh root";
 }
 net {
  allow-two-primaries no;
  after-sb-0pri discard-zero-changes;
  after-sb-1pri discard-secondary;
  after-sb-2pri disconnect;
  rr-conflict disconnect;
 }
 disk {
  on-io-error detach;
 }
 syncer {
  verify-alg sha1;
 }
 on estrela {
  address  10.108.248.165:7789;
 }
 on rafeiro {
  address  10.108.248.166:7789;
 }
}
EOL

* clear previously created filesystem
dd if=/dev/zero of=/dev/vg_mysql/lv_mysql bs=1M count=128   


drbdadm create-md mysql01
systemctl start drbd
systemctl enable drbd
systemctl status drbd

#[estrela]
drbdadm primary --force mysql01
estrela:     cat /sys/kernel/debug/drbd/resources/mysql01/connections/rafeiro/0/proc_drbd
rafeiro:      cat /sys/kernel/debug/drbd/resources/mysql01/connections/estrela/0/proc_drbd
drbdadm status

# WAIT UNTIL DRBD IS SYNCED

#[estrela]
mkfs.xfs -f  -L drbd /dev/drbd0
mount /dev/drbd0 /mnt


## INSTALL MYSQL on all
## BOTH
yum install mysql-server -y

# [estrela]
mysql_install_db --datadir=/mnt --user=mysql
systemctl stop mysqld
umount /mnt

#BOTH  
cp -p /etc/my.cnf /etc/my.cnf.ORIG
set up my.cnf as needed (migtated from existing mysql server)

#BOTH
mv /var/lib/mysql /var/lib/mysql.orig
mkdir /var/lib/mysql
chown mysql:mysql /var/lib/mysql
chmod 751 /var/lib/mysql
mkdir  /var/lib/mysql/innodb
chown mysql:mysql /var/lib/mysql/innodb
chmod 755 /var/lib/mysql/innodb


# estrela
mount /dev/drbd0 /var/lib/mysql
systemctl start mysqld

# set up mysql
grep 'temporary password' /var/log/mysqld.log
mysql_secure_installation
rm /root/.mysql_secret


# set up grants
<grants etc>
flush privileges;

# test grants
[estrela]# mysql -uroot --skip-column-names -A -e"SELECT CONCAT('SHOW GRANTS FOR ''',user,'''@''',host,''';') FROM mysql.user WHERE user<>''" | mysql -uroot --skip-column-names -A | sed 's/$/;/g'
[rafeiro]# mysql -h estrela -uroot --skip-column-names -A -e"SELECT CONCAT('SHOW GRANTS FOR ''',user,'''@''',host,''';') FROM mysql.user WHERE user<>''" | mysql -hestrela -uroot --skip-column-names -A | sed 's/$/;/g'
mysql -h mysqldbdynabookHA -uroot --skip-column-names -A -e"SELECT CONCAT('SHOW GRANTS FOR ''',user,'''@''',host,''';') FROM mysql.user WHERE user<>''" | mysql -hestrela -uroot --skip-column-names -A | sed 's/$/;/g'
# stop test_200

# [estrela]
systemctl stop mysqld
umount /var/lib/mysql

# snapshot servers - pre-clustered

# Configure Pacemaker Cluster
# [estrela]
pcs cluster cib clust_cfg


pcs -f clust_cfg property set stonith-enabled=false
pcs -f clust_cfg property set no-quorum-policy=ignore
pcs -f clust_cfg resource defaults resource-stickiness=200
pcs -f clust_cfg resource create mysql_data01 ocf:linbit:drbd drbd_resource=mysql01 op monitor interval=30s
pcs -f clust_cfg resource master MySQLClone01 mysql_data01 master-max=1 master-node-max=1 clone-max=2 clone-node-max=1 notify=true
#pcs -f clust_cfg resource create mysql_fs01 Filesystem device="/dev/drbd0" directory="/var/lib/mysql" fstype="ext4"
pcs -f clust_cfg resource create mysql_fs01 Filesystem device="/dev/drbd0" directory="/var/lib/mysql" fstype="xfs"
pcs -f clust_cfg constraint colocation add mysql_fs01 with MySQLClone01 INFINITY with-rsc-role=Master
pcs -f clust_cfg constraint order promote MySQLClone01 then start mysql_fs01
pcs -f clust_cfg resource create mysql_service01 ocf:heartbeat:mysql \
  binary="/usr/sbin/mysqld" \
  config="/etc/my.cnf" \
  datadir="/var/lib/mysql" \
  socket="/var/lib/mysql/mysql.sock" \
  additional_parameters="--bind-address=0.0.0.0" \
  op start timeout=60s \
  op stop timeout=60s \
  op monitor interval=20s timeout=30s
pcs -f clust_cfg constraint colocation add mysql_service01 with mysql_fs01 INFINITY
pcs -f clust_cfg constraint order mysql_fs01 then mysql_service01
pcs -f clust_cfg resource create mysql_VIP01 ocf:heartbeat:IPaddr2 ip=10.108.248.164 cidr_netmask=32 op monitor interval=30s
pcs -f clust_cfg constraint colocation add mysql_VIP01 with mysql_service01 INFINITY
pcs -f clust_cfg constraint order mysql_service01 then mysql_VIP01

pcs -f clust_cfg constraint  
pcs -f clust_cfg resource show
pcs cluster cib-push clust_cfg
pcs status


#TEST service
[estrela]# mysql -uroot --skip-column-names -A -e"SELECT CONCAT('SHOW GRANTS FOR ''',user,'''@''',host,''';') FROM mysql.user WHERE user<>''" | mysql -uroot --skip-column-names -A | sed 's/$/;/g'
[rafeiro]# mysql -h estrela -uroot --skip-column-names -A -e"SELECT CONCAT('SHOW GRANTS FOR ''',user,'''@''',host,''';') FROM mysql.user WHERE user<>''" | mysql -hestrela -uroot --skip-column-names -A | sed 's/$/;/g'
mysql -hmysqldbdynabookHA -uroot --skip-column-names -A -e"SELECT CONCAT('SHOW GRANTS FOR ''',user,'''@''',host,''';') FROM mysql.user WHERE user<>''" | mysql -hmysqldbdynabookHA -uroot --skip-column-names -A | sed 's/$/;/g'


===

test with
pcs status
pcs cluster stop <hostname>
pcs status to check failover
above tests (grants etc) to vip address
cluster start and failover again etc.





  
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.clusterlabs.org/pipermail/users/attachments/20211020/34507ca0/attachment.htm>