If you’re interested in running an NFSv4-only server in a
Pacemaker cluster on Debian,
you’ll probably find the default nfsserver RA in the
Debian-packaged resource agents
to be lacking.
It suffers from serveral issues:
The default behavior is to use the NFS init script to handle
starting/stopping/monitoring the NFS server. This is broken in the monitor
case as the service is reported as active even if it’s been killed.
Starting/stopping the service is dog slow. I’d venture to say at least
half of the total cluster failover time in my setup with the default RA was
consumed by starting/stopping the NFS daemon and all the related v2/v3
services.
So if you’re willing to go with NFSv4 only, here’s a much better approach:
Follow the Debian NFS server setup ,
including the adjustments under NFSv4 only .
Drop the below RA in a valid path for Pacemaker to find it, and make it
executable.
Configure your resource(s).
CAVEATS:
Does not work for v2/v3, and will not manage any of the services that are
v2/v3 only.
Only tested on Debian Buster, YMMV.
#!/bin/sh
# nfsserver
#
# Description: Manages NFS server as OCF resource
# by hxinwei@gmail.com
# NFSv4-only modifications by thehunmonkgroup at gmail dot com
# License: GNU General Public License v2 (GPLv2) and later
if [ -n "$OCF_DEBUG_LIBRARY" ]; then
. $OCF_DEBUG_LIBRARY
else
: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
fi
if is_redhat_based; then
. ${OCF_FUNCTIONS_DIR}/nfsserver-redhat.sh
fi
DEFAULT_RPCPIPEFS_DIR="/var/lib/nfs/rpc_pipefs"
nfsserver_meta_data() {
cat <<END
<?xml version="1.0"?>
<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
<resource-agent name="nfsserver">
<version>1.0</version>
<longdesc lang="en">
Nfsserver helps one to manage the Linux nfs server as a failover-able resource in Linux-HA.
It depends on Linux specific NFS implementation details, so is considered not portable to other platforms yet.
</longdesc>
<shortdesc lang="en">Manages an NFS server</shortdesc>
<parameters>
<parameter name="nfs_ip" unique="0" required="0">
<longdesc lang="en">
Comma separated list of floating IP addresses used to access the nfs service
</longdesc>
<shortdesc lang="en">
IP addresses.
</shortdesc>
<content type="string"/>
</parameter>
<parameter name="nfs_shared_infodir" unique="0" required="0">
<longdesc lang="en">
The nfsserver resource agent will save nfs related information in this specific directory.
And this directory must be able to fail-over before nfsserver itself.
</longdesc>
<shortdesc lang="en">
Directory to store nfs server related information.
</shortdesc>
<content type="string" default="" />
</parameter>
<parameter name="rpcpipefs_dir" unique="0" required="0">
<longdesc lang="en">
The mount point for the sunrpc file system. Default is $DEFAULT_RPCPIPEFS_DIR.
This script will mount (bind) nfs_shared_infodir on /var/lib/nfs/ (cannot be changed),
and this script will mount the sunrpc file system on $DEFAULT_RPCPIPEFS_DIR (default, can be changed by this parameter).
If you want to move only rpc_pipefs/ (e.g. to keep rpc_pipefs/ local) from default, please set this value.
</longdesc>
<shortdesc lang="en">
The mount point for the sunrpc file system.
</shortdesc>
<content type="string" default="$DEFAULT_RPCPIPEFS_DIR" />
</parameter>
</parameters>
<actions>
<action name="start" timeout="40s" />
<action name="stop" timeout="20s" />
<action name="monitor" depth="0" timeout="20s" interval="10s" />
<action name="meta-data" timeout="5s" />
<action name="validate-all" timeout="30s" />
</actions>
</resource-agent>
END
return $OCF_SUCCESS
}
nfsserver_usage() {
cat <<END
usage: $0 {start|stop|monitor|status|validate-all|meta-data}
END
}
if [ $# -ne 1 ]; then
nfsserver_usage
exit $OCF_ERR_ARGS
fi
case $__OCF_ACTION in
meta-data) nfsserver_meta_data
exit $OCF_SUCCESS
;;
usage|help) nfsserver_usage
exit $OCF_SUCCESS
;;
*)
;;
esac
fp="$OCF_RESKEY_nfs_shared_infodir"
: ${OCF_RESKEY_rpcpipefs_dir="$DEFAULT_RPCPIPEFS_DIR"}
OCF_RESKEY_rpcpipefs_dir=${OCF_RESKEY_rpcpipefs_dir%/}
##
# wrapper for init script and systemd calls.
##
nfs_exec()
{
local cmd=$1
local svc=$2
if ! echo $svc | grep -q "\."; then
svc="${svc}.service"
fi
systemctl $cmd $svc
}
nfsserver_systemd_monitor()
{
local threads_num
local rc
nfs_exec is-active nfs-server
rc=$?
# Now systemctl is-active can't detect the failure of kernel process like nfsd.
# So, if the return value of systemctl is-active is 0, check the threads number
# to make sure the process is running really.
# /proc/fs/nfsd/threads has the numbers of the nfsd threads.
if [ $rc -eq 0 ]; then
threads_num=`cat /proc/fs/nfsd/threads 2>/dev/null`
if [ $? -eq 0 ]; then
if [ $threads_num -gt 0 ]; then
return $OCF_SUCCESS
else
return 3
fi
else
return $OCF_ERR_GENERIC
fi
fi
return $rc
}
nfsserver_monitor ()
{
local fn
fn=`mktemp`
nfsserver_systemd_monitor > $fn 2>&1
rc=$?
ocf_log debug "$(cat $fn)"
rm -f $fn
#Adapte LSB status code to OCF return code
if [ $rc -eq 0 ]; then
return $rc
elif [ $rc -eq 3 ] || [ $rc -eq $OCF_NOT_RUNNING ]; then
return $OCF_NOT_RUNNING
else
return $OCF_ERR_GENERIC
fi
}
prepare_directory ()
{
if [ -z "$fp" ]; then
return
fi
[ -d "$OCF_RESKEY_rpcpipefs_dir" ] || mkdir -p $OCF_RESKEY_rpcpipefs_dir
[ -d "$fp/v4recovery" ] || mkdir -p $fp/v4recovery
[ -f "$fp/etab" ] || touch "$fp/etab"
[ -f "$fp/xtab" ] || touch "$fp/xtab"
[ -f "$fp/rmtab" ] || touch "$fp/rmtab"
}
is_bound ()
{
if mount | grep -q "on $1 type"; then
return 0
fi
return 1
}
bind_tree ()
{
if [ -z "$fp" ]; then
return
fi
[ -d "$fp" ] || mkdir -p $fp
if is_bound /var/lib/nfs; then
ocf_log debug "$fp is already bound to /var/lib/nfs"
return 0
fi
if nfs_exec status var-lib-nfs-rpc_pipefs.mount > /dev/null 2>&1; then
ocf_log debug "/var/lib/nfs/rpc_pipefs already mounted. Unmounting in preparation to bind mount nfs dir"
systemctl stop var-lib-nfs-rpc_pipefs.mount
fi
mount --bind $fp /var/lib/nfs
}
unbind_tree ()
{
local i=1
while `mount | grep -q " on $OCF_RESKEY_rpcpipefs_dir "` && [ "$i" -le 10 ]; do
ocf_log info "Stop: umount ($i/10 attempts)"
umount -t rpc_pipefs $OCF_RESKEY_rpcpipefs_dir
sleep 1
i=$((i + 1))
done
if is_bound /var/lib/nfs; then
umount /var/lib/nfs
fi
}
binary_status()
{
local binary=$1
local pid
pid=$(pgrep ${binary})
case $? in
0)
echo "$pid"
return $OCF_SUCCESS;;
1)
return $OCF_NOT_RUNNING;;
*)
return $OCF_ERR_GENERIC;;
esac
}
terminate()
{
local pids
local i=0
while : ; do
pids=$(binary_status $1)
[ -z "$pids" ] && return 0
kill $pids
sleep 1
i=$((i + 1))
[ $i -gt 3 ] && return 1
done
}
killkill()
{
local pids
local i=0
while : ; do
pids=$(binary_status $1)
[ -z "$pids" ] && return 0
kill -9 $pids
sleep 1
i=$((i + 1))
[ $i -gt 3 ] && return 1
done
}
stop_process()
{
local process=$1
ocf_log info "Stopping $process"
if terminate $process; then
ocf_log debug "$process is stopped"
else
if killkill $process; then
ocf_log debug "$process is stopped"
else
ocf_log debug "Failed to stop $process"
return 1
fi
fi
return 0
}
nfsserver_start ()
{
local rc;
local fn
if nfsserver_monitor; then
ocf_log debug "NFS server is already started"
return $OCF_SUCCESS
fi
bind_tree
prepare_directory
if ! `mount | grep -q " on $OCF_RESKEY_rpcpipefs_dir "`; then
mount -t rpc_pipefs sunrpc $OCF_RESKEY_rpcpipefs_dir
fi
ocf_log info "Starting NFS server ..."
# mounts /proc/fs/nfsd for us
lsmod | grep -q nfsd
if [ $? -ne 0 ]; then
modprobe nfsd
fi
fn=`mktemp`
nfs_exec start nfs-server > $fn 2>&1
rc=$?
ocf_log debug "$(cat $fn)"
rm -f $fn
if [ $rc -ne 0 ]; then
ocf_exit_reason "Failed to start NFS server"
return $rc
fi
tfn="/proc/fs/nfsd/threads"
if [ ! -f "$tfn" ] || [ "$(cat $tfn)" -le "0" ]; then
ocf_exit_reason "Failed to start NFS server: /proc/fs/nfsd/threads"
return $OCF_ERR_GENERIC
fi
ocf_log info "NFS server started"
return $OCF_SUCCESS
}
nfsserver_stop ()
{
local fn
ocf_log info "Stopping NFS server ..."
fn=`mktemp`
nfs_exec stop nfs-server > $fn 2>&1
rc=$?
ocf_log debug "$(cat $fn)"
rm -f $fn
if [ $rc -ne 0 ]; then
ocf_exit_reason "Failed to stop NFS server"
return $rc
fi
ocf_log info "Stop: threads"
tfn="/proc/fs/nfsd/threads"
while [ -f "$tfn" ] && [ "$(cat $tfn)" -gt "0" ]; do
ocf_log err "NFS server failed to stop: /proc/fs/nfsd/threads"
sleep 1
done
nfs_exec stop rpc-gssd > /dev/null 2>&1
ocf_log info "Stop: rpc-gssd"
unbind_tree
ocf_log info "NFS server stopped"
return 0
}
nfsserver_validate ()
{
if [ -n "$OCF_RESKEY_CRM_meta_clone" ] && [ -n "$OCF_RESKEY_nfs_shared_infodir" ]; then
ocf_exit_reason "This RA does not support clone mode when a shared info directory is in use."
exit $OCF_ERR_CONFIGURED
fi
return $OCF_SUCCESS
}
nfsserver_validate
case $__OCF_ACTION in
start) nfsserver_start
;;
stop) nfsserver_stop
;;
monitor) nfsserver_monitor
;;
validate-all) exit $OCF_SUCCESS
;;
*) nfsserver_usage
exit $OCF_ERR_UNIMPLEMENTED
;;
esac