blob: 64c5da4cd66d81d1e0379c975b56be7d51d4eda9 [file]
#!/usr/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -eo pipefail
# CloudStack B&R NAS Backup and Recovery Tool for KVM
# TODO: do libvirt/logging etc checks
### Declare variables ###
OP=""
VM=""
NAS_TYPE=""
NAS_ADDRESS=""
MOUNT_OPTS=""
BACKUP_DIR=""
DISK_PATHS=""
QUIESCE=""
QUIESCE_TIMEOUT="" # Seconds to wait for the guest agent freeze/thaw; empty => libvirt default
# Incremental backup parameters (all optional; legacy callers omit them)
MODE="" # "full" or "incremental"; empty => legacy full-only behavior (no checkpoint created)
METHOD="qcow2" # How the backup is taken: "qcow2" (push mode, incrementals from dirty bitmaps) or
# "content" (pull mode, incrementals by content comparison; raw block storage)
BITMAP_NEW="" # Bitmap/checkpoint name to create with this backup (e.g. "backup-1711586400")
BITMAP_PARENT="" # For incremental: parent bitmap name to read changes since
PARENT_PATHS="" # For incremental: comma-separated list of parent backup file paths,
# one per VM volume in the same order as DISK_PATHS. Each new qcow2
# is rebased onto its corresponding parent file. Required because
# data-disk backup files don't share the root volume's UUID, so
# each disk must be rebased onto its own parent.
SCRATCH_DEVS="" # Pull mode: comma-separated list of copy-before-write scratch block devices, one
# per VM volume in the same order as DISK_PATHS, created by the agent on the
# VM's primary storage and at least as large as the disk plus qcow2 metadata.
PULL_WORK_PARENT="/var/tmp" # Parent dir for the pull-mode work dir (NBD socket + backup XML)
logFile="/var/log/cloudstack/agent/agent.log"
UNMOUNT_TIMEOUT=60
EXIT_CLEANUP_FAILED=20
BACKUP_JOB_ACTIVE=0
PULL_WORK_DIR="" # Work dir of a running pull-mode backup, removed by on_pull_backup_exit()
log() {
[[ "$verb" -eq 1 ]] && builtin echo "$@"
if [[ "$1" == "-ne" || "$1" == "-e" || "$1" == "-n" ]]; then
builtin echo -e "$(date '+%Y-%m-%d %H-%M-%S>')" "${@: 2}" >> "$logFile"
else
builtin echo "$(date '+%Y-%m-%d %H-%M-%S>')" "$@" >> "$logFile"
fi
}
vercomp() {
local IFS=.
local i ver1=($1) ver2=($3)
# Compare each segment of the version numbers
for ((i=0; i<${#ver1[@]}; i++)); do
if [[ -z ${ver2[i]} ]]; then
ver2[i]=0
fi
if ((10#${ver1[i]} > 10#${ver2[i]})); then
return 0 # Version 1 is greater
elif ((10#${ver1[i]} < 10#${ver2[i]})); then
return 2 # Version 2 is greater
fi
done
return 0 # Versions are equal
}
sanity_checks() {
hvVersion=$(virsh version | grep hypervisor | awk '{print $(NF)}')
libvVersion=$(virsh version | grep libvirt | awk '{print $(NF)}' | tail -n 1)
apiVersion=$(virsh version | grep API | awk '{print $(NF)}')
# Compare qemu version (hvVersion >= 4.2.0)
vercomp "$hvVersion" ">=" "4.2.0"
hvStatus=$?
# Compare libvirt version (libvVersion >= 7.2.0)
vercomp "$libvVersion" ">=" "7.2.0"
libvStatus=$?
if [[ $hvStatus -eq 0 && $libvStatus -eq 0 ]]; then
log -ne "Success... [ QEMU: $hvVersion Libvirt: $libvVersion apiVersion: $apiVersion ]"
else
echo "Failure... Your QEMU version $hvVersion or libvirt version $libvVersion is unsupported. Consider upgrading to the required minimum version of QEMU: 4.2.0 and Libvirt: 7.2.0"
exit 1
fi
log -ne "Environment Sanity Checks successfully passed"
}
### Operation methods ###
get_ceph_uuid_from_path() {
local fullpath="$1"
# disk for rbd => rbd:<pool>/<uuid>:mon_host=<monitor_host>...
# sample: rbd:cloudstack/53d5c355-d726-4d3e-9422-046a503a0b12:mon_host=10.0.1.2...
local beforeUuid="${fullpath#*/}" # Remove up to first slash after rbd:
local volUuid="${beforeUuid%%:*}" # Remove everything after colon to get the uuid
echo ""$volUuid""
}
get_linstor_uuid_from_path() {
local fullpath="$1"
# disk for linstor => /dev/drbd/by-res/cs-<uuid>/0
# sample: /dev/drbd/by-res/cs-53d5c355-d726-4d3e-9422-046a503a0b12/0
local beforeUuid="${fullpath#/dev/drbd/by-res/}"
local volUuid="${beforeUuid%%/*}"
volUuid="${volUuid#cs-}"
echo "$volUuid"
}
get_linstor_uuid_from_device() {
local fullpath="$1"
# VMs started before the /dev/drbd/by-res/ change still reference the raw DRBD
# device node (e.g. /dev/drbd1098) in their live libvirt XML. Ask udev for the
# device's symlinks and map it back to the volume UUID via the by-res symlink.
local link
for link in $(udevadm info --query=symlink --name="$fullpath" 2>/dev/null || true); do
if [[ "$link" == drbd/by-res/cs-* ]]; then
get_linstor_uuid_from_path "/dev/$link"
return 0
fi
done
# Without a by-res symlink we cannot derive the volume UUID. Falling back to the
# raw device name would produce a backup that restore cannot find, so fail hard.
return 1
}
qemu_agent_command() {
local timeout_args=()
if [[ -n "$QUIESCE_TIMEOUT" ]]; then
timeout_args=(--timeout "$QUIESCE_TIMEOUT")
fi
virsh -c qemu:///system qemu-agent-command "$VM" "${timeout_args[@]}" "$1"
}
guest_fsfreeze_status() {
local response
response=$(qemu_agent_command '{"execute":"guest-fsfreeze-status"}' 2>>"$logFile") || return 1
grep -oE 'thawed|frozen' <<< "$response" | head -n 1
}
guest_agent_connected() {
local xml
xml=$(virsh -c qemu:///system dumpxml "$VM" 2>/dev/null) || return 1
grep -q "name='org.qemu.guest_agent.0' state='connected'" <<< "$xml"
}
# Thaws the guest filesystems. A thaw that times out on the host can still complete in the
# guest, so each failure is checked against guest-fsfreeze-status before retrying. Succeeds
# once the guest reports thawed, or when the freeze failed and the guest agent is not
# connected, which means the freeze never reached the guest.
thaw_guest() {
local freeze_ok=$1
local attempt response state
for ((attempt = 1; attempt <= 3; attempt++)); do
if response=$(qemu_agent_command '{"execute":"guest-fsfreeze-thaw"}' 2>&1); then
return 0
fi
log -e "Thaw attempt $attempt for vm $VM failed: $response"
state=$(guest_fsfreeze_status || true)
if [[ "$state" == "thawed" ]]; then
log -e "Guest filesystem of vm $VM reports thawed after the failed thaw"
return 0
fi
if [[ -z "$state" && $freeze_ok -eq 0 ]] && ! guest_agent_connected; then
log -e "Guest agent of vm $VM is not connected, so the failed freeze never reached the guest"
return 0
fi
sleep 5
done
echo "Failed to thaw the filesystem for vm $VM, guest filesystem state: ${state:-unknown}: $response"
return 1
}
# ---------------------------------------------------------------------------
# Content-based backup path (libvirt pull mode).
#
# Used for raw block-device disks such as LINSTOR/DRBD. Those cannot carry QEMU
# persistent dirty bitmaps -- persistence is a qcow2-only feature -- so libvirt
# checkpoints, and with them the push-mode incremental path, are unavailable.
#
# Instead libvirt is asked for a pull-mode backup: it starts an NBD server exposing a
# point-in-time view of each disk, held consistent by copy-before-write into a scratch
# block device per disk, which the agent creates on the VM's primary storage and libvirt
# formats as qcow2. The backup is then derived from that export:
#
# full qemu-img convert -> sparse standalone qcow2. Zero detection happens
# while streaming, so the fully allocated push-mode output and its
# separate re-convert pass are both avoided.
# incremental a qcow2 overlay backed by the NBD export, then a SAFE rebase onto the
# parent backup. That merges exactly the clusters which differ between
# export and parent into the overlay, yielding a delta qcow2 backed by the
# parent -- the same chain shape the push-mode incremental produces, so
# restore/delete are unchanged.
#
# Unlike a storage-snapshot based diff there is no trailing DRBD metadata to clip: the
# DRBD device, and therefore the NBD export, is already net-sized.
#
# Assumes mount_operation() already ran, so $dest and $mount_point are set.
# ---------------------------------------------------------------------------
# EXIT trap of the pull-mode path, armed once the work dir exists and cleared once the
# backup is complete. Any exit in between -- an explicit failure, an unexpected error under
# "set -e", or SIGINT/SIGTERM/SIGHUP -- has to end the pull job: unlike a push job it never
# finishes on its own, so it would keep the NBD server and the scratch devices busy and
# block every later backup or migration of the VM. cleanup() aborts the job (via
# BACKUP_JOB_ACTIVE) before removing the partial backup. SIGKILL cannot be trapped, so the
# agent wrapper aborts the job itself when the script times out.
on_pull_backup_exit() {
local rc=$?
trap - EXIT INT TERM HUP
[[ $rc -ne 0 ]] || rc=1
# The NBD socket belongs to the job: only remove it once the job is gone.
if abort_backup_job; then
rm -rf "$PULL_WORK_DIR"
fi
cleanup
exit $rc
}
backup_running_vm_pull() {
local effective_mode="$1"
# Work dir for the NBD socket and the backup XML. 711 lets the qemu process traverse in
# to the socket.
local work_dir
work_dir=$(mktemp -d "$PULL_WORK_PARENT/csbackup-pull.XXXXXX") || {
echo "Failed to create work directory under $PULL_WORK_PARENT"; cleanup; exit 1;
}
chmod 711 "$work_dir"
PULL_WORK_DIR="$work_dir"
trap on_pull_backup_exit EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
local nbd_sock="$work_dir/nbd.sock"
local backupxml="$work_dir/backup.xml"
# Snapshot the disk list once so the backup XML and the read-back loop stay in sync.
local -a disks
mapfile -t disks < <(
virsh -c qemu:///system domblklist "$VM" --details 2>/dev/null | awk '$2=="disk"{print $3, $4}'
)
if [[ ${#disks[@]} -eq 0 ]]; then
echo "No disks found for vm $VM"
exit 1
fi
# Copy-before-write saves each guest block overwritten during the backup into the disk's
# scratch device, at most once per block. The agent sizes each device for the whole disk,
# so it cannot fill up and pause the VM.
local -a scratch_devs=()
IFS=',' read -ra scratch_devs <<< "$SCRATCH_DEVS"
if [[ ${#scratch_devs[@]} -ne ${#disks[@]} ]]; then
echo "Pull-mode backup of vm $VM needs one scratch device per disk:" \
"got ${#scratch_devs[@]} for ${#disks[@]} disks"
exit 1
fi
local scratch_dev
for scratch_dev in "${scratch_devs[@]}"; do
if [[ ! -b "$scratch_dev" ]]; then
echo "Scratch device $scratch_dev for the backup of vm $VM is not a block device"
exit 1
fi
done
# Resolve each disk's parent backup up front. PARENT_PATHS is comma separated, one entry
# per volume in the same order as the disk list. Anything missing degrades this run to a
# full backup rather than failing it, mirroring the push-mode fallback.
local -a parent_paths_arr=()
if [[ "$effective_mode" == "incremental" ]]; then
if [[ -z "$PARENT_PATHS" ]]; then
log -e "content incremental: no parent paths supplied -- falling back to full"
echo "INCREMENTAL_FALLBACK=true"
effective_mode="full"
else
IFS=',' read -ra parent_paths_arr <<< "$PARENT_PATHS"
if [[ ${#parent_paths_arr[@]} -lt ${#disks[@]} ]]; then
log -e "content incremental: parent path list shorter than disk list -- falling back to full"
echo "INCREMENTAL_FALLBACK=true"
effective_mode="full"
else
local pp
for pp in "${parent_paths_arr[@]}"; do
if [[ ! -f "$mount_point/$pp" ]]; then
log -e "content incremental: parent $mount_point/$pp missing on NAS -- falling back to full"
echo "INCREMENTAL_FALLBACK=true"
effective_mode="full"
break
fi
done
fi
fi
fi
# Pull-mode backup XML: one NBD export and one scratch device per disk.
{
echo "<domainbackup mode='pull'>"
echo "<server transport='unix' socket='$nbd_sock'/>"
echo "<disks>"
local entry idx=0
for entry in "${disks[@]}"; do
echo "<disk name='${entry%% *}' backup='yes' type='block' backupmode='full'><scratch dev='${scratch_devs[$idx]}'/></disk>"
idx=$((idx + 1))
done
echo "</disks></domainbackup>"
} > "$backupxml"
local thaw=0
local freeze_ok=0
if [[ ${QUIESCE} == "true" ]]; then
# Always thaw after a freeze attempt, see backup_running_vm().
thaw=1
if freeze_err=$(qemu_agent_command '{"execute":"guest-fsfreeze-freeze"}' 2>&1 > /dev/null); then
freeze_ok=1
else
log -e "Failed to freeze the filesystem for vm $VM, continuing without quiescing: $freeze_err"
fi
fi
# backup-begin establishes the point-in-time and starts the NBD server.
local backup_begin=0
local backup_out
if backup_out=$(virsh -c qemu:///system backup-begin --domain "$VM" --backupxml "$backupxml" 2>&1); then
backup_begin=1
BACKUP_JOB_ACTIVE=1
fi
if [[ $thaw -eq 1 ]] && ! thaw_guest "$freeze_ok"; then
exit 1
fi
if [[ $backup_begin -ne 1 ]]; then
echo "Failed to start pull-mode backup for $VM: $backup_out"
exit 1
fi
# Backup domain information
virsh -c qemu:///system dumpxml $VM > $dest/domain-config.xml 2>/dev/null
virsh -c qemu:///system dominfo $VM > $dest/dominfo.xml 2>/dev/null
virsh -c qemu:///system domiflist $VM > $dest/domiflist.xml 2>/dev/null
virsh -c qemu:///system domblklist $VM > $dest/domblklist.xml 2>/dev/null
local name="root"
local disk_idx=0
local entry disk fullpath volUuid output export_uri parent_abs parent_rel
for entry in "${disks[@]}"; do
disk="${entry%% *}"
fullpath="${entry#* }"
if [[ "$fullpath" == /dev/drbd/by-res/* ]]; then
volUuid=$(get_linstor_uuid_from_path "$fullpath")
elif [[ "$fullpath" == /dev/drbd[0-9]* ]]; then
if ! volUuid=$(get_linstor_uuid_from_device "$fullpath"); then
echo "Failed to resolve LINSTOR volume UUID for $fullpath"
exit 1
fi
else
volUuid="${fullpath##*/}"
fi
output="$dest/$name.$volUuid.qcow2"
# The export name defaults to the disk target (e.g. vda) and presents raw bytes.
export_uri="nbd+unix:///$disk?socket=$nbd_sock"
if [[ "$effective_mode" == "incremental" ]]; then
parent_abs="$mount_point/${parent_paths_arr[$disk_idx]}"
parent_rel=$(realpath --relative-to="$dest" "$parent_abs")
# Overlay over the export, safe-rebase onto the parent to merge the differing
# clusters in, then a metadata-only rebase to store the parent as a relative path
# so the chain survives being mounted at a different mount point.
if ! qemu-img create -f qcow2 -b "$export_uri" -F raw "$output" >> "$logFile" 2> >(cat >&2) \
|| ! qemu-img rebase -b "$parent_abs" -F qcow2 "$output" >> "$logFile" 2> >(cat >&2) \
|| ! qemu-img rebase -u -b "$parent_rel" -F qcow2 "$output" >> "$logFile" 2> >(cat >&2); then
echo "qemu-img incremental delta failed for $disk -> $output"
exit 1
fi
else
# Zero detection while streaming gives a sparse qcow2 in a single pass.
if ! qemu-img convert -f raw -O qcow2 "$export_uri" "$output" >> "$logFile" 2> >(cat >&2); then
echo "qemu-img convert failed for pull export $disk -> $output"
exit 1
fi
fi
name="datadisk"
disk_idx=$((disk_idx + 1))
done
# End the pull job. Aborting the active job is how a pull backup finishes (there is no
# backup-end command): libvirt tears down the NBD server and releases the scratch devices,
# which the agent then deletes. The data already pulled is a complete backup, so from here
# on a failure must no longer discard it.
trap - EXIT INT TERM HUP
if ! abort_backup_job; then
log -e "warning: failed to end (domjobabort) backup job for $VM"
fi
rm -rf "$work_dir"
sync
# Statistics -- the size on the last stdout line is parsed by the Java wrapper.
du -sb $dest | cut -f1
umount $mount_point
rmdir $mount_point
}
backup_running_vm() {
mount_operation
mkdir -p "$dest" || { echo "Failed to create backup directory $dest"; exit 1; }
# Determine effective mode for this run.
# Legacy callers (no -M argument) get the original full-only behavior with no checkpoint.
# The Java wrapper (LibvirtTakeBackupCommandWrapper) pre-validates required args before
# invoking the script; the case below is a defensive fallback for direct invocations.
local effective_mode="${MODE:-legacy-full}"
local make_checkpoint=0
case "$METHOD" in
qcow2)
;;
content)
# Raw block-device storage (LINSTOR/DRBD) cannot carry persistent dirty bitmaps,
# so checkpoints are unavailable. Use the content-based pull-mode path instead.
case "$effective_mode" in
full|incremental)
backup_running_vm_pull "$effective_mode"
return
;;
*)
echo "Mode $effective_mode is not supported by the content method"
cleanup
exit 1
;;
esac
;;
*)
echo "Unknown method: $METHOD"
cleanup
exit 1
;;
esac
case "$effective_mode" in
incremental|full)
make_checkpoint=1
;;
legacy-full)
make_checkpoint=0
;;
*)
echo "Unknown mode: $effective_mode"
cleanup
exit 1
;;
esac
# Incremental needs the parent checkpoint registered with libvirt. CloudStack rebuilds the
# domain XML on every VM start, wiping libvirt's checkpoint registry while the dirty bitmap
# persists on the qcow2, so a fresh checkpoint-create fails with "Bitmap already exists".
# Re-register the parent with --redefine (needs only a name + creationTime) via a minimal
# synthesized XML. If the parent bitmap is missing from the qcow2 (e.g. after a migration),
# fall back to a full backup instead of letting backup-begin fail below.
if [[ "$effective_mode" == "incremental" ]]; then
# The parent bitmap must be present on EVERY disk, not just one. A snapshot restore or partial
# migration can wipe it on some disks; require it on all by comparing the disk count to the
# number of disks that carry it.
disk_count=$(virsh -c qemu:///system domblklist "$VM" --details 2>/dev/null | awk '$2=="disk"{c++} END{print c+0}')
# Count per-device (one per inserted.file whose dirty-bitmaps holds the parent), mirroring
# getVmDiskPathHasFromCheckpointMap(): query-block lists a bitmap under multiple nodes, so raw
# name matches double-count. "|| echo 0" keeps a no-match from aborting under "set -eo pipefail"
# before the fallback runs.
bitmap_count=$(virsh -c qemu:///system qemu-monitor-command "$VM" '{"execute":"query-block"}' 2>/dev/null | python3 -c '
import sys, json
target = sys.argv[1]
try:
data = json.load(sys.stdin)
except Exception:
print(0); sys.exit(0)
files = set()
for dev in data.get("return", []) or []:
inserted = dev.get("inserted") or {}
f = inserted.get("file")
if not f:
continue
if any((b or {}).get("name") == target for b in (inserted.get("dirty-bitmaps") or [])):
files.add(f)
print(len(files))
' "$BITMAP_PARENT" 2>/dev/null || echo 0)
if [[ "$disk_count" -eq 0 || "$bitmap_count" -lt "$disk_count" ]]; then
log -e "incremental: parent bitmap $BITMAP_PARENT present on $bitmap_count/$disk_count disk(s) — falling back to full"
echo "INCREMENTAL_FALLBACK=true"
effective_mode="full"
fi
fi
if [[ "$effective_mode" == "incremental" ]]; then
if ! virsh -c qemu:///system checkpoint-list "$VM" --name 2>/dev/null | grep -qx "$BITMAP_PARENT"; then
redefine_xml=$(mktemp)
printf '<domaincheckpoint><name>%s</name><creationTime>%s</creationTime></domaincheckpoint>' \
"$BITMAP_PARENT" "$(date +%s)" > "$redefine_xml"
if virsh -c qemu:///system checkpoint-create "$VM" --xmlfile "$redefine_xml" --redefine > /dev/null 2>&1; then
rm -f "$redefine_xml" # parent checkpoint re-registered; the incremental can proceed against it
else
rm -f "$redefine_xml"
# Parent checkpoint could not be re-registered — fall back to a full backup in place so
# the chain restarts cleanly instead of failing. Emit a stdout marker so the wrapper
# records this backup as a full (incrementalFallback=true).
log -e "incremental: parent checkpoint $BITMAP_PARENT could not be re-registered — falling back to full"
echo "INCREMENTAL_FALLBACK=true"
effective_mode="full"
fi
fi
fi
# Build backup XML (and matching checkpoint XML when applicable).
name="root"
echo "<domainbackup mode='push'>" > $dest/backup.xml
if [[ "$effective_mode" == "incremental" ]]; then
echo "<incremental>$BITMAP_PARENT</incremental>" >> $dest/backup.xml
fi
echo "<disks>" >> $dest/backup.xml
if [[ $make_checkpoint -eq 1 ]]; then
echo "<domaincheckpoint><name>$BITMAP_NEW</name><disks>" > $dest/checkpoint.xml
fi
while read -r disk fullpath; do
if [[ "$fullpath" == /dev/drbd/by-res/* ]]; then
volUuid=$(get_linstor_uuid_from_path "$fullpath")
elif [[ "$fullpath" == /dev/drbd[0-9]* ]]; then
if ! volUuid=$(get_linstor_uuid_from_device "$fullpath"); then
echo "Failed to resolve LINSTOR volume UUID for $fullpath"
cleanup
exit 1
fi
else
volUuid="${fullpath##*/}"
fi
if [[ "$effective_mode" == "incremental" ]]; then
# Incremental disk entry — no backupmode attr, libvirt picks it up from <incremental>.
echo "<disk name='$disk' backup='yes' type='file'><driver type='qcow2'/><target file='$dest/$name.$volUuid.qcow2' /></disk>" >> $dest/backup.xml
else
echo "<disk name='$disk' backup='yes' type='file' backupmode='full'><driver type='qcow2'/><target file='$dest/$name.$volUuid.qcow2' /></disk>" >> $dest/backup.xml
fi
if [[ $make_checkpoint -eq 1 ]]; then
echo "<disk name='$disk'/>" >> $dest/checkpoint.xml
fi
name="datadisk"
done < <(
virsh -c qemu:///system domblklist "$VM" --details 2>/dev/null | awk '$2=="disk"{print $3, $4}'
)
echo "</disks></domainbackup>" >> $dest/backup.xml
if [[ $make_checkpoint -eq 1 ]]; then
echo "</disks></domaincheckpoint>" >> $dest/checkpoint.xml
fi
local thaw=0
local freeze_ok=0
if [[ ${QUIESCE} == "true" ]]; then
# Always thaw after a freeze attempt: a freeze that times out on the host can still
# complete inside the guest, and nothing else would unfreeze it.
thaw=1
if freeze_err=$(qemu_agent_command '{"execute":"guest-fsfreeze-freeze"}' 2>&1 > /dev/null); then
freeze_ok=1
else
log -e "Failed to freeze the filesystem for vm $VM, continuing without quiescing: $freeze_err"
fi
fi
# Start push backup, atomically registering the new checkpoint when applicable.
local backup_begin=0
if [[ $make_checkpoint -eq 1 ]]; then
# Order matters: redirect stdout to /dev/null first, then merge stderr into stdout.
# The reversed `2>&1 > /dev/null` form leaves stderr pointing at the original tty.
if virsh -c qemu:///system backup-begin --domain $VM --backupxml $dest/backup.xml --checkpointxml $dest/checkpoint.xml > /dev/null 2>&1; then
backup_begin=1;
fi
else
if virsh -c qemu:///system backup-begin --domain $VM --backupxml $dest/backup.xml > /dev/null 2>&1; then
backup_begin=1;
fi
fi
if [[ $backup_begin -eq 1 ]]; then
BACKUP_JOB_ACTIVE=1
fi
if [[ $thaw -eq 1 ]] && ! thaw_guest "$freeze_ok"; then
cleanup
exit 1
fi
if [[ $backup_begin -ne 1 ]]; then
cleanup
exit 1
fi
# Backup domain information
virsh -c qemu:///system dumpxml $VM > $dest/domain-config.xml 2>/dev/null
virsh -c qemu:///system dominfo $VM > $dest/dominfo.xml 2>/dev/null
virsh -c qemu:///system domiflist $VM > $dest/domiflist.xml 2>/dev/null
virsh -c qemu:///system domblklist $VM > $dest/domblklist.xml 2>/dev/null
local info query_failures=0
while true; do
# A failed query says nothing about the job, so retry before giving up on it.
if ! info=$(virsh -c qemu:///system domjobinfo $VM --completed --keep-completed 2>&1); then
log -e "Backup job query $((query_failures + 1)) for vm $VM failed: $info"
if (( ++query_failures >= 12 )); then
echo "Unable to query the backup job for vm $VM"
cleanup
exit 1
fi
sleep 5
continue
fi
query_failures=0
status=$(awk '/Job type:/ {print $3}' <<< "$info")
case "$status" in
Completed)
BACKUP_JOB_ACTIVE=0
break ;;
Failed)
BACKUP_JOB_ACTIVE=0
echo "Virsh backup job failed"
cleanup
exit 1 ;;
esac
sleep 5
done
# Sparsify behavior:
# - For LINSTOR backups (existing): qemu-img convert sparsifies the bloated output.
# - For INCREMENTAL: rebase the resulting thin qcow2 onto its parent so the chain is self-describing
# (so a future restore can flatten without external chain metadata).
name="root"
# PARENT_PATHS arrives as a comma-separated list, one entry per VM volume in the same
# order as DISK_PATHS. Split into a bash array so we can index by disk position.
local -a parent_paths_arr=()
if [[ "$effective_mode" == "incremental" && -n "$PARENT_PATHS" ]]; then
IFS=',' read -ra parent_paths_arr <<< "$PARENT_PATHS"
fi
local disk_idx=0
while read -r disk fullpath; do
if [[ "$effective_mode" == "incremental" ]]; then
volUuid="${fullpath##*/}"
# Pick this disk's specific parent file. Each volume's backup is named after its
# own UUID, so a single PARENT_PATH would wrongly rebase data disks onto the root
# parent.
if [[ $disk_idx -ge ${#parent_paths_arr[@]} ]]; then
echo "PARENT_PATHS list shorter than DISK_PATHS — missing parent for disk index $disk_idx"
cleanup
exit 1
fi
local this_parent_rel="${parent_paths_arr[$disk_idx]}"
local parent_abs="$mount_point/$this_parent_rel"
if [[ ! -f "$parent_abs" ]]; then
echo "Parent backup file does not exist on NAS: $parent_abs"
cleanup
exit 1
fi
local parent_rel
parent_rel=$(realpath --relative-to="$dest" "$parent_abs")
if ! qemu-img rebase -u -b "$parent_rel" -F qcow2 "$dest/$name.$volUuid.qcow2" >> "$logFile" 2> >(cat >&2); then
echo "qemu-img rebase failed for $dest/$name.$volUuid.qcow2 onto $parent_rel"
cleanup
exit 1
fi
name="datadisk"
disk_idx=$((disk_idx + 1))
continue
fi
if [[ "$fullpath" == /dev/drbd/by-res/* ]]; then
volUuid=$(get_linstor_uuid_from_path "$fullpath")
elif [[ "$fullpath" == /dev/drbd[0-9]* ]]; then
if ! volUuid=$(get_linstor_uuid_from_device "$fullpath"); then
echo "Failed to resolve LINSTOR volume UUID for $fullpath"
cleanup
exit 1
fi
else
name="datadisk"
continue
fi
if ! qemu-img convert -O qcow2 "$dest/$name.$volUuid.qcow2" "$dest/$name.$volUuid.qcow2.tmp" >> "$logFile" 2> >(cat >&2); then
echo "qemu-img convert failed for $dest/$name.$volUuid.qcow2"
cleanup
exit 1
fi
mv "$dest/$name.$volUuid.qcow2.tmp" "$dest/$name.$volUuid.qcow2"
name="datadisk"
done < <(
virsh -c qemu:///system domblklist "$VM" --details 2>/dev/null | awk '$2=="disk"{print $3, $4}'
)
rm -f $dest/backup.xml $dest/checkpoint.xml
sync
# Free the parent bitmap now that the incremental is written and rebased: its delta is captured
# here and BITMAP_NEW tracks changes going forward, so it only accrues metadata/IO cost over a
# long chain. Remove it per-disk with block-dirty-bitmap-remove (a clean free) rather than
# checkpoint-delete, which would merge its bits into BITMAP_NEW and re-copy backed-up regions.
# Best-effort: a failure here does not fail the backup, the bitmap is reclaimed on a later run.
if [[ "$effective_mode" == "incremental" && -n "$BITMAP_PARENT" ]]; then
while read -r node; do
[[ -z "$node" ]] && continue
if ! virsh -c qemu:///system qemu-monitor-command "$VM" \
"{\"execute\":\"block-dirty-bitmap-remove\",\"arguments\":{\"node\":\"$node\",\"name\":\"$BITMAP_PARENT\"}}" \
> /dev/null 2>>"$logFile"; then
log -e "cleanup: failed to remove parent bitmap $BITMAP_PARENT on node $node (non-fatal)"
fi
done < <(
virsh -c qemu:///system qemu-monitor-command "$VM" '{"execute":"query-block"}' 2>/dev/null | python3 -c '
import sys, json
target = sys.argv[1]
try:
data = json.load(sys.stdin)
except Exception:
sys.exit(0)
seen = set()
for dev in data.get("return", []) or []:
inserted = dev.get("inserted") or {}
node = inserted.get("node-name")
if not node or node in seen:
continue
if any((b or {}).get("name") == target for b in (inserted.get("dirty-bitmaps") or [])):
seen.add(node)
print(node)
' "$BITMAP_PARENT" 2>/dev/null || true
)
fi
# Print statistics
virsh -c qemu:///system domjobinfo $VM --completed
backup_size=$(du -sb "$dest" 2>>"$logFile" | cut -f1) || { log -ne "WARNING: du failed for $dest, reporting size as 0"; backup_size=0; }
timeout "$UNMOUNT_TIMEOUT" umount "$mount_point" 2>>"$logFile" || { log "WARNING: umount of $mount_point failed or timed out"; true; }
rmdir "$mount_point" 2>>"$logFile" || { log "WARNING: rmdir of $mount_point failed"; true; }
echo "$backup_size"
}
backup_stopped_vm() {
# Stopped VMs cannot use libvirt's backup-begin (no QEMU process); take a full backup via
# qemu-img convert. The orchestrator never sends incremental mode for a stopped VM.
mount_operation
mkdir -p "$dest" || { echo "Failed to create backup directory $dest"; exit 1; }
IFS=","
name="root"
for disk in $DISK_PATHS; do
if [[ "$disk" == rbd:* ]]; then
volUuid=$(get_ceph_uuid_from_path "$disk")
elif [[ "$disk" == /dev/drbd/by-res/* ]]; then
volUuid=$(get_linstor_uuid_from_path "$disk")
elif [[ "$disk" == /dev/drbd[0-9]* ]]; then
if ! volUuid=$(get_linstor_uuid_from_device "$disk"); then
echo "Failed to resolve LINSTOR volume UUID for $disk"
cleanup
exit 1
fi
else
volUuid="${disk##*/}"
fi
output="$dest/$name.$volUuid.qcow2"
if ! qemu-img convert -O qcow2 "$disk" "$output" >> "$logFile" 2> >(cat >&2); then
echo "qemu-img convert failed for $disk $output"
cleanup
exit 1
fi
# Pre-seed a persistent bitmap on the source disk so the NEXT backup (taken
# after this VM is started again) can be incremental against the qcow2 we
# just wrote. Without this, every backup after a stopped-VM backup would
# fall back to full because no parent bitmap exists on the host yet.
# Only applies to file-backed qcow2 sources — RBD/LINSTOR have their own
# snapshot mechanisms and qemu-img bitmap is not the right primitive there.
# bitmap --add should not fail on a file-backed qcow2; if it does, fail the backup so the
# underlying problem is surfaced rather than silently degrading future backups to full.
if [[ -n "$BITMAP_NEW" && "$disk" != rbd:* && "$disk" != /dev/drbd/by-res/* ]]; then
if ! qemu-img bitmap --add "$disk" "$BITMAP_NEW" 2>>"$logFile"; then
echo "Failed to pre-seed bitmap $BITMAP_NEW on $disk"
cleanup
exit 1
fi
fi
name="datadisk"
done
sync
find "$dest" -type f -exec stat -c '%s' {} +
}
delete_backup() {
mount_operation
rm -frv $dest
sync
umount $mount_point
rmdir $mount_point
}
get_backup_stats() {
mount_operation
echo $mount_point
df -P $mount_point 2>/dev/null | awk 'NR==2 {print $2, $3}'
umount $mount_point
rmdir $mount_point
}
mount_operation() {
mount_point=$(mktemp -d -t csbackup.XXXXX)
dest="$mount_point/${BACKUP_DIR}"
if [ ${NAS_TYPE} == "cifs" ]; then
MOUNT_OPTS="${MOUNT_OPTS},nobrl"
fi
mount -t ${NAS_TYPE} ${NAS_ADDRESS} ${mount_point} $([[ ! -z "${MOUNT_OPTS}" ]] && echo -o ${MOUNT_OPTS}) 2>&1 | tee -a "$logFile"
if [ $? -eq 0 ]; then
log -ne "Successfully mounted ${NAS_TYPE} store"
else
echo "Failed to mount ${NAS_TYPE} store"
exit 1
fi
}
abort_backup_job() {
[[ $BACKUP_JOB_ACTIVE -eq 1 ]] || return 0
# The push backup job writes into $dest until it ends, so it must be gone before the
# destination is removed or unmounted.
virsh -c qemu:///system domjobabort "$VM" > /dev/null 2>>"$logFile" || true
local i info
for ((i = 0; i < 60; i++)); do
# Only a successful query reporting no job proves the job has ended.
if info=$(virsh -c qemu:///system domjobinfo "$VM" 2>>"$logFile") \
&& [[ "$(awk '/Job type:/ {print $3}' <<< "$info")" == "None" ]]; then
BACKUP_JOB_ACTIVE=0
return 0
fi
sleep 1
done
return 1
}
cleanup() {
local status=0
# Resume the VM if it was paused during backup to prevent it from
# remaining indefinitely paused when the backup job fails (e.g. due
# to storage full or I/O errors on the backup target)
local vm_state
vm_state=$(virsh -c qemu:///system domstate "$VM" 2>/dev/null || true)
if [[ "$vm_state" == "paused" ]]; then
log -ne "Resuming paused VM $VM during backup cleanup"
if ! virsh -c qemu:///system resume "$VM" > /dev/null 2>&1; then
echo "Failed to resume VM $VM"
status=1
fi
fi
if ! abort_backup_job; then
echo "Backup job for vm $VM is still running after abort, leaving $dest mounted at $mount_point"
echo "Backup cleanup failed"
exit $EXIT_CLEANUP_FAILED
fi
rm -rf "$dest" || { echo "Failed to delete $dest"; status=1; }
umount "$mount_point" || { echo "Failed to unmount $mount_point"; status=1; }
rmdir "$mount_point" || { echo "Failed to remove mount point $mount_point"; status=1; }
if [[ $status -ne 0 ]]; then
echo "Backup cleanup failed"
exit $EXIT_CLEANUP_FAILED
fi
}
function usage {
echo ""
echo "Usage: $0 -o <operation> -v|--vm <domain name> -t <storage type> -s <storage address> -m <mount options> -p <backup path> -d <disks path> -q|--quiesce <true|false>"
echo " [--quiesce-timeout <seconds>]"
echo " [-M|--mode <full|incremental>] [--method <qcow2|content>] [--bitmap-new <name>] [--bitmap-parent <name>] [--parent-paths <p1,p2,...>]"
echo ""
echo "Incremental backup options (running VMs only; requires QEMU >= 4.2 and libvirt >= 7.2):"
echo " -M|--mode full Take a full backup AND create a checkpoint (--bitmap-new required) for future incrementals."
echo " -M|--mode incremental Take an incremental backup since --bitmap-parent and create new checkpoint --bitmap-new."
echo " Requires --bitmap-parent, --bitmap-new, and --parent-paths (comma-separated list, one"
echo " parent qcow2 path per disk: root.<uuid>.qcow2, datadisk.<uuid>.qcow2, … same order"
echo " as -d|--disks)."
echo " --method qcow2 Default. Push-mode backup; incrementals come from QEMU dirty bitmaps."
echo " --method content Libvirt pull-mode backup, for raw block-device storage (LINSTOR/DRBD) which"
echo " cannot carry the persistent dirty bitmaps checkpoints need. A full is written"
echo " as a sparse qcow2; an incremental is a qcow2 overlay on the NBD export,"
echo " safe-rebased onto the parent, keeping only the clusters that differ."
echo " Takes no bitmap names; an incremental requires --parent-paths."
echo " --scratch-devs <d1,d2,...> Copy-before-write scratch block devices for --method content on a running"
echo " VM, one per disk in the same order as -d|--disks, each large enough for"
echo " the whole disk plus qcow2 metadata. libvirt formats them."
echo " Without -M, behaves as legacy full-only backup with no checkpoint creation."
echo ""
exit 1
}
while [[ $# -gt 0 ]]; do
case $1 in
-o|--operation)
OP="$2"
shift
shift
;;
-v|--vm)
VM="$2"
shift
shift
;;
-t|--type)
NAS_TYPE="$2"
shift
shift
;;
-s|--storage)
NAS_ADDRESS="$2"
shift
shift
;;
-m|--mount)
MOUNT_OPTS="$2"
shift
shift
;;
-p|--path)
BACKUP_DIR="$2"
shift
shift
;;
-q|--quiesce)
QUIESCE="$2"
shift
shift
;;
--quiesce-timeout)
QUIESCE_TIMEOUT="$2"
shift
shift
;;
-d|--diskpaths)
DISK_PATHS="$2"
shift
shift
;;
-M|--mode)
MODE="$2"
shift
shift
;;
--method)
METHOD="$2"
shift
shift
;;
--bitmap-new)
BITMAP_NEW="$2"
shift
shift
;;
--bitmap-parent)
BITMAP_PARENT="$2"
shift
shift
;;
--parent-paths)
PARENT_PATHS="$2"
shift
shift
;;
--scratch-devs)
SCRATCH_DEVS="$2"
shift
shift
;;
-h|--help)
usage
shift
;;
*)
echo "Invalid option: $1"
usage
;;
esac
done
# Perform initial environment sanity checks (QEMU/libvirt version).
sanity_checks
if [ "$OP" = "backup" ]; then
STATE=$(virsh -c qemu:///system list | awk -v vm="$VM" '$2 == vm {print $3}')
if [ -n "$STATE" ] && [ "$STATE" = "running" ]; then
backup_running_vm
else
backup_stopped_vm
fi
elif [ "$OP" = "delete" ]; then
delete_backup
elif [ "$OP" = "stats" ]; then
get_backup_stats
fi