| #!/usr/bin/bash | |
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| set -eo pipefail | |
| # CloudStack B&R NAS Backup and Recovery Tool for KVM | |
| # TODO: do libvirt/logging etc checks | |
| ### Declare variables ### | |
| OP="" | |
| VM="" | |
| NAS_TYPE="" | |
| NAS_ADDRESS="" | |
| MOUNT_OPTS="" | |
| BACKUP_DIR="" | |
| DISK_PATHS="" | |
| QUIESCE="" | |
| logFile="/var/log/cloudstack/agent/agent.log" | |
| EXIT_CLEANUP_FAILED=20 | |
| log() { | |
| [[ "$verb" -eq 1 ]] && builtin echo "$@" | |
| if [[ "$1" == "-ne" || "$1" == "-e" || "$1" == "-n" ]]; then | |
| builtin echo -e "$(date '+%Y-%m-%d %H-%M-%S>')" "${@: 2}" >> "$logFile" | |
| else | |
| builtin echo "$(date '+%Y-%m-%d %H-%M-%S>')" "$@" >> "$logFile" | |
| fi | |
| } | |
| vercomp() { | |
| local IFS=. | |
| local i ver1=($1) ver2=($3) | |
| # Compare each segment of the version numbers | |
| for ((i=0; i<${#ver1[@]}; i++)); do | |
| if [[ -z ${ver2[i]} ]]; then | |
| ver2[i]=0 | |
| fi | |
| if ((10#${ver1[i]} > 10#${ver2[i]})); then | |
| return 0 # Version 1 is greater | |
| elif ((10#${ver1[i]} < 10#${ver2[i]})); then | |
| return 2 # Version 2 is greater | |
| fi | |
| done | |
| return 0 # Versions are equal | |
| } | |
| sanity_checks() { | |
| hvVersion=$(virsh version | grep hypervisor | awk '{print $(NF)}') | |
| libvVersion=$(virsh version | grep libvirt | awk '{print $(NF)}' | tail -n 1) | |
| apiVersion=$(virsh version | grep API | awk '{print $(NF)}') | |
| # Compare qemu version (hvVersion >= 4.2.0) | |
| vercomp "$hvVersion" ">=" "4.2.0" | |
| hvStatus=$? | |
| # Compare libvirt version (libvVersion >= 7.2.0) | |
| vercomp "$libvVersion" ">=" "7.2.0" | |
| libvStatus=$? | |
| if [[ $hvStatus -eq 0 && $libvStatus -eq 0 ]]; then | |
| log -ne "Success... [ QEMU: $hvVersion Libvirt: $libvVersion apiVersion: $apiVersion ]" | |
| else | |
| echo "Failure... Your QEMU version $hvVersion or libvirt version $libvVersion is unsupported. Consider upgrading to the required minimum version of QEMU: 4.2.0 and Libvirt: 7.2.0" | |
| exit 1 | |
| fi | |
| log -ne "Environment Sanity Checks successfully passed" | |
| } | |
| ### Operation methods ### | |
| get_ceph_uuid_from_path() { | |
| local fullpath="$1" | |
| # disk for rbd => rbd:<pool>/<uuid>:mon_host=<monitor_host>... | |
| # sample: rbd:cloudstack/53d5c355-d726-4d3e-9422-046a503a0b12:mon_host=10.0.1.2... | |
| local beforeUuid="${fullpath#*/}" # Remove up to first slash after rbd: | |
| local volUuid="${beforeUuid%%:*}" # Remove everything after colon to get the uuid | |
| echo ""$volUuid"" | |
| } | |
| get_linstor_uuid_from_path() { | |
| local fullpath="$1" | |
| # disk for linstor => /dev/drbd/by-res/cs-<uuid>/0 | |
| # sample: /dev/drbd/by-res/cs-53d5c355-d726-4d3e-9422-046a503a0b12/0 | |
| local beforeUuid="${fullpath#/dev/drbd/by-res/}" | |
| local volUuid="${beforeUuid%%/*}" | |
| volUuid="${volUuid#cs-}" | |
| echo "$volUuid" | |
| } | |
| backup_running_vm() { | |
| mount_operation | |
| mkdir -p "$dest" || { echo "Failed to create backup directory $dest"; exit 1; } | |
| name="root" | |
| echo "<domainbackup mode='push'><disks>" > $dest/backup.xml | |
| while read -r disk fullpath; do | |
| if [[ "$fullpath" == /dev/drbd/by-res/* ]]; then | |
| volUuid=$(get_linstor_uuid_from_path "$fullpath") | |
| else | |
| volUuid="${fullpath##*/}" | |
| fi | |
| echo "<disk name='$disk' backup='yes' type='file' backupmode='full'><driver type='qcow2'/><target file='$dest/$name.$volUuid.qcow2' /></disk>" >> $dest/backup.xml | |
| name="datadisk" | |
| done < <( | |
| virsh -c qemu:///system domblklist "$VM" --details 2>/dev/null | awk '$2=="disk"{print $3, $4}' | |
| ) | |
| echo "</disks></domainbackup>" >> $dest/backup.xml | |
| local thaw=0 | |
| if [[ ${QUIESCE} == "true" ]]; then | |
| if virsh -c qemu:///system qemu-agent-command "$VM" '{"execute":"guest-fsfreeze-freeze"}' > /dev/null 2>/dev/null; then | |
| thaw=1 | |
| fi | |
| fi | |
| # Start push backup | |
| local backup_begin=0 | |
| if virsh -c qemu:///system backup-begin --domain $VM --backupxml $dest/backup.xml 2>&1 > /dev/null; then | |
| backup_begin=1; | |
| fi | |
| if [[ $thaw -eq 1 ]]; then | |
| if ! response=$(virsh -c qemu:///system qemu-agent-command "$VM" '{"execute":"guest-fsfreeze-thaw"}' 2>&1 > /dev/null); then | |
| echo "Failed to thaw the filesystem for vm $VM: $response" | |
| cleanup | |
| exit 1 | |
| fi | |
| fi | |
| if [[ $backup_begin -ne 1 ]]; then | |
| cleanup | |
| exit 1 | |
| fi | |
| # Backup domain information | |
| virsh -c qemu:///system dumpxml $VM > $dest/domain-config.xml 2>/dev/null | |
| virsh -c qemu:///system dominfo $VM > $dest/dominfo.xml 2>/dev/null | |
| virsh -c qemu:///system domiflist $VM > $dest/domiflist.xml 2>/dev/null | |
| virsh -c qemu:///system domblklist $VM > $dest/domblklist.xml 2>/dev/null | |
| while true; do | |
| status=$(virsh -c qemu:///system domjobinfo $VM --completed --keep-completed | awk '/Job type:/ {print $3}') | |
| case "$status" in | |
| Completed) | |
| break ;; | |
| Failed) | |
| echo "Virsh backup job failed" | |
| cleanup ;; | |
| esac | |
| sleep 5 | |
| done | |
| # Use qemu-img convert to sparsify linstor backups which get bloated due to virsh backup-begin. | |
| name="root" | |
| while read -r disk fullpath; do | |
| if [[ "$fullpath" != /dev/drbd/by-res/* ]]; then | |
| continue | |
| fi | |
| volUuid=$(get_linstor_uuid_from_path "$fullpath") | |
| if ! qemu-img convert -O qcow2 "$dest/$name.$volUuid.qcow2" "$dest/$name.$volUuid.qcow2.tmp" >> "$logFile" 2> >(cat >&2); then | |
| echo "qemu-img convert failed for $dest/$name.$volUuid.qcow2" | |
| cleanup | |
| exit 1 | |
| fi | |
| mv "$dest/$name.$volUuid.qcow2.tmp" "$dest/$name.$volUuid.qcow2" | |
| name="datadisk" | |
| done < <( | |
| virsh -c qemu:///system domblklist "$VM" --details 2>/dev/null | awk '$2=="disk"{print $3, $4}' | |
| ) | |
| rm -f $dest/backup.xml | |
| sync | |
| # Print statistics | |
| virsh -c qemu:///system domjobinfo $VM --completed | |
| du -sb $dest | cut -f1 | |
| umount $mount_point | |
| rmdir $mount_point | |
| } | |
| backup_stopped_vm() { | |
| mount_operation | |
| mkdir -p "$dest" || { echo "Failed to create backup directory $dest"; exit 1; } | |
| IFS="," | |
| name="root" | |
| for disk in $DISK_PATHS; do | |
| if [[ "$disk" == rbd:* ]]; then | |
| volUuid=$(get_ceph_uuid_from_path "$disk") | |
| elif [[ "$disk" == /dev/drbd/by-res/* ]]; then | |
| volUuid=$(get_linstor_uuid_from_path "$disk") | |
| else | |
| volUuid="${disk##*/}" | |
| fi | |
| output="$dest/$name.$volUuid.qcow2" | |
| if ! qemu-img convert -O qcow2 "$disk" "$output" > "$logFile" 2> >(cat >&2); then | |
| echo "qemu-img convert failed for $disk $output" | |
| cleanup | |
| fi | |
| name="datadisk" | |
| done | |
| sync | |
| ls -l --numeric-uid-gid $dest | awk '{print $5}' | |
| } | |
| delete_backup() { | |
| mount_operation | |
| rm -frv $dest | |
| sync | |
| umount $mount_point | |
| rmdir $mount_point | |
| } | |
| get_backup_stats() { | |
| mount_operation | |
| echo $mount_point | |
| df -P $mount_point 2>/dev/null | awk 'NR==2 {print $2, $3}' | |
| umount $mount_point | |
| rmdir $mount_point | |
| } | |
| mount_operation() { | |
| mount_point=$(mktemp -d -t csbackup.XXXXX) | |
| dest="$mount_point/${BACKUP_DIR}" | |
| if [ ${NAS_TYPE} == "cifs" ]; then | |
| MOUNT_OPTS="${MOUNT_OPTS},nobrl" | |
| fi | |
| mount -t ${NAS_TYPE} ${NAS_ADDRESS} ${mount_point} $([[ ! -z "${MOUNT_OPTS}" ]] && echo -o ${MOUNT_OPTS}) 2>&1 | tee -a "$logFile" | |
| if [ $? -eq 0 ]; then | |
| log -ne "Successfully mounted ${NAS_TYPE} store" | |
| else | |
| echo "Failed to mount ${NAS_TYPE} store" | |
| exit 1 | |
| fi | |
| } | |
| cleanup() { | |
| local status=0 | |
| rm -rf "$dest" || { echo "Failed to delete $dest"; status=1; } | |
| umount "$mount_point" || { echo "Failed to unmount $mount_point"; status=1; } | |
| rmdir "$mount_point" || { echo "Failed to remove mount point $mount_point"; status=1; } | |
| if [[ $status -ne 0 ]]; then | |
| echo "Backup cleanup failed" | |
| exit $EXIT_CLEANUP_FAILED | |
| fi | |
| } | |
| function usage { | |
| echo "" | |
| echo "Usage: $0 -o <operation> -v|--vm <domain name> -t <storage type> -s <storage address> -m <mount options> -p <backup path> -d <disks path> -q|--quiesce <true|false>" | |
| echo "" | |
| exit 1 | |
| } | |
| while [[ $# -gt 0 ]]; do | |
| case $1 in | |
| -o|--operation) | |
| OP="$2" | |
| shift | |
| shift | |
| ;; | |
| -v|--vm) | |
| VM="$2" | |
| shift | |
| shift | |
| ;; | |
| -t|--type) | |
| NAS_TYPE="$2" | |
| shift | |
| shift | |
| ;; | |
| -s|--storage) | |
| NAS_ADDRESS="$2" | |
| shift | |
| shift | |
| ;; | |
| -m|--mount) | |
| MOUNT_OPTS="$2" | |
| shift | |
| shift | |
| ;; | |
| -p|--path) | |
| BACKUP_DIR="$2" | |
| shift | |
| shift | |
| ;; | |
| -q|--quiesce) | |
| QUIESCE="$2" | |
| shift | |
| shift | |
| ;; | |
| -d|--diskpaths) | |
| DISK_PATHS="$2" | |
| shift | |
| shift | |
| ;; | |
| -h|--help) | |
| usage | |
| shift | |
| ;; | |
| *) | |
| echo "Invalid option: $1" | |
| usage | |
| ;; | |
| esac | |
| done | |
| # Perform Initial sanity checks | |
| sanity_checks | |
| if [ "$OP" = "backup" ]; then | |
| STATE=$(virsh -c qemu:///system list | awk -v vm="$VM" '$2 == vm {print $3}') | |
| if [ -n "$STATE" ] && [ "$STATE" = "running" ]; then | |
| backup_running_vm | |
| else | |
| backup_stopped_vm | |
| fi | |
| elif [ "$OP" = "delete" ]; then | |
| delete_backup | |
| elif [ "$OP" = "stats" ]; then | |
| get_backup_stats | |
| fi |