411 lines
13 KiB
Bash
Executable File
411 lines
13 KiB
Bash
Executable File
#!/bin/bash
|
|
set -uo pipefail
|
|
#==============================================================#
|
|
# File : pg-basebackup
|
|
# Desc : PostgreSQL base backup script using pg_basebackup
|
|
# Ctime : 2018-12-06
|
|
# Mtime : 2026-01-18
|
|
# Path : /pg/bin/pg-basebackup
|
|
# Deps : pg_basebackup, psql, lz4, openssl, .pgpass
|
|
# License : Apache-2.0 @ https://pigsty.io/docs/about/license/
|
|
# Copyright : 2018-2026 Ruohang Feng / Vonng (rh@vonng.com)
|
|
#==============================================================#
|
|
# DEPRECATED: This script is a legacy backup tool using pg_basebackup.
|
|
# For production use, prefer `pg-backup` which uses pgbackrest with:
|
|
# - Incremental/differential backups (saves storage & time)
|
|
# - Built-in compression, encryption, and parallelism
|
|
# - Point-in-time recovery (PITR) support
|
|
# - Remote backup repository support (S3, Azure, GCS, etc.)
|
|
#
|
|
# Use this script only when:
|
|
# - pgbackrest is not available or configured
|
|
# - You need a simple one-time backup to local disk
|
|
# - You're migrating data between clusters manually
|
|
#
|
|
# Also consider pg-fork to create instant copy efficiently.
|
|
# Note: The RC4 encryption option (-e) uses a legacy cipher for
|
|
# performance. For sensitive data, consider pgbackrest's built-in
|
|
# encryption or use a stronger cipher externally.
|
|
#==============================================================#
|
|
PROG_NAME="$(basename $0)"
|
|
PROG_DIR="$(cd $(dirname $0) && pwd)"
|
|
|
|
|
|
#--------------------------------------------------------------#
|
|
# Usage #
|
|
#--------------------------------------------------------------#
|
|
function usage() {
|
|
cat <<-'EOF'
|
|
NAME
|
|
pg-basebackup -- make base backup from PostgreSQL instance
|
|
|
|
SYNOPSIS
|
|
pg-basebackup -sdfeukr
|
|
pg-basebackup --src postgres:/// --dst . --file backup.tar.lz4
|
|
|
|
DESCRIPTION
|
|
-s, --src, --url
|
|
Backup source URL, optional, "postgres:///" by default
|
|
Note: if password is required, it should be given in url, ENV or .pgpass
|
|
|
|
-d, --dst, --dir
|
|
Where to put backup files, "/pg/backup" by default
|
|
|
|
-f, --file
|
|
Overwrite default backup filename, "backup_${tag}_${date}.tar.lz4"
|
|
|
|
-r, --remove
|
|
.lz4 Files mtime before n minutes ago will be removed, default is 1200 (20hour)
|
|
|
|
-t, --tag
|
|
Backup file tag, if not set, target cluster_name or local ip address will be used.
|
|
Also used as part of DEFAULT filename
|
|
|
|
-k, --key
|
|
Encryption key when --encrypt is specified, default key is ${tag}
|
|
|
|
-u, --upload
|
|
Upload backup files to cloud storage, (need your own implementation)
|
|
|
|
-e, --encryption
|
|
Encrypt with RC4 using OpenSSL, if not key is specified, tag is used as key
|
|
|
|
-h, --help
|
|
Print this message
|
|
|
|
EXAMPLES
|
|
routine backup crontab:
|
|
00 01 * * * /pg/bin/pg-basebackup --encrypt --upload --tag=test --key=<secret> 2>> /pg/log/backup.log
|
|
|
|
one-time manual backup:
|
|
pg-basebackup -s postgres:/// -d . -f one_time_backup.tar.lz4 -e
|
|
|
|
extract backup files:
|
|
backup_dir="/pg/backup"
|
|
backup_latest=$(ls -t ${backup_dir} | head -n1)
|
|
unlz4 -d -c ${backup_latest} | tar -xC ${DATA_DIR}
|
|
openssl enc -rc4 -d -k ${PASSWORD} -in ${backup_latest} | unlz4 -d -c | tar -xC ${DATA_DIR}
|
|
EOF
|
|
}
|
|
|
|
|
|
#--------------------------------------------------------------#
|
|
# Param #
|
|
#--------------------------------------------------------------#
|
|
export PATH=/usr/pgsql/bin:${PATH}
|
|
|
|
|
|
#--------------------------------------------------------------#
|
|
# Log Util
|
|
#--------------------------------------------------------------#
|
|
if [[ -t 1 ]]; then
|
|
__CN='\033[0m';__CK='\033[0;30m';__CR='\033[0;31m';__CG='\033[0;32m';
|
|
__CY='\033[0;33m';__CB='\033[0;34m';__CM='\033[0;35m';__CC='\033[0;36m';__CW='\033[0;37m';
|
|
else
|
|
__CN='';__CK='';__CR='';__CG='';__CY='';__CB='';__CM='';__CC='';__CW='';
|
|
fi
|
|
function log_info() { printf "[${__CG} OK ${__CN}] ${__CG}$*${__CN}\n"; }
|
|
function log_warn() { printf "[${__CY}WARN${__CN}] ${__CY}$*${__CN}\n"; }
|
|
function log_error() { printf "[${__CR}FAIL${__CN}] ${__CR}$*${__CN}\n"; }
|
|
function log_debug() { printf "[${__CB}HINT${__CN}] ${__CB}$*${__CN}\n"; }
|
|
function log_title() { printf "[${__CG}$1${__CN}] ${__CG}$2${__CN}\n"; }
|
|
function log_hint() { printf "${__CB}$*${__CN}\n"; }
|
|
function log_line() { printf "${__CM}===== $* =====${__CN}\n"; }
|
|
|
|
# returns 't' on replica, psql access required
|
|
function is_in_recovery() {
|
|
local pg_url=$1
|
|
echo $(psql ${pg_url} -AXtwqc "SELECT pg_is_in_recovery();")
|
|
}
|
|
|
|
# get primary IP address (which could be 10.x.x.x or 192.x.x.x)
|
|
function local_ip() {
|
|
# try ip command first (modern Linux), fallback to ifconfig
|
|
if command -v ip &>/dev/null; then
|
|
ip -4 addr show | grep -oP 'inet \K(10|192)\.[0-9.]+' | head -n1
|
|
elif command -v ifconfig &>/dev/null; then
|
|
ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -v 127.0.0.1 | grep -Eo '(10|192)\.([0-9]*\.){2}[0-9]*' | head -n1
|
|
else
|
|
hostname -I 2>/dev/null | awk '{print $1}'
|
|
fi
|
|
}
|
|
|
|
# return cluster_name, psql access required
|
|
function get_cluster_name() {
|
|
local pg_url=$1
|
|
local cluster_name=$(psql ${pg_url} -AXtwqc "SHOW cluster_name")
|
|
if [[ -z "${cluster_name}" ]]; then
|
|
echo $(local_ip)
|
|
else
|
|
echo ${cluster_name}
|
|
fi
|
|
}
|
|
|
|
#==============================================================#
|
|
# Backup #
|
|
#==============================================================#
|
|
|
|
#--------------------------------------------------------------#
|
|
# Name: make_backup
|
|
# Desc: make pg base backup to given path
|
|
# Arg1: Postgres URI
|
|
# Arg2: Backup filepath
|
|
# Arg3: Encrytion key, optional
|
|
# Note: if key is provided, encrypt backup with openssl rc4
|
|
#--------------------------------------------------------------#
|
|
function make_backup() {
|
|
local pg_url=$1
|
|
local backup_path=$2
|
|
local key=${3-''}
|
|
|
|
if [[ ! -z "${key}" ]]; then
|
|
# if key is provided, encrypt with rc4 using openssl
|
|
pg_basebackup -w -d ${pg_url} -Xf -Ft -c fast -v -D - | lz4 -q -z | openssl enc -rc4 -k ${key} >"${backup_path}"
|
|
# extract: openssl enc -rc4 -d -k ${KEY} -in ${BKUP_FILE} | unlz4 -c | tar -xC ${DATA_DIR}
|
|
return $?
|
|
else
|
|
pg_basebackup -w -d ${pg_url} -Xf -Ft -c fast -v -D - | lz4 -q -z >"${backup_path}"
|
|
# extract: unlz4 ${BKUP_FILE} -c | tar -xC ${DATA_DIR}
|
|
return $?
|
|
fi
|
|
}
|
|
|
|
#--------------------------------------------------------------#
|
|
# Name: kill_base_backup
|
|
# Desc: kill existing running backup process
|
|
#--------------------------------------------------------------#
|
|
function kill_base_backup() {
|
|
local pids
|
|
pids=$(pgrep -f "pg_basebackup.*-Xf" 2>/dev/null || true)
|
|
if [[ -z "${pids}" ]]; then
|
|
log_info "no pg_basebackup processes found"
|
|
return 0
|
|
fi
|
|
log_warn "killing basebackup processes: ${pids}"
|
|
for pid in ${pids}; do
|
|
if kill -0 "${pid}" 2>/dev/null; then
|
|
log_warn "kill basebackup process: ${pid}"
|
|
kill "${pid}" 2>/dev/null || true
|
|
fi
|
|
done
|
|
log_warn "basebackup processes killed"
|
|
}
|
|
|
|
#--------------------------------------------------------------#
|
|
# Name: remove_backup
|
|
# Desc: remove old backup files (*.lz4) in given backup dir
|
|
# Arg1: backup directory
|
|
# Arg2: remove threshhold (minutes, default 1200, i.e 20hour)
|
|
#--------------------------------------------------------------#
|
|
function remove_backup() {
|
|
# delete *.lz4 file mtime before 20h ago by default
|
|
local backup_dir=$1
|
|
local remove_condition=${2-'1200'}
|
|
remove_condition="-mmin +${remove_condition}"
|
|
|
|
log_info "[BKUP] find obsolete backups: find "${backup_dir}/" -maxdepth 1 -type f ${remove_condition} -name 'backup*.lz4'"
|
|
local obsolete_backups="$(find "${backup_dir}/" -maxdepth 1 -type f ${remove_condition} -name 'backup*.lz4')"
|
|
log_warn "[BKUP] remove obsolete backups: ${obsolete_backups}"
|
|
find "${backup_dir}/" -maxdepth 1 -type f -name 'backup_*.lz4' ${remove_condition} -delete
|
|
return $?
|
|
}
|
|
|
|
#--------------------------------------------------------------#
|
|
# Name: upload_backup
|
|
# Desc: upload backup files to ufile
|
|
# Arg1: backup_filepath
|
|
# Arg2: tag , backup taged with it will be removed
|
|
#--------------------------------------------------------------#
|
|
function upload_backup() {
|
|
local backup_filepath=$1
|
|
local tag=$2
|
|
local filename=$(basename ${backup_filepath})
|
|
|
|
# HINT: customize upload logic here
|
|
log_info "[UPLOAD] upload ${backup_filepath}"
|
|
log_info "[UPLOAD] upload to ${filename}"
|
|
log_warn "[UPLOAD] obsolete backups: None"
|
|
log_warn "[UPLOAD] remove obsolete backups due to retention policy"
|
|
|
|
return 0
|
|
}
|
|
|
|
|
|
#--------------------------------------------------------------#
|
|
# Main #
|
|
#--------------------------------------------------------------#
|
|
function main() {
|
|
# default settings
|
|
local lock_path="/tmp/backup.lock"
|
|
local src="postgres:///"
|
|
local dst="/pg/backup"
|
|
local tag=$(get_cluster_name ${src})
|
|
local remove="1200"
|
|
local upload="false"
|
|
local encrypt="false"
|
|
|
|
local filename=""
|
|
local key=""
|
|
local provided_filename=""
|
|
local provided_key=""
|
|
|
|
# parse arguments
|
|
while (($# > 0)); do
|
|
case "$1" in
|
|
-s | --src=* | --url=*)
|
|
[ "$1" == "-s" ] && shift
|
|
src=${1##*=}
|
|
shift
|
|
;;
|
|
-d | --dst=* | --dir=*)
|
|
[ "$1" == "-d" ] && shift
|
|
dst=${1##*=}
|
|
shift
|
|
;;
|
|
-f | --file=*)
|
|
[ "$1" == "-f" ] && shift
|
|
provided_filename=${1##*=}
|
|
shift
|
|
;;
|
|
-r | --remove=*)
|
|
[ "$1" == "-r" ] && shift
|
|
remove=${1##*=}
|
|
shift
|
|
;;
|
|
-k | --key=*)
|
|
[ "$1" == "-k" ] && shift
|
|
provided_key=${1##*=}
|
|
shift
|
|
;;
|
|
-t | --tag=*)
|
|
[ "$1" == "-t" ] && shift
|
|
tag=${1##*=}
|
|
shift
|
|
;;
|
|
-u | --upload)
|
|
upload="true"
|
|
shift
|
|
;;
|
|
-e | --encrypt)
|
|
encrypt="true"
|
|
shift
|
|
;;
|
|
-h)
|
|
usage
|
|
exit
|
|
;;
|
|
*)
|
|
usage
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# overwrite filename & key with tag
|
|
if [[ -z "${provided_filename}" ]]; then
|
|
# if filename is not specified, use "backup_${tag}_${date}.tar.lz4" as filename
|
|
filename="backup_${tag}_$(date +%Y%m%d).tar.lz4"
|
|
else
|
|
filename=${provided_filename}
|
|
fi
|
|
local backup_filepath="${dst}/${filename}"
|
|
|
|
if [[ -z "${provided_key}" ]]; then
|
|
# if key is not specified, use ${tag} as key
|
|
key=${tag}
|
|
else
|
|
key=${provided_key}
|
|
fi
|
|
|
|
log_info "================================================================"
|
|
# check parameters
|
|
log_info "[INIT] pg-basebackup begin, checking parameters"
|
|
|
|
if [[ ! -d ${dst} ]]; then
|
|
log_error "[INIT] destination directory ${dst} not exist"
|
|
exit 2
|
|
fi
|
|
|
|
if [[ ${remove} != [0-9]* ]]; then
|
|
log_error "[INIT] -r,--remove should be an integer represent minutes of retention"
|
|
exit 3
|
|
fi
|
|
|
|
if [[ -z $(command -v pg_basebackup) ]]; then
|
|
log_error "[INIT] pg_basebackup binary not found in PATH"
|
|
exit 4
|
|
fi
|
|
|
|
# if [[ ${upload} == "true" ]]; then
|
|
# # HINT: YOUR IMPLEMENTATION HERE
|
|
# fi
|
|
|
|
if [[ ${encrypt} == "true" ]]; then
|
|
# if encrypt is specified, openssl sould exist
|
|
if [[ -z $(command -v openssl) ]]; then
|
|
log_error "[INIT] openssl binary not found in PATH when encrypt is specified"
|
|
exit 7
|
|
fi
|
|
fi
|
|
|
|
log_debug "[INIT] #====== BINARY"
|
|
log_debug "[INIT] pg_basebackup : $(command -v pg_basebackup)"
|
|
log_debug "[INIT] openssl : $(command -v openssl)"
|
|
|
|
log_debug "[INIT] #====== PARAMETER"
|
|
log_debug "[INIT] filename (-f) : ${filename}"
|
|
log_debug "[INIT] src (-s) : ${src}"
|
|
log_debug "[INIT] dst (-d) : ${dst}"
|
|
log_debug "[INIT] tag (-t) : ${tag}"
|
|
log_debug "[INIT] key (-k) : ${key}"
|
|
log_debug "[INIT] encrypt (-e) : ${encrypt}"
|
|
log_debug "[INIT] upload (-u) : ${upload}"
|
|
log_debug "[INIT] remove (-r) : -mmin +${remove}"
|
|
|
|
# Lock (Avoid multiple instance)
|
|
if [ -e ${lock_path} ] && kill -0 $(cat ${lock_path}); then
|
|
log_error "[LOCK] acquire lock @ ${lock_path} failed, other_pid=$(cat ${lock_path})"
|
|
exit 8
|
|
fi
|
|
log_info "[LOCK] acquire lock @ ${lock_path}"
|
|
trap "rm -f ${lock_path}; exit" INT TERM EXIT
|
|
echo $$ >${lock_path}
|
|
log_info "[LOCK] lock acquired success on ${lock_path}, pid=$$"
|
|
|
|
# Start Backup
|
|
log_info "[BKUP] backup begin, from ${src} to ${backup_filepath}"
|
|
if [[ ${encrypt} != "true" ]]; then
|
|
log_info "[BKUP] backup in normal mode"
|
|
key=""
|
|
else
|
|
log_info "[BKUP] backup in encryption mode"
|
|
fi
|
|
|
|
make_backup ${src} ${backup_filepath} ${key}
|
|
|
|
if [[ $? != 0 ]]; then
|
|
log_error "[BKUP] backup failed!"
|
|
exit 9
|
|
fi
|
|
log_info "[BKUP] backup complete!"
|
|
|
|
# remove old local backup
|
|
log_info "[RMBK] remove local obsolete backup: ${remove}"
|
|
remove_backup ${dst} ${remove}
|
|
if [[ $? != 0 ]]; then
|
|
log_error "[RMBK] remove local obsolete backup failed!"
|
|
exit 10
|
|
fi
|
|
log_info "[RMBK] remove old backup complete"
|
|
|
|
# unlock
|
|
rm -f ${lock_path}
|
|
log_info "[LOCK] release lock @ ${lock_path}"
|
|
|
|
# done
|
|
log_info "[DONE] backup procedure complete!"
|
|
log_info "================================================================"
|
|
}
|
|
|
|
main "$@"
|