SandW
/
concourse-utilities
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
concourse-utilities/hangar

807 lines
19 KiB
Bash
Executable File
Raw Permalink Blame History

#!/bin/bash
VERSION=1.3.0
USERNAME=$(whoami)
CANON_REPO=https://github.com/starkandwayne/hangar
HANGAR_DIR="${HOME}/.hangar"
FLY_CMD="${HANGAR_DIR}/fly"
# Options
opt_target=${CONCOURSE_TARGET}
opt_pause_after_restore='a'
opt_backup_dir="${HANGAR_DIR}/backups/(target)"
opt_clean=""
target_pipelines=()
####################################################
# pipelines and backups info
declare -a pipelines pipeline_states
get_pipelines() {
local pipeline_info
local name
local state
pipeline_info="$(${FLY_CMD} -t $opt_target pipelines)"
if [[ "$?" -ne 0 ]]
then
fatal \
"Could not retrieve pipelines from $opt_target:" \
$pipeline_info
exit 1
fi
pipelines=()
pipeline_states=()
while read name state
do
pipelines+=("$name")
pipeline_states+=("$state")
done < <(echo "$pipeline_info" | tail -n +1)
}
declare -a backups backup_states
get_backups() {
local backup_files
local name
local state
echo "Using backup directory '$backup_dir'"
echo
shopt -s nullglob
backup_files=(${backup_dir}/*.yml)
shopt -u nullglob
backups=()
backup_states=()
for file in "${backup_files[@]}"
do
name="$( echo "$file" | sed -E 's#.*/([^/]*)\.yml$#\1#' )"
if [[ -f "${backup_dir}/${name}.state" ]]
then
state="$(cat ${backup_dir}/${name}.state)"
else
state="unknown"
fi
backups+=("$name")
backup_states+=("$state")
done
}
pipeline_state() {
name=$1
index="$(echo ${pipelines[@]/${name}//} | cut -d/ -f1 | wc -w | tr -d ' ')"
echo "${pipeline_states[$index]}"
}
pipeline_exists() {
name=$1
index="$(echo ${pipelines[@]/${name}//} | cut -d/ -f1 | wc -w | tr -d ' ')"
[[ $index -lt ${#pipelines[@]} ]]
}
backup_state() {
name=$1
index="$(echo ${backups[@]/${name}//} | cut -d/ -f1 | wc -w | tr -d ' ')"
echo "${backup_states[$index]}"
}
backup_exists() {
name=$1
index="$(echo ${backups[@]/${name}//} | cut -d/ -f1 | wc -w | tr -d ' ')"
[[ $index -lt ${#backups[@]} ]]
}
####################################################
# checks and validating functions
need_command() {
local cmd=${1:?need_command() - no command name given}
[[ -x "$(command -v $cmd)" ]] || fatal "${cmd} is not installed."
}
# First arg is path. Don't put the scheme://hostname:port in
# Second arg is the output path
# Third arg is basic auth params "username:password"
fly_curl() {
local fly_url fly_insecure fly_ca_cert insecure_opt output_opt basic_auth_opt
fly_url="$(flyrc ".targets.${opt_target}.api")${1}"
fly_insecure="$(flyrc ".targets.${opt_target}.insecure")"
fly_token="$(flyrc ".targets.${opt_target}.token.value")"
fly_ca_cert="$(flyrc ".targets.${opt_target}.ca_cert")"
[[ "$fly_insecure" == 'true' ]] && insecure_opt='-k'
[[ -n $2 ]] && output_opt="-o ${2}"
[[ -n $3 ]] && basic_auth_opt="--basic --user \"${3}\""
if [[ -n $fly_ca_cert ]] ; then
curl -sS ${insecure_opt} ${output_opt} ${basic_auth_opt} -H "Authorization: Bearer $fly_token" --cacert <(cat <<<"${fly_ca_cert}") "${fly_url}"
rc=$?
return $rc
else
curl -sS ${insecure_opt} ${output_opt} ${basic_auth_opt} -H "Authorization: Bearer $fly_token" "${fly_url}"
rc=$?
return $rc
fi
}
check_target() {
[[ -n "$opt_target" ]] || fatal "No target specified. Please set in \$CONCOURSE_TARGET or use the -t|--target option"
fly_curl "" >/dev/null || fatal "Cannot reach Concourse target at ${fly_url}"
}
check_backup_directory_empty() {
shopt -s nullglob
files=(${backup_dir}/*.{yml,state})
[[ "${#files[@]}" -eq 0 ]] || fatal "Backup directory (${backup_dir}) cannot contain any .yml or .state files"
}
get_backup_dir() {
if [[ "$opt_backup_dir" == *"(target)"* ]]
then
if [[ -z ${opt_target} ]]
then
fatal "No target specified, but backup directory uses (target) placeholder"
fi
local target=$(echo $opt_target | sed -e "s#/#\\/#g")
echo "${opt_backup_dir}" | sed -e "s/(target)/${target}/"
else
echo ${opt_backup_dir}
fi
}
need_dirs() {
backup_dir="$(get_backup_dir)" || exit 1
[[ -d "$HANGAR_DIR" ]] || mkdir -p "${HANGAR_DIR}"
[[ -n "$backup_dir" ]] || fatal "No backup directory specified"
mkdir -p -- "$backup_dir"
}
need_fly() {
echo "Ensuring ${FLY_CMD} to the matching version for the target"
local insecure_opt=''; [[ $fly_insecure == 'true' ]] && insecure_opt='-k'
local platform=$(uname -s | tr A-Z a-z)
fly_curl "/api/v1/cli?arch=amd64&platform=${platform}" "${FLY_CMD}"
if [[ $? -eq 0 && ! "$(file "${FLY_CMD}")" =~ ASCII\ text ]]; then
echo "Retrieved!"
echo ""
chmod u+x ${FLY_CMD}
else
echo "Could not fetch fly -- need to log in with HTTP Basic Auth"
read -p "username: " username
read -p "password: " -s password
auth="$username:$password"
fly_curl "/api/v1/cli?arch=amd64&platform=${platform}" "${FLY_CMD}" "${auth}"
if [[ "$(file "${FLY_CMD}")" =~ ASCII\ text ]]
then
rm ${FLY_CMD}
fatal "Could not log into Concourse to fetch fly!"
fi
chmod u+x ${FLY_CMD}
echo ""
echo "Successfully fetched"
fi
test="$(${FLY_CMD} -t ${opt_target} workers 2>&1)"
if [[ "$test" =~ not\ authorized ]]
then
echo "Log into Concourse"
${FLY_CMD} -t $opt_target login $insecure_opt
fi
[[ -x ${FLY_CMD} ]] || fatal "ERROR: ${FLY_CMD} not found or not executable -- cannot continue."
}
####################################################
# parser functions
parse_opts() {
local arg
while (( $# )); do
arg=$1 ; shift
case ${arg} in
(-t|--target)
opt_target="$1"
shift
;;
(-p|--path)
opt_backup_dir="$1"
shift
;;
(*)
if [[ $INLINE_PARSING == 'true' ]]
then
NEW_ARGS=("$arg" "$@")
return
fi
cmd_opt_parser="${action}_opt_parser"
offset=0
if command -V $cmd_opt_parser | grep "is a function" 2>/dev/null 1>&2
then
$cmd_opt_parser $arg $*
fi
[[ "$offset" -gt 0 ]] || fatal "Unknown option '$arg'"
while (( offset = offset - 1 ))
do
shift
done
;;
esac
done
}
cmd_backup_opt_parser() {
local arg
arg=$1 ; shift
case ${arg} in
(-P|--pause)
opt_pause_first=true
offset=1
;;
(-c|--clean)
opt_clean=true
offset=1
;;
(-*)
offset=0
;;
(*)
target_pipelines+=($arg)
offset=1
;;
esac
}
cmd_restore_opt_parser() {
local arg
local opt
arg=$1 ; shift
case ${arg} in
(-P|--pause)
opt=$1
case ${opt} in
(y|yes|n|no|a|auto)
opt_pause_after_restore="${opt:0:1}"
;;
(*)
fatal "Error: $arg expects (y)es, (n)o or (a)uto"
;;
esac
offset=2
;;
(-c|--clean)
opt_clean=true
offset=1
;;
(--missing)
opt_restore_missing=true
offset=1
;;
(-*)
offset=0
;;
(*)
target_pipelines+=($arg)
offset=1
;;
esac
}
####################################################
# common functions used by other parts of hangar
fatal() {
for msg in "$@"
do
echo >&2 $msg
done
exit 1
}
flyrc() {
spruce json "$HOME/.flyrc" | jq -Mrj "$1"
}
parse_yaml() {
local prefix=$2
local s='[[:space:]]*' w='[-a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034')
sed -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \
-e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $1 |
awk -F$fs '{
indent = length($1)/2;
vname[indent] = $2;
for (i in vname) {if (i > indent) {delete vname[i]}}
if (length($3) > 0) {
vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
printf("%s%s%s=\"%s\"\n", "'$prefix'",vn, $2, $3);
}
}'
}
####################################################
# multi-call handlers
cmd_help() {
local topic=${1:-usage}
case ${topic} in
(usage)
cat >&2 <<EOF
USAGE: hangar <command> [arguments]
Common commands:
hangar clean Expunge the given (or default) backup directory
hangar backup Backup all pipelines
hangar restore Restore pipelines
hangar restore-state Restore the paused state of existing pipelines
hangar inventory List the backed up pipelines (inv for short)
hangar targets List the known targets, including the URL, if it's
the default target, and when last backed up to the
default backup directory.
hangar help Peruse the help! Give it a topic argument, or the name
of another command to get more detailed information.
Documentation!
Common options:
-t | --target <name> Specify the target concourse. By default, it will
use the value in \$CONCOURSE_TARGET if available.
-p | --path <path> Specify the backup path for storage or source for
restoring. By default, it is set to
~/.hangar/backups/<target>
EOF
exit 0
;;
(clean)
cat >&2 <<EOF
USAGE: hangar backup <options> <pipelines...>
Cleans out the backups from the backup directory
EOF
exit 0
;;
(backup)
cat >&2 <<EOF
USAGE: hangar backup <options> <pipelines...>
Retrieves the pipelines from concourse, and stores them in the backup directory,
including the paused state of the backup. If pipelines are specified, only those
pipelines will be backed up.
Options:
-P | --pause Pause the pipeline to be backed up before backing it up
-c | --clean Clean the backup directory before fetching the backups.
NOTE: The backup directory has to be empty before
the backups are taken.
EOF
exit 0
;;
(restore)
cat >&2 <<EOF
USAGE: hangar restore <options> <pipelines...>
Restores the pipelines from the backup directory to concourse. If pipelines are
specified, only those pipelines will be restored.
Options:
--missing Only restore pipelines that aren't present on the
target concourse.
-c | --clean Remove the backups after restoring them.
-P | --pause (yes|no) Pause or unpause the restored pipeline after restoring
it. 'yes' and 'no' can be abbreviated as 'y' and 'n'
respectively. By default, the pipelines will be
paused only if they were paused prior to being backed
up, otherwise they will be unpaused.
EOF
exit 0
;;
(*)
cat >&2 <<EOF
Unrecognized help topic '${topic}'.
Try one of these:
hangar help usage
hangar help <command>
EOF
exit 0
;;
esac
exit 1
}
cmd_clean() {
local backup
check_target
need_dirs
get_backups
local backup_dir="$(get_backup_dir)"
for backup in ${backups[@]}
do
rm "$backup_dir/$backup.yml"
rm "$backup_dir/$backup.state"
done
backups=()
}
cmd_backup() {
local pipeline
local result
check_target
need_dirs
need_fly
if [[ -n "$opt_clean" ]]
then
cmd_clean
fi
check_backup_directory_empty
get_pipelines
if [[ ${#target_pipelines[@]} -gt 0 ]]
then
for pipeline in "${target_pipelines[@]}"
do
if ! pipeline_exists $pipeline
then
fatal "$pipeline does not exist on specified Concourse target"
fi
done
else
target_pipelines=("${pipelines[@]}")
fi
echo "Backing up pipelines"
for pipeline in "${target_pipelines[@]}"
do
if [[ -n "$opt_pause_first" ]]
then
if [[ "$(pipeline_state $pipeline)" == "yes" ]]
then
echo " * ${pipeline} is already paused."
else
echo -n " * Pausing ${pipeline} first..."
result="$(${FLY_CMD} -t "$opt_target" pause-pipeline -p $pipeline)"
if [[ "$?" -ne 0 ]]
then
fatal "Could not pause pipeline ${pipeline}: $result"
fi
echo 'done!'
fi
fi
echo -n " * Fetching ${pipeline}..."
result="$(${FLY_CMD} -t "$opt_target" get-pipeline -p $pipeline)"
if [[ "$?" -ne 0 ]]
then
fatal "Could not fetch pipeline ${pipeline}: $result"
fi
echo "$result" > "${backup_dir}/${pipeline}.yml"
pipeline_state $pipeline > "${backup_dir}/${pipeline}.state"
echo 'done!'
echo ""
done
}
cmd_restore() {
local pipeline
local result
local desc
check_target
need_dirs
need_fly
echo "Restoring backups to $opt_target"
get_backups
if [[ ${#target_pipelines[@]} -gt 0 ]]
then
for pipeline in "${target_pipelines[@]}"
do
backup_exists $pipeline || fatal "No backup found for pipeline $pipeline in $backup_dir"
done
elif [[ -n "$opt_restore_missing" ]]
then
local missing_pipelines=()
get_pipelines
for pipeline in "${backups[@]}"
do
if ! pipeline_exists $pipeline
then
missing_pipelines+=($pipeline)
fi
done
target_pipelines=("${missing_pipelines[@]}")
else
target_pipelines=("${backups[@]}")
fi
if [[ "${#target_pipelines[@]}" -eq 0 ]]
then
echo "No pipelines to restore"
return
fi
for pipeline in "${target_pipelines[@]}"
do
echo -n " * Uploading ${pipeline}..."
result="$(${FLY_CMD} -t "$opt_target" set-pipeline -n -p $pipeline -c "${backup_dir}/${pipeline}.yml")"
[[ "$?" -eq 0 ]] || fatal "Could not set pipeline ${pipeline}: $result"
echo 'done!'
local pause_cmd
# TODO: why aren't we using $(backup_state $pipeline)
pipeline_state="$(cat "${backup_dir}/${pipeline}.state")"
case ${opt_pause_after_restore/a/$pipeline_state} in
(y|yes)
pause_cmd="pp"
desc="Pausing"
;;
(n|no)
pause_cmd="up"
desc="Unpausing"
;;
(*)
pause_cmd=""
;;
esac
if [[ -n "$pause_cmd" ]]
then
echo -n " * ${desc} ${pipeline}..."
result="$(${FLY_CMD} -t "$opt_target" $pause_cmd -p $pipeline)"
[[ "$?" -eq 0 ]] || fatal "$desc pipeline ${pipeline} failed: $result"
echo 'done!'
fi
if [[ -n "$opt_clean" ]]
then
echo " * Removing backup"
rm "${backup_dir}/${pipeline}.yml"
rm "${backup_dir}/${pipeline}.state"
fi
echo ""
done
}
cmd_restore_state() {
local pipeline
local result
check_target
need_dirs
need_fly
get_backups
if [[ ${#target_pipelines[@]} -gt 0 ]]
then
for pipeline in "${target_pipelines[@]}"
do
backup_exists $pipeline || fatal "No backup found for pipeline $pipeline in $backup_dir"
done
else
local present_pipelines=()
get_pipelines
for pipeline in "${backups[@]}"
do
if pipeline_exists $pipeline
then
present_pipelines+=($pipeline)
else
echo " ! WARNING: Pipeline $pipeline was not found on specified Concourse target"
fi
done
target_pipelines=("${present_pipelines[@]}")
fi
for pipeline in "${target_pipelines[@]}"
do
local pause_cmd
# TODO: why aren't we using $(backup_state $pipeline)
pipeline_state="$(cat "${backup_dir}/${pipeline}.state")"
case ${pipeline_state} in
(y|yes)
pause_cmd="pp"
state_action=" - Pausing"
;;
(n|no)
pause_cmd="up"
state_action=" - Unpausing"
;;
(*)
echo " ! WARNING: $pipeline state of ${pipeline_state} is not valid: expecting yes or no -- cannot restore"
continue
;;
esac
result="$(${HANGAR_DIR}/fly -t "$opt_target" $pause_cmd -p $pipeline)"
echo "$state_action pipeline $pipeline"
[[ "$?" -eq 0 ]] || echo " ! Could not restore state of pipeline ${pipeline}: $result"
done
}
cmd_inv() {
local pipeline
local result
check_target
need_dirs
get_backups
local on_mac=no; [[ "$(uname -s)" == "Darwin" ]] && on_mac=yes
echo "Date Paused Name"
echo "------------------ ------ --------------------------------------------------------------"
for pipeline in "${backups[@]}"
do
if [[ ${on_mac} == "yes" ]]
then
ts="$(stat -f "%Sa" -t '%Y-%m-%d@%H:%M:%S' ~/.hangar/backups/sw/vault-boshrelease.yml)"
else
ts="$(date -d "$(stat -c "%z" ${backup_dir}/${pipeline}.yml)" +"%Y-%m-%d@%H:%M:%S")"
fi
# TODO: why aren't we using $(backup_state $pipeline)
pipeline_state="$(cat "${backup_dir}/${pipeline}.state")"
printf "%-18s%-8s%s\n" "$ts" "$pipeline_state" "$pipeline"
done
}
cmd_targets() {
local backup_dir
local latest_file
local backup_date
echo Known Targets:
echo --------------
while read -r target url || [[ -n $target ]]
do
default=""
[[ "$CONCOURSE_TARGET" == "$target" ]] && default=" (default)"
echo ""
echo "$target$default"
echo " URL: $url"
opt_target="$target"
backup_dir=$(get_backup_dir)
if [[ -d $backup_dir ]]
then
latest_file=$(ls -1tr $backup_dir | tail -n1)
if [[ -f "${backup_dir}/${latest_file}" ]]
then
backup_date=$(stat -f "%Sc" -t '%Y-%m-%d@%H:%M' "${backup_dir}/${latest_file}")
echo " Last backed up to $backup_dir on $backup_date"
fi
fi
done < <(parse_yaml ~/.flyrc fly_ | grep 'fly_targets_.*_api=' | sed -E 's/^fly_targets_(.*)_api="(.*)"$/\1 \2/')
}
bad_command() {
local cmd=${1}
cat >&2 <<EOF
Unrecognized sub-command: '$cmd'
Try one of these:
hangar help
hangar version
hangar clean
hangar backup
hangar restore
hangar restore-state
hangar inv[entory]
hangar targets
EOF
exit 1
}
checksum() {
if [[ -z $(command -v sha1sum) ]]; then
shasum "$@"
else
sha1sum "$@"
fi
}
####################################################
# multi-call interface
main() {
action=''
while [[ -z ${action} ]]
do
local cmd_name=${1:-help} ; shift
case "${cmd_name}" in
(ping)
exit 0
;;
(version|-v|--version)
s=$(cat ${BASH_SOURCE[0]} | checksum)
echo "hangar v${VERSION} (${s:0:12})"
exit 0
;;
(help|-h|--help|-\?)
cmd_help $*
exit 0
;;
(-t|-p)
# Common options before commmand
INLINE_PARSING=true
set -- "$cmd_name" "$@"
parse_opts "$@"
set -- "$NEW_ARGS"
INLINE_PARSING=false
;;
(clean)
# Delete all existing backups
action=cmd_clean
;;
(backup)
# Backup the pipelines
action=cmd_backup
;;
(restore)
# Restore the pipelines
action=cmd_restore
;;
(restore-state)
# Restore the paused state of existing pipelines
action=cmd_restore_state
;;
(inv|inventory)
# List the contents of the hanger
action=cmd_inv
;;
(targets)
# List available/default targets
action=cmd_targets
;;
(*)
bad_command $cmd_name
esac
done
parse_opts "$@"
$action
}
main "$@"
echo ""
# fin
Reference in New Issue View Git Blame Copy Permalink