SandW
/
show-me-secrets
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

setup ci to build helm chart

This commit is contained in:
Dr Nic Williams 2018-11-16 11:14:53 +10:00
parent bb404c4a88
commit c6b6b3f247
5 changed files with 592 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

288
ci/pipeline.yml Normal file
View File

@ -0,0 +1,288 @@
---
#
# ci/pipeline.yml
#
# Pipeline structure file for a Helm Chart & Docker Image pipeline
#
# DO NOT MAKE CHANGES TO THIS FILE. Instead, modify
# ci/settings.yml and override what needs overridden.
# This uses spruce, so you have some options there.
#
# author: Dr Nic Williams <drnicwilliams@gmail.com>
# created: 2018-11-09
meta:
name: (( param "Please name your pipeline" ))
release: (( grab meta.name ))
target: (( param "Please identify the name of the target Concourse CI" ))
url: (( param "Please specify the full url of the target Concourse CI" ))
pipeline: (( grab meta.name ))
image:
name: starkandwayne/concourse-kubernetes
tag: latest
git:
email: (( param "Please provide the git email for automated commits" ))
name: (( param "Please provide the git name for automated commits" ))
aws:
bucket: (( concat meta.name "-pipeline" ))
region_name: us-east-1
access_key: (( param "Please set your AWS Access Key ID" ))
secret_key: (( param "Please set your AWS Secret Key ID" ))
charts_uri: (( param "Please set your s3://bucket-name/charts URI" ))
github:
uri: (( concat "git@github.com:" meta.github.owner "/" meta.github.repo ))
owner: (( param "Please specify the name of the user / organization that owns the Github repository" ))
repo: (( param "Please specify the name of the Github repository" ))
branch: master
private_key: (( param "Please generate an SSH Deployment Key for this repo and specify it here" ))
access_token: (( param "Please generate a Personal Access Token and specify it here" ))
dockerhub:
username: (( param "Please specify the username for your Dockerhub account" ))
password: (( param "Please specify the password for your Dockerhub account" ))
repository: (( param "Please specify the name of the image (repo/name) that you are building" ))
slack:
webhook: (( param "Please specify your Slack Incoming Webhook Integration URL" ))
success_moji: ":airplane_departure:"
fail_moji: ":airplane_arriving:"
upset_moji: ":sad_panda:"
channel: (( param "Please specify the channel (#name) or user (@user) to send messages to" ))
username: concourse
icon: https://cl.ly/2F421Y300u07/concourse-logo-blue-transparent.png
fail_url: '(( concat "<" meta.url "/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME| Concourse Failure! " meta.slack.upset_moji ">" ))'
groups:
- name: (( grab meta.name ))
jobs:
- latest-image
- rc
- shipit
- name: versioning
jobs:
- major
- minor
- patch
jobs:
- name: latest-image
public: true
serial: true
plan:
- get: git
trigger: true
- put: image-latest
params:
build: git
on_failure:
put: notify
params:
channel: (( grab meta.slack.channel ))
username: (( grab meta.slack.username ))
icon_url: (( grab meta.slack.icon ))
text: '(( concat meta.slack.fail_url " " meta.pipeline ": patch job failed" ))'
- name: shipit
public: true
serial: true
plan:
- do:
- name: inputs
aggregate:
# - { get: version, params: {bump: final} }
# - { get: git }
- { get: version, passed: [rc], params: {bump: final} }
- { get: git, passed: [rc] }
- { get: image-latest, passed: [latest-image], params: { save: true } }
- aggregate:
- name: docker-push-tag
put: image-latest # as 'vX.Y.Z'
params:
tag: version/number
load: image-latest
- name: package-chart
task: package-chart
config:
platform: linux
image_resource:
type: docker-image
source:
repository: (( grab meta.image.name ))
tag: (( grab meta.image.tag ))
inputs:
- name: version
- name: git
outputs:
- name: gh
- name: pushme
- name: notifications
run:
path: ./git/ci/scripts/shipit
args: []
params:
CHART_NAME: (( grab meta.name ))
CHART_ROOT: .
REPO_ROOT: git
VERSION_FROM: version/number
RELEASE_ROOT: gh
REPO_OUT: pushme
BRANCH: (( grab meta.github.branch ))
GITHUB_OWNER: (( grab meta.github.owner ))
GIT_EMAIL: (( grab meta.git.email ))
GIT_NAME: (( grab meta.git.name ))
NOTIFICATION_OUT: notifications
AWS_ACCESS_KEY_ID: (( grab meta.aws.access_key ))
AWS_SECRET_ACCESS_KEY: (( grab meta.aws.secret_key ))
AWS_DEFAULT_REGION: (( grab meta.aws.region_name ))
HELM_S3_BUCKET_URI: (( grab meta.aws.charts_uri ))
- name: upload-git
put: git
params:
rebase: true
repository: pushme
- name: github-release
put: github
params:
name: gh/name
tag: gh/tag
body: gh/notes.md
globs: [gh/artifacts/*]
- name: version-bump
put: version
params:
bump: patch
# - name: notify
# aggregate:
# - put: notify
# params:
# channel: (( grab meta.slack.channel ))
# username: (( grab meta.slack.username ))
# icon_url: (( grab meta.slack.icon ))
# text_file: notifications/message
on_failure:
put: notify
params:
channel: (( grab meta.slack.channel ))
username: (( grab meta.slack.username ))
icon_url: (( grab meta.slack.icon ))
text: '(( concat meta.slack.fail_url " " meta.pipeline ": shipit job failed" ))'
- name: rc
public: true
serial: true
plan:
- do:
- aggregate:
# - { get: git, trigger: true }
- { get: git, trigger: true, passed: [latest-image] }
- { get: version, trigger: true, params: {pre: rc} }
- task: release-notes
config:
platform: linux
image_resource:
type: docker-image
source:
repository: (( grab meta.image.name ))
tag: (( grab meta.image.tag ))
inputs:
- { name: git }
run:
path: sh
args:
- -ce
- |
cd git
if [ -f ci/release_notes.md ]; then
echo "###### RELEASE NOTES ###############"
echo
cat ci/release_notes.md
echo
echo "########################################"
echo
else
echo "NO RELEASE NOTES HAVE BEEN WRITTEN"
echo "You *might* want to do that before"
echo "hitting (+) on that shipit job..."
echo
fi
- put: version
params: {file: version/number}
on_failure:
put: notify
params:
channel: (( grab meta.slack.channel ))
username: (( grab meta.slack.username ))
icon_url: (( grab meta.slack.icon ))
text: '(( concat meta.slack.fail_url " " meta.pipeline ": rc job failed" ))'
- name: minor
public: true
plan:
- { get: version, trigger: false, params: {bump: minor} }
- { put: version, params: {file: version/number} }
- name: major
public: true
plan:
- { get: version, trigger: false, params: {bump: major} }
- { put: version, params: {file: version/number} }
- name: patch
public: true
plan:
- do:
- { get: version, trigger: false, params: {bump: patch} }
- { put: version, params: {file: version/number} }
on_failure:
put: notify
params:
channel: (( grab meta.slack.channel ))
username: (( grab meta.slack.username ))
icon_url: (( grab meta.slack.icon ))
text: '(( concat meta.slack.fail_url " " meta.pipeline ": patch job failed" ))'
resource_types:
- name: slack-notification
type: docker-image
source:
repository: cfcommunity/slack-notification-resource
resources:
- name: git
type: git
source:
uri: (( grab meta.github.uri ))
branch: (( grab meta.github.branch ))
private_key: (( grab meta.github.private_key ))
- name: image-latest
type: docker-image
source:
.: (( inject meta.dockerhub ))
tag: latest
- name: version
type: semver
source :
driver: s3
bucket: (( grab meta.aws.bucket ))
region_name: (( grab meta.aws.region_name ))
key: version
access_key_id: (( grab meta.aws.access_key ))
secret_access_key: (( grab meta.aws.secret_key ))
initial_version: (( grab meta.initial_version || "0.0.1" ))
- name: notify
type: slack-notification
source:
url: (( grab meta.slack.webhook ))
- name: github
type: github-release
source:
user: (( grab meta.github.owner ))
repository: (( grab meta.github.repo ))
access_token: (( grab meta.github.access_token ))

1
ci/release_notes.md Normal file
View File

@ -0,0 +1 @@
Initial release of Helm chart

151
ci/repipe Executable file
View File

@ -0,0 +1,151 @@
#!/bin/bash
#
# ci/repipe
#
# Script for merging together pipeline configuration files
# (via Spruce!) and configuring Concourse.
#
# author: James Hunt <james@niftylogic.com>
# Dennis Bell <dennis.j.bell@gmail.com>
# created: 2016-03-04
need_command() {
local cmd=${1:?need_command() - no command name given}
if [[ ! -x "$(command -v $cmd)" ]]; then
echo >&2 "${cmd} is not installed."
if [[ "${cmd}" == "spruce" ]]; then
echo >&2 "Please download it from https://github.com/geofffranks/spruce/releases"
fi
exit 2
fi
}
NO_FLY=
SAVE_MANIFEST=
VALIDATE_PIPELINE=
NON_INTERACTIVE=
cleanup() {
rm -f save-manifest.yml
if [[ -n ${SAVE_MANIFEST} && -e .deploy.yml ]]; then
mv .deploy.yml save-manifest.yml
fi
rm -f .deploy.yml
}
usage() {
echo Command line arguments:
echo "no-fly Do not execute any fly commands"
echo "save-manifest Save manifest to file save-manifest"
echo "validate Validate pipeline instead of set pipeline"
echo "validate-strict Validate pipeline with strict mode"
echo "non-interactive Run set-pipeline in non-interactive mode"
echo "open Open pipeline dashboard to browser (if possible)"
}
open_pipeline() {
url=$(show_pipeline_url)
cleanup
if [[ -x /usr/bin/open ]]; then
exec /usr/bin/open "$url"
else
echo "Sorry, but I was not able to automagically open"
echo "your Concourse Pipeline in the browser."
echo
echo "Here's a link you can click on, though:"
echo
echo " $url"
echo
exit 0;
fi
}
show_pipeline_url() {
spruce merge --skip-eval pipeline.yml ${settings_file} > .deploy.yml
concourse_url=$(spruce json .deploy.yml | jq -r ".meta.url")
team=$(spruce json .deploy.yml | jq -r ".meta.team // \"main\"")
pipeline=$(spruce merge --skip-eval \
--cherry-pick meta.pipeline \
--cherry-pick meta.name \
.deploy.yml | spruce merge - | spruce json | jq -r ".meta.pipeline")
echo "$concourse_url/teams/$team/pipelines/$pipeline"
exit 0
}
for arg do
case "${arg}" in
no-fly|no_fly) NO_FLY="yes" ;;
save-manifest|save_manifest) SAVE_MANIFEST="yes" ;;
validate) VALIDATE_PIPELINE="normal" ;;
validate-strict|validate_strict) VALIDATE_PIPELINE="strict" ;;
non-interactive|non_interactive) NON_INTERACTIVE="--non-interactive" ;;
url) SHOW_PIPELINE_URL="yes" ;;
open) OPEN_PIPELINE="yes" ;;
help|-h|--help) usage; exit 0 ;;
*) echo Invalid argument
usage
exit 1
esac
done
cd $(dirname $BASH_SOURCE[0])
echo >&2 "Working in $(pwd)"
need_command spruce
# Allow for target-specific settings
settings_file="$(ls -1 settings.yml ${CONCOURSE_TARGET:+"settings-${CONCOURSE_TARGET}.yml"} 2>/dev/null | tail -n1)"
if [[ -z "$settings_file" ]]
then
echo >&2 "Missing local settings in ci/settings.yml${CONCOURSE_TARGET:+" or ci/settings-${CONCOURSE_TARGET}.yml"}!"
exit 1
fi
echo >&2 "Using settings found in ${settings_file}"
set -e
trap "cleanup" QUIT TERM EXIT INT
[[ -n ${SHOW_PIPELINE_URL} ]] && { show_pipeline_url; exit 0; }
[[ -n ${OPEN_PIPELINE} ]] && { open_pipeline; exit 0; }
spruce merge pipeline.yml ${settings_file} > .deploy.yml
PIPELINE=$(spruce json .deploy.yml | jq -r '.meta.pipeline // ""')
if [[ -z ${PIPELINE} ]]; then
echo >&2 "Missing pipeline name in ci/settings.yml!"
exit 1
fi
TARGET_FROM_SETTINGS=$(spruce json .deploy.yml | jq -r '.meta.target // ""')
if [[ -z ${CONCOURSE_TARGET} ]]; then
TARGET=${TARGET_FROM_SETTINGS}
elif [[ "$CONCOURSE_TARGET" != "$TARGET_FROM_SETTINGS" ]]
then
echo >&2 "Target in {$settings_file} differs from target in \$CONCOURSE_TARGET"
echo >&2 " \$CONCOURSE_TARGET: $CONCOURSE_TARGET"
echo >&2 " Target in file: $TARGET_FROM_SETTINGS"
exit 1
else
TARGET=${CONCOURSE_TARGET}
fi
if [[ -z ${TARGET} ]]; then
echo >&2 "Missing Concourse Target in ci/settings.yml!"
exit 1
fi
fly_cmd="${FLY_CMD:-fly}"
[[ -n ${NO_FLY} ]] && { echo no fly execution requested ; exit 0; }
case "${VALIDATE_PIPELINE}" in
normal) fly_opts="validate-pipeline" ;;
strict) fly_opts="validate-pipeline --strict" ;;
*) fly_opts="set-pipeline ${NON_INTERACTIVE} --pipeline ${PIPELINE}" ;;
esac
set +x
$fly_cmd --target ${TARGET} ${fly_opts} --config .deploy.yml
[[ -n ${VALIDATE_PIPELINE} ]] && exit 0
$fly_cmd --target ${TARGET} unpause-pipeline --pipeline ${PIPELINE}

115
ci/scripts/shipit Executable file
View File

@ -0,0 +1,115 @@
#!/bin/bash
#
# ci/scripts/shipit
#
# Script for finalizing and packaging Helm chart
# and managing release notes
#
# author: Dr Nic Williams <drnicwilliams@gmail.com>
# created: 2018-11-09
set -eu
header() {
echo
echo "###############################################"
echo
echo $*
echo
}
: ${CHART_NAME:?required}
: ${CHART_ROOT:?required}
: ${REPO_ROOT:?required}
: ${VERSION_FROM:?required}
: ${RELEASE_ROOT:?required}
: ${REPO_OUT:?required}
: ${BRANCH:?required}
: ${GITHUB_OWNER:?required}
: ${GIT_EMAIL:?required}
: ${GIT_NAME:?required}
: ${NOTIFICATION_OUT:?required}
: ${AWS_ACCESS_KEY_ID:?required}
: ${AWS_SECRET_ACCESS_KEY:?required}
: ${HELM_S3_BUCKET_URI:?required}
HELM_VERSION=2.11.0
if [[ ! -f ${VERSION_FROM} ]]; then
echo >&2 "Version file (${VERSION_FROM}) not found. Did you misconfigure Concourse?"
exit 2
fi
VERSION=$(cat ${VERSION_FROM})
if [[ -z ${VERSION} ]]; then
echo >&2 "Version file (${VERSION_FROM}) was empty. Did you misconfigure Concourse?"
exit 2
fi
if [[ ! -f ${REPO_ROOT}/ci/release_notes.md ]]; then
echo >&2 "ci/release_notes.md not found. Did you forget to write them?"
exit 1
fi
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
###############################################################
git clone ${REPO_ROOT} ${REPO_OUT}
pushd ${REPO_OUT}
header "Bump version in Chart.yaml"
tmpfile=$(mktemp /tmp/chart-yaml.XXXX)
sed -e "s/^version:.*$/version: ${VERSION}/g" ${CHART_ROOT}/Chart.yaml > $tmpfile
cp $tmpfile ${CHART_ROOT}/Chart.yaml
header "Bump docker image version in Values.yaml"
tmpfile=$(mktemp /tmp/chart-yaml.XXXX)
sed -e "s/^ tag:.*$/ tag: ${VERSION}/g" ${CHART_ROOT}/values.yaml > $tmpfile
cp $tmpfile ${CHART_ROOT}/values.yaml
popd
header "Install helm"
curl -LO https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz
tar xfz helm-*.tar.gz
mv linux-amd64/helm /usr/local/bin
helm version --client
helm init --client-only
header "Install helm-s3 plugin"
helm plugin install https://github.com/hypnoglow/helm-s3.git
header "Build helm chart"
mkdir -p ${RELEASE_ROOT}/artifacts
helm package ${REPO_OUT}/${CHART_ROOT} -d ${RELEASE_ROOT}/artifacts
helm repo add our-repo ${HELM_S3_BUCKET_URI}
header "Uploading helm chart to ${HELM_S3_BUCKET_URI}"
set -e
helm s3 push ${RELEASE_ROOT}/artifacts/${CHART_NAME}*.tgz our-repo --force
set +e
popd
header "Update git repo with final release..."
if [[ -z $(git config --global user.email) ]]; then
git config --global user.email "${GIT_EMAIL}"
fi
if [[ -z $(git config --global user.name) ]]; then
git config --global user.name "${GIT_NAME}"
fi
echo "v${VERSION}" > ${RELEASE_ROOT}/tag
echo "v${VERSION}" > ${RELEASE_ROOT}/name
mv ${REPO_OUT}/ci/release_notes.md ${RELEASE_ROOT}/notes.md
pushd $REPO_OUT
git merge --no-edit ${BRANCH}
git add -A
git status
git commit -m "release v${VERSION}"
popd
cat > ${NOTIFICATION_OUT:-notifications}/message <<EOS
New ${CHART_NAME} v${VERSION} released. <https://github.com/${GITHUB_OWNER}/${CHART_NAME}/releases/tag/v${VERSION}|Release notes>.
EOS

37
ci/settings.yml Normal file
View File

@ -0,0 +1,37 @@
---
meta:
name: show-me-secrets
target: ohio-sw
url: https://ci2.starkandwayne.com
team: starkandwayne
initial_version: 0.1.0
git:
email: ((git-commit-email))
name: ((git-commit-name))
aws:
bucket: (( grab meta.pipeline ))
region_name: us-east-1
access_key: ((aws-access-key))
secret_key: ((aws-secret-key))
charts_uri: s3://helm.starkandwayne.com/charts
github:
owner: starkandwayne
repo: show-me-secrets
branch: master
private_key: ((github-private-key))
access_token: ((github-access-token))
dockerhub:
username: ((docker-hub-username))
password: ((docker-hub-password))
repository: ((docker-hub-username))/show-me-secrets
slack:
webhook: ((slack-webhook))
username: ((slack-username))
icon: ((slack-icon-url))
channel: "#show-me-secrets"