From c6b6b3f2479fd5d3c8825a77bf40e0ca114b0bc9 Mon Sep 17 00:00:00 2001 From: Dr Nic Williams Date: Fri, 16 Nov 2018 11:14:53 +1000 Subject: [PATCH] setup ci to build helm chart --- ci/pipeline.yml | 288 ++++++++++++++++++++++++++++++++++++++++++++ ci/release_notes.md | 1 + ci/repipe | 151 +++++++++++++++++++++++ ci/scripts/shipit | 115 ++++++++++++++++++ ci/settings.yml | 37 ++++++ 5 files changed, 592 insertions(+) create mode 100644 ci/pipeline.yml create mode 100644 ci/release_notes.md create mode 100755 ci/repipe create mode 100755 ci/scripts/shipit create mode 100644 ci/settings.yml diff --git a/ci/pipeline.yml b/ci/pipeline.yml new file mode 100644 index 0000000..bccc8f9 --- /dev/null +++ b/ci/pipeline.yml @@ -0,0 +1,288 @@ +--- +# +# ci/pipeline.yml +# +# Pipeline structure file for a Helm Chart & Docker Image pipeline +# +# DO NOT MAKE CHANGES TO THIS FILE. Instead, modify +# ci/settings.yml and override what needs overridden. +# This uses spruce, so you have some options there. +# +# author: Dr Nic Williams +# created: 2018-11-09 + +meta: + name: (( param "Please name your pipeline" )) + release: (( grab meta.name )) + target: (( param "Please identify the name of the target Concourse CI" )) + url: (( param "Please specify the full url of the target Concourse CI" )) + pipeline: (( grab meta.name )) + image: + name: starkandwayne/concourse-kubernetes + tag: latest + + git: + email: (( param "Please provide the git email for automated commits" )) + name: (( param "Please provide the git name for automated commits" )) + + aws: + bucket: (( concat meta.name "-pipeline" )) + region_name: us-east-1 + access_key: (( param "Please set your AWS Access Key ID" )) + secret_key: (( param "Please set your AWS Secret Key ID" )) + charts_uri: (( param "Please set your s3://bucket-name/charts URI" )) + + github: + uri: (( concat "git@github.com:" meta.github.owner "/" meta.github.repo )) + owner: (( param "Please specify the name of the user / organization that owns the Github repository" )) + repo: (( param "Please specify the name of the Github repository" )) + branch: master + private_key: (( param "Please generate an SSH Deployment Key for this repo and specify it here" )) + access_token: (( param "Please generate a Personal Access Token and specify it here" )) + + dockerhub: + username: (( param "Please specify the username for your Dockerhub account" )) + password: (( param "Please specify the password for your Dockerhub account" )) + repository: (( param "Please specify the name of the image (repo/name) that you are building" )) + + slack: + webhook: (( param "Please specify your Slack Incoming Webhook Integration URL" )) + success_moji: ":airplane_departure:" + fail_moji: ":airplane_arriving:" + upset_moji: ":sad_panda:" + channel: (( param "Please specify the channel (#name) or user (@user) to send messages to" )) + username: concourse + icon: https://cl.ly/2F421Y300u07/concourse-logo-blue-transparent.png + fail_url: '(( concat "<" meta.url "/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME| Concourse Failure! " meta.slack.upset_moji ">" ))' + +groups: + - name: (( grab meta.name )) + jobs: + - latest-image + - rc + - shipit + - name: versioning + jobs: + - major + - minor + - patch +jobs: + - name: latest-image + public: true + serial: true + plan: + - get: git + trigger: true + - put: image-latest + params: + build: git + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": patch job failed" ))' + + - name: shipit + public: true + serial: true + plan: + - do: + - name: inputs + aggregate: + # - { get: version, params: {bump: final} } + # - { get: git } + - { get: version, passed: [rc], params: {bump: final} } + - { get: git, passed: [rc] } + - { get: image-latest, passed: [latest-image], params: { save: true } } + - aggregate: + - name: docker-push-tag + put: image-latest # as 'vX.Y.Z' + params: + tag: version/number + load: image-latest + - name: package-chart + task: package-chart + config: + platform: linux + image_resource: + type: docker-image + source: + repository: (( grab meta.image.name )) + tag: (( grab meta.image.tag )) + inputs: + - name: version + - name: git + outputs: + - name: gh + - name: pushme + - name: notifications + run: + path: ./git/ci/scripts/shipit + args: [] + params: + CHART_NAME: (( grab meta.name )) + CHART_ROOT: . + REPO_ROOT: git + VERSION_FROM: version/number + RELEASE_ROOT: gh + REPO_OUT: pushme + BRANCH: (( grab meta.github.branch )) + GITHUB_OWNER: (( grab meta.github.owner )) + GIT_EMAIL: (( grab meta.git.email )) + GIT_NAME: (( grab meta.git.name )) + NOTIFICATION_OUT: notifications + AWS_ACCESS_KEY_ID: (( grab meta.aws.access_key )) + AWS_SECRET_ACCESS_KEY: (( grab meta.aws.secret_key )) + AWS_DEFAULT_REGION: (( grab meta.aws.region_name )) + HELM_S3_BUCKET_URI: (( grab meta.aws.charts_uri )) + + - name: upload-git + put: git + params: + rebase: true + repository: pushme + - name: github-release + put: github + params: + name: gh/name + tag: gh/tag + body: gh/notes.md + globs: [gh/artifacts/*] + - name: version-bump + put: version + params: + bump: patch + # - name: notify + # aggregate: + # - put: notify + # params: + # channel: (( grab meta.slack.channel )) + # username: (( grab meta.slack.username )) + # icon_url: (( grab meta.slack.icon )) + # text_file: notifications/message + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": shipit job failed" ))' + + - name: rc + public: true + serial: true + plan: + - do: + - aggregate: + # - { get: git, trigger: true } + - { get: git, trigger: true, passed: [latest-image] } + - { get: version, trigger: true, params: {pre: rc} } + - task: release-notes + config: + platform: linux + image_resource: + type: docker-image + source: + repository: (( grab meta.image.name )) + tag: (( grab meta.image.tag )) + inputs: + - { name: git } + run: + path: sh + args: + - -ce + - | + cd git + if [ -f ci/release_notes.md ]; then + echo "###### RELEASE NOTES ###############" + echo + cat ci/release_notes.md + echo + echo "########################################" + echo + else + echo "NO RELEASE NOTES HAVE BEEN WRITTEN" + echo "You *might* want to do that before" + echo "hitting (+) on that shipit job..." + echo + fi + - put: version + params: {file: version/number} + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": rc job failed" ))' + + - name: minor + public: true + plan: + - { get: version, trigger: false, params: {bump: minor} } + - { put: version, params: {file: version/number} } + + - name: major + public: true + plan: + - { get: version, trigger: false, params: {bump: major} } + - { put: version, params: {file: version/number} } + + - name: patch + public: true + plan: + - do: + - { get: version, trigger: false, params: {bump: patch} } + - { put: version, params: {file: version/number} } + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": patch job failed" ))' + +resource_types: + - name: slack-notification + type: docker-image + source: + repository: cfcommunity/slack-notification-resource + +resources: + - name: git + type: git + source: + uri: (( grab meta.github.uri )) + branch: (( grab meta.github.branch )) + private_key: (( grab meta.github.private_key )) + + - name: image-latest + type: docker-image + source: + .: (( inject meta.dockerhub )) + tag: latest + + - name: version + type: semver + source : + driver: s3 + bucket: (( grab meta.aws.bucket )) + region_name: (( grab meta.aws.region_name )) + key: version + access_key_id: (( grab meta.aws.access_key )) + secret_access_key: (( grab meta.aws.secret_key )) + initial_version: (( grab meta.initial_version || "0.0.1" )) + + - name: notify + type: slack-notification + source: + url: (( grab meta.slack.webhook )) + + - name: github + type: github-release + source: + user: (( grab meta.github.owner )) + repository: (( grab meta.github.repo )) + access_token: (( grab meta.github.access_token )) diff --git a/ci/release_notes.md b/ci/release_notes.md new file mode 100644 index 0000000..7761285 --- /dev/null +++ b/ci/release_notes.md @@ -0,0 +1 @@ +Initial release of Helm chart \ No newline at end of file diff --git a/ci/repipe b/ci/repipe new file mode 100755 index 0000000..72653df --- /dev/null +++ b/ci/repipe @@ -0,0 +1,151 @@ +#!/bin/bash +# +# ci/repipe +# +# Script for merging together pipeline configuration files +# (via Spruce!) and configuring Concourse. +# +# author: James Hunt +# Dennis Bell +# created: 2016-03-04 + +need_command() { + local cmd=${1:?need_command() - no command name given} + + if [[ ! -x "$(command -v $cmd)" ]]; then + echo >&2 "${cmd} is not installed." + if [[ "${cmd}" == "spruce" ]]; then + echo >&2 "Please download it from https://github.com/geofffranks/spruce/releases" + fi + exit 2 + fi +} + +NO_FLY= +SAVE_MANIFEST= +VALIDATE_PIPELINE= +NON_INTERACTIVE= + +cleanup() { + rm -f save-manifest.yml + if [[ -n ${SAVE_MANIFEST} && -e .deploy.yml ]]; then + mv .deploy.yml save-manifest.yml + fi + rm -f .deploy.yml +} + +usage() { + echo Command line arguments: + echo "no-fly Do not execute any fly commands" + echo "save-manifest Save manifest to file save-manifest" + echo "validate Validate pipeline instead of set pipeline" + echo "validate-strict Validate pipeline with strict mode" + echo "non-interactive Run set-pipeline in non-interactive mode" + echo "open Open pipeline dashboard to browser (if possible)" +} + +open_pipeline() { + url=$(show_pipeline_url) + cleanup + if [[ -x /usr/bin/open ]]; then + exec /usr/bin/open "$url" + else + echo "Sorry, but I was not able to automagically open" + echo "your Concourse Pipeline in the browser." + echo + echo "Here's a link you can click on, though:" + echo + echo " $url" + echo + exit 0; + fi +} + +show_pipeline_url() { + spruce merge --skip-eval pipeline.yml ${settings_file} > .deploy.yml + concourse_url=$(spruce json .deploy.yml | jq -r ".meta.url") + team=$(spruce json .deploy.yml | jq -r ".meta.team // \"main\"") + pipeline=$(spruce merge --skip-eval \ + --cherry-pick meta.pipeline \ + --cherry-pick meta.name \ + .deploy.yml | spruce merge - | spruce json | jq -r ".meta.pipeline") + + echo "$concourse_url/teams/$team/pipelines/$pipeline" + exit 0 +} + +for arg do + case "${arg}" in + no-fly|no_fly) NO_FLY="yes" ;; + save-manifest|save_manifest) SAVE_MANIFEST="yes" ;; + validate) VALIDATE_PIPELINE="normal" ;; + validate-strict|validate_strict) VALIDATE_PIPELINE="strict" ;; + non-interactive|non_interactive) NON_INTERACTIVE="--non-interactive" ;; + url) SHOW_PIPELINE_URL="yes" ;; + open) OPEN_PIPELINE="yes" ;; + help|-h|--help) usage; exit 0 ;; + *) echo Invalid argument + usage + exit 1 + esac +done + +cd $(dirname $BASH_SOURCE[0]) +echo >&2 "Working in $(pwd)" +need_command spruce + +# Allow for target-specific settings +settings_file="$(ls -1 settings.yml ${CONCOURSE_TARGET:+"settings-${CONCOURSE_TARGET}.yml"} 2>/dev/null | tail -n1)" +if [[ -z "$settings_file" ]] +then + echo >&2 "Missing local settings in ci/settings.yml${CONCOURSE_TARGET:+" or ci/settings-${CONCOURSE_TARGET}.yml"}!" + exit 1 +fi + +echo >&2 "Using settings found in ${settings_file}" + +set -e +trap "cleanup" QUIT TERM EXIT INT + +[[ -n ${SHOW_PIPELINE_URL} ]] && { show_pipeline_url; exit 0; } +[[ -n ${OPEN_PIPELINE} ]] && { open_pipeline; exit 0; } + +spruce merge pipeline.yml ${settings_file} > .deploy.yml +PIPELINE=$(spruce json .deploy.yml | jq -r '.meta.pipeline // ""') +if [[ -z ${PIPELINE} ]]; then + echo >&2 "Missing pipeline name in ci/settings.yml!" + exit 1 +fi + +TARGET_FROM_SETTINGS=$(spruce json .deploy.yml | jq -r '.meta.target // ""') +if [[ -z ${CONCOURSE_TARGET} ]]; then + TARGET=${TARGET_FROM_SETTINGS} +elif [[ "$CONCOURSE_TARGET" != "$TARGET_FROM_SETTINGS" ]] +then + echo >&2 "Target in {$settings_file} differs from target in \$CONCOURSE_TARGET" + echo >&2 " \$CONCOURSE_TARGET: $CONCOURSE_TARGET" + echo >&2 " Target in file: $TARGET_FROM_SETTINGS" + exit 1 +else + TARGET=${CONCOURSE_TARGET} +fi + +if [[ -z ${TARGET} ]]; then + echo >&2 "Missing Concourse Target in ci/settings.yml!" + exit 1 +fi + +fly_cmd="${FLY_CMD:-fly}" + +[[ -n ${NO_FLY} ]] && { echo no fly execution requested ; exit 0; } + +case "${VALIDATE_PIPELINE}" in + normal) fly_opts="validate-pipeline" ;; + strict) fly_opts="validate-pipeline --strict" ;; + *) fly_opts="set-pipeline ${NON_INTERACTIVE} --pipeline ${PIPELINE}" ;; +esac + +set +x +$fly_cmd --target ${TARGET} ${fly_opts} --config .deploy.yml +[[ -n ${VALIDATE_PIPELINE} ]] && exit 0 +$fly_cmd --target ${TARGET} unpause-pipeline --pipeline ${PIPELINE} diff --git a/ci/scripts/shipit b/ci/scripts/shipit new file mode 100755 index 0000000..48f3f5d --- /dev/null +++ b/ci/scripts/shipit @@ -0,0 +1,115 @@ +#!/bin/bash + +# +# ci/scripts/shipit +# +# Script for finalizing and packaging Helm chart +# and managing release notes +# +# author: Dr Nic Williams +# created: 2018-11-09 + +set -eu + +header() { + echo + echo "###############################################" + echo + echo $* + echo +} + +: ${CHART_NAME:?required} +: ${CHART_ROOT:?required} +: ${REPO_ROOT:?required} +: ${VERSION_FROM:?required} +: ${RELEASE_ROOT:?required} +: ${REPO_OUT:?required} +: ${BRANCH:?required} +: ${GITHUB_OWNER:?required} +: ${GIT_EMAIL:?required} +: ${GIT_NAME:?required} +: ${NOTIFICATION_OUT:?required} +: ${AWS_ACCESS_KEY_ID:?required} +: ${AWS_SECRET_ACCESS_KEY:?required} +: ${HELM_S3_BUCKET_URI:?required} + +HELM_VERSION=2.11.0 + +if [[ ! -f ${VERSION_FROM} ]]; then + echo >&2 "Version file (${VERSION_FROM}) not found. Did you misconfigure Concourse?" + exit 2 +fi +VERSION=$(cat ${VERSION_FROM}) +if [[ -z ${VERSION} ]]; then + echo >&2 "Version file (${VERSION_FROM}) was empty. Did you misconfigure Concourse?" + exit 2 +fi + +if [[ ! -f ${REPO_ROOT}/ci/release_notes.md ]]; then + echo >&2 "ci/release_notes.md not found. Did you forget to write them?" + exit 1 +fi + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +############################################################### + +git clone ${REPO_ROOT} ${REPO_OUT} + +pushd ${REPO_OUT} +header "Bump version in Chart.yaml" +tmpfile=$(mktemp /tmp/chart-yaml.XXXX) +sed -e "s/^version:.*$/version: ${VERSION}/g" ${CHART_ROOT}/Chart.yaml > $tmpfile +cp $tmpfile ${CHART_ROOT}/Chart.yaml + +header "Bump docker image version in Values.yaml" +tmpfile=$(mktemp /tmp/chart-yaml.XXXX) +sed -e "s/^ tag:.*$/ tag: ${VERSION}/g" ${CHART_ROOT}/values.yaml > $tmpfile +cp $tmpfile ${CHART_ROOT}/values.yaml +popd + +header "Install helm" +curl -LO https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz +tar xfz helm-*.tar.gz +mv linux-amd64/helm /usr/local/bin +helm version --client +helm init --client-only + +header "Install helm-s3 plugin" +helm plugin install https://github.com/hypnoglow/helm-s3.git + +header "Build helm chart" +mkdir -p ${RELEASE_ROOT}/artifacts +helm package ${REPO_OUT}/${CHART_ROOT} -d ${RELEASE_ROOT}/artifacts + +helm repo add our-repo ${HELM_S3_BUCKET_URI} + +header "Uploading helm chart to ${HELM_S3_BUCKET_URI}" +set -e +helm s3 push ${RELEASE_ROOT}/artifacts/${CHART_NAME}*.tgz our-repo --force +set +e +popd + +header "Update git repo with final release..." +if [[ -z $(git config --global user.email) ]]; then + git config --global user.email "${GIT_EMAIL}" +fi +if [[ -z $(git config --global user.name) ]]; then + git config --global user.name "${GIT_NAME}" +fi + +echo "v${VERSION}" > ${RELEASE_ROOT}/tag +echo "v${VERSION}" > ${RELEASE_ROOT}/name +mv ${REPO_OUT}/ci/release_notes.md ${RELEASE_ROOT}/notes.md + +pushd $REPO_OUT +git merge --no-edit ${BRANCH} +git add -A +git status +git commit -m "release v${VERSION}" +popd + +cat > ${NOTIFICATION_OUT:-notifications}/message <. +EOS \ No newline at end of file diff --git a/ci/settings.yml b/ci/settings.yml new file mode 100644 index 0000000..7fa4db2 --- /dev/null +++ b/ci/settings.yml @@ -0,0 +1,37 @@ +--- +meta: + name: show-me-secrets + target: ohio-sw + url: https://ci2.starkandwayne.com + team: starkandwayne + + initial_version: 0.1.0 + + git: + email: ((git-commit-email)) + name: ((git-commit-name)) + + aws: + bucket: (( grab meta.pipeline )) + region_name: us-east-1 + access_key: ((aws-access-key)) + secret_key: ((aws-secret-key)) + charts_uri: s3://helm.starkandwayne.com/charts + + github: + owner: starkandwayne + repo: show-me-secrets + branch: master + private_key: ((github-private-key)) + access_token: ((github-access-token)) + + dockerhub: + username: ((docker-hub-username)) + password: ((docker-hub-password)) + repository: ((docker-hub-username))/show-me-secrets + + slack: + webhook: ((slack-webhook)) + username: ((slack-username)) + icon: ((slack-icon-url)) + channel: "#show-me-secrets"