Initial Commit

This commit is contained in:
James Hunt 2016-06-21 15:52:55 -04:00
commit f1b7cb6780
14 changed files with 1392 additions and 0 deletions

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/genesis-index

21
LICENSE Normal file
View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2016 Stark & Wayne
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software..
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
IN THE SOFTWARE.

192
README.md Normal file
View File

@ -0,0 +1,192 @@
Genesis Index
=============
`genesis-index` is a small CF-ready application that tracks
stemcells and releases from [bosh.io](https://bosh.io) and other
places. The [genesis][genesis] utility uses the index to look up
versions, URLs and SHA1 checksums of said releases and stemcells.
Using the CLI
=============
`indexer` is a small Bash script that provides a basic
command-line interface for dealing with the Genesis Index.
It obeys the following environment variables:
- `GENESIS_INDEX` - The base URL of the Genesis Index. If not
set, defaults to `https://genesis.starkandwayne.com`
- `GENESIS_CREDS` - The username and password for accessing the
protected parts of the Index API, separated by a colon.
- `INDEXER_DEBUG` - Set to a non-empty value to enable debugging
Here are the commands:
```
indexer version (release|stemcell) NAME [VERSION]
indexer show (release|stemcell) NAME
indexer check (release|stemcell) NAME VERSION
indexer create (release|stemcell) NAME URL
indexer remove (release|stemcell) NAME [VERSION]
indexer releases
indexer stemcells
indexer help
```
So, for example, to get the latest version of the SHIELD BOSH
release:
```
$ indexer version release shield
```
Or, to get the SHA1 sum of v19 of the Consul BOSH release:
```
$ indexer version release consul 19
```
API Overview
============
The Genesis Index API strives to be simple and clean
## Get a List of Tracked Releases
```
GET /v1/release
```
## Get All Release Versions
```
GET /v1/release/:name
```
## Get The Latest Release Version
```
GET /v1/release/:name/latest
```
## Get a Specific Release Version
```
GET /v1/release/:name/v/:version
```
## Start Tracking a New Release
(this endpoint requires authentication)
```
POST /v1/release
{
"name": "release name",
"url": "https://wherever/to/get/it?v={{version}}"
}
```
## Check a Specific Version
(this endpoint requires authentication)
```
PUT /v1/release/:name/v/:version
```
## Stop Tracking a Release
(this endpoint requires authentication)
```
DELETE /v1/release/:name
```
## Drop a Release Version
(this endpoint requires authentication)
```
DELETE /v1/release/:name/v/:version
```
## Get a List of Tracked Stemcells
```
GET /v1/stemcell
```
## Get All Stemcell Versions
```
GET /v1/stemcell/:name
```
## Get The Latest Stemcell Version
```
GET /v1/stemcell/:name/latest
```
## Get a Specific Stemcell Version
```
GET /v1/stemcell/:name/v/:version
```
## Start Tracking a New Stemcell
(this endpoint requires authentication)
```
POST /v1/stemcell
{
"name": "stemcell name",
"url": "https://wherever/to/get/it?v={{version}}"
}
```
## Check a Specific Version
(this endpoint requires authentication)
```
PUT /v1/stemcell/:name/v/:version
```
## Stop Tracking a Stemcell
(this endpoint requires authentication)
```
DELETE /v1/stemcell/:name
```
## Drop a Stemcell Version
(this endpoint requires authentication)
```
DELETE /v1/stemcell/:name/v/:version
```
Installation And Operation
==========================
To deploy to Pivotal Web Services:
```
cf push
```
You need to bind a PostgreSQL database to your running app. The
application will automatically detect the service if it is tagged
`postgres`.
The following environment variables should also be set:
- `AUTH_USERNAME` - The username for authenticated endpoints
- `AUTH_PASSWORD` - The password for authenticated endpoints

96
api_release.go Normal file
View File

@ -0,0 +1,96 @@
package main
import (
"encoding/json"
"fmt"
"net/http"
"github.com/jhunt/go-db"
"github.com/starkandwayne/goutils/log"
)
type ReleaseAPI struct {
db *db.DB
}
func (api ReleaseAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
log.Debugf("RECV: %s %s", r.Method, r.URL.Path)
switch {
case match(r, `GET /v1/release`):
releases, err := FindAllReleases(api.db)
respond(w, err, 200, releases)
return
case match(r, `POST /v1/release`):
if !authed(w, r) {
return
}
var payload struct {
Name string `json:"name"`
URL string `json:"url"`
}
json.NewDecoder(r.Body).Decode(&payload)
log.Debugf("creating release '%s' at '%s'", payload.Name, payload.URL)
err := CreateRelease(api.db, payload.Name, payload.URL)
respond(w, err, 200, "success")
return
case match(r, `GET /v1/release/[^/]+`):
name := extract(r, `/v1/release/([^/]+)$`)
log.Debugf("retrieving all versions of release '%s'", name)
releases, err := FindAllReleaseVersions(api.db, name)
respond(w, err, 200, releases)
return
case match(r, `DELETE /v1/release/[^/]+`):
if !authed(w, r) {
return
}
name := extract(r, `/v1/release/([^/]+)`)
log.Debugf("will stop tracking release '%s'", name)
err := DeleteRelease(api.db, name)
respond(w, err, 200, "deleted")
return
case match(r, `GET /v1/release/[^/]+/v/[^/]+`):
name := extract(r, `/v1/release/([^/]+)/v/[^/]+`)
vers := extract(r, `/v1/release/[^/]+/v/([^/]+)`)
log.Debugf("retrieving version '%s' of release '%s'", vers, name)
release, err := FindReleaseVersion(api.db, name, vers)
respond(w, err, 200, release)
return
case match(r, `GET /v1/release/[^/]+/latest`):
name := extract(r, `/v1/release/([^/]+)/latest$`)
log.Debugf("retrieving latest version of release '%s'", name)
release, err := FindReleaseVersion(api.db, name, "")
respond(w, err, 200, release)
return
case match(r, `PUT /v1/release/[^/]+/v/[^/]+`):
if !authed(w, r) {
return
}
name := extract(r, `/v1/release/([^/]+)/v/[^/]+`)
vers := extract(r, `/v1/release/[^/]+/v/([^/]+)`)
log.Debugf("checking for version '%s' of release '%s'", vers, name)
go CheckReleaseVersion(api.db, name, vers)
respond(w, nil, 200, "task started in background")
return
case match(r, `DELETE /v1/release/[^/]+/v/[^/]+`):
if !authed(w, r) {
return
}
name := extract(r, `/v1/release/([^/]+)/v/[^/]+`)
vers := extract(r, `/v1/release/[^/]+/v/([^/]+)`)
log.Debugf("dropping version '%s' of release '%s'", vers, name)
err := DeleteReleaseVersion(api.db, name, vers)
respond(w, err, 200, fmt.Sprintf("v%s deleted", vers))
return
}
w.WriteHeader(404)
}

96
api_stemcell.go Normal file
View File

@ -0,0 +1,96 @@
package main
import (
"encoding/json"
"fmt"
"net/http"
"github.com/jhunt/go-db"
"github.com/starkandwayne/goutils/log"
)
type StemcellAPI struct {
db *db.DB
}
func (api StemcellAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
log.Debugf("RECV: %s %s", r.Method, r.URL.Path)
switch {
case match(r, `GET /v1/stemcell`):
stemcells, err := FindAllStemcells(api.db)
respond(w, err, 200, stemcells)
return
case match(r, `POST /v1/stemcell`):
if !authed(w, r) {
return
}
var payload struct {
Name string `json:"name"`
URL string `json:"url"`
}
json.NewDecoder(r.Body).Decode(&payload)
log.Debugf("creating stemcell '%s' at '%s'", payload.Name, payload.URL)
err := CreateStemcell(api.db, payload.Name, payload.URL)
respond(w, err, 200, "success")
return
case match(r, `GET /v1/stemcell/[^/]+`):
name := extract(r, `/v1/stemcell/([^/]+)$`)
log.Debugf("retrieving all versions of stemcell '%s'", name)
stemcells, err := FindAllStemcellVersions(api.db, name)
respond(w, err, 200, stemcells)
return
case match(r, `DELETE /v1/stemcell/[^/]+`):
if !authed(w, r) {
return
}
name := extract(r, `/v1/stemcell/([^/]+)`)
log.Debugf("will stop tracking stemcell '%s'", name)
err := DeleteStemcell(api.db, name)
respond(w, err, 200, "deleted")
return
case match(r, `GET /v1/stemcell/[^/]+/v/[^/]+`):
name := extract(r, `/v1/stemcell/([^/]+)/v/[^/]+`)
vers := extract(r, `/v1/stemcell/[^/]+/v/([^/]+)`)
log.Debugf("retrieving version '%s' of stemcell '%s'", vers, name)
stemcell, err := FindStemcellVersion(api.db, name, vers)
respond(w, err, 200, stemcell)
return
case match(r, `GET /v1/stemcell/[^/]+/latest`):
name := extract(r, `/v1/stemcell/([^/]+)/latest$`)
log.Debugf("retrieving latest version of stemcell '%s'", name)
stemcell, err := FindStemcellVersion(api.db, name, "")
respond(w, err, 200, stemcell)
return
case match(r, `PUT /v1/stemcell/[^/]+/v/[^/]+`):
if !authed(w, r) {
return
}
name := extract(r, `/v1/stemcell/([^/]+)/v/[^/]+`)
vers := extract(r, `/v1/stemcell/[^/]+/v/([^/]+)`)
log.Debugf("checking for version '%s' of stemcell '%s'", vers, name)
go CheckStemcellVersion(api.db, name, vers)
respond(w, nil, 200, "task started in background")
return
case match(r, `DELETE /v1/stemcell/[^/]+/v/[^/]+`):
if !authed(w, r) {
return
}
name := extract(r, `/v1/stemcell/([^/]+)/v/[^/]+`)
vers := extract(r, `/v1/stemcell/[^/]+/v/([^/]+)`)
log.Debugf("dropping version '%s' of stemcell '%s'", vers, name)
err := DeleteStemcellVersion(api.db, name, vers)
respond(w, err, 200, fmt.Sprintf("v%s deleted", vers))
return
}
w.WriteHeader(404)
}

241
indexer Executable file
View File

@ -0,0 +1,241 @@
#!/bin/bash
# indexer - A CLI for interacting with the Genesis Index
#
# author: James Hunt <james@niftylogic.com>
# created: 2016-06-22
#
[ -n "$INDEXER_DEBUG" ] && set -x
GENESIS_INDEX=${GENESIS_INDEX:-https://genesis.starkandwayne.com/}
GENESIS_INDEX=${GENESIS_INDEX%%/}
need_auth() {
if [[ -z ${GENESIS_CREDS} ]]; then
echo >&2 "You must be authorized to perform this action."
echo >&2 "Try setting the GENESIS_CREDS environment variable to the username:password"
exit 1
fi
}
cmd_help() {
cat <<EOF
USAGE: $0 version (release|stemcell) NAME [VERSION]
$0 show (release|stemcell) NAME
$0 check (release|stemcell) NAME VERSION
$0 create (release|stemcell) NAME URL
$0 remove (release|stemcell) NAME [VERSION]
$0 releases
$0 stemcells
$0 help
EOF
}
cmd_create() {
local USAGE="create (release|stemcell) NAME URL"
local type=$1 ; shift
local name=$1 ; shift
local url=$1 ; shift
if [[ -z $type || -z $name || -z $url || -n $1 ]]; then
echo >&2 "USAGE: $0 $USAGE"
exit 1
fi
case $type in
(release|stemcell)
need_auth
curl --fail -Lsk -XPOST -u "${GENESIS_CREDS}" ${GENESIS_INDEX}/v1/${type} \
-d '{"name":"'$name'","url":"'$url'"}'
exit $?
;;
(*)
echo >&2 "unrecognized type '$type'"
echo >&2 "USAGE: $0 $USAGE"
exit 1
;;
esac
exit 0
}
cmd_remove() {
local USAGE="remove (release|stemcell) NAME [VERSION]"
local type=$1 ; shift
local name=$1 ; shift
local vers=$1 ; shift
if [[ -z $type || -z $name || -n $1 ]]; then
echo >&2 "USAGE: $0 $USAGE"
exit 1
fi
if [[ -z $vers ]]; then
echo "This will delete all versions of the '${name}' ${type}!"
echo -n "Are you sure? [yes/no] "
read CONFIRM
if [[ ${CONFIRM} != "yes" ]]; then
echo "Aborting..."
exit 2
fi
fi
case $type in
(release|stemcell)
need_auth
if [[ -z $vers ]]; then
curl --fail -Lsk -XDELETE -u "${GENESIS_CREDS}" ${GENESIS_INDEX}/v1/${type}/${name}
else
curl --fail -Lsk -XDELETE -u "${GENESIS_CREDS}" ${GENESIS_INDEX}/v1/${type}/${name}/v/${vers}
fi
exit $?
;;
(*)
echo >&2 "unrecognized type '$type'"
echo >&2 "USAGE: $0 $USAGE"
exit 1
;;
esac
exit 0
}
cmd_check() {
local USAGE="check (release|stemcell) NAME VERSION"
local type=$1 ; shift
local name=$1 ; shift
local vers=$1 ; shift
if [[ -z $type || -z $name || -z $vers || -n $1 ]]; then
echo >&2 "USAGE: $0 $USAGE"
exit 1
fi
case $type in
(release|stemcell)
need_auth
curl --fail -Lsk -XPUT -u "${GENESIS_CREDS}" ${GENESIS_INDEX}/v1/${type}/${name}/v/${vers}
exit $?
;;
(*)
echo >&2 "unrecognized type '$type'"
echo >&2 "USAGE: $0 $USAGE"
exit 1
;;
esac
exit 0
}
cmd_version() {
local USAGE="version (release|stemcell) NAME [VERSION]"
local type=$1 ; shift
local name=$1 ; shift
local vers=$1 ; shift
if [[ -z $type || -z $name || -n $1 ]]; then
echo >&2 "USAGE: $0 $USAGE"
exit 1
fi
case $type in
(release|stemcell)
if [[ -z ${vers} ]]; then
curl --fail -Lsk -XGET ${GENESIS_INDEX}/v1/${type}/${name}/latest
else
curl --fail -Lsk -XGET ${GENESIS_INDEX}/v1/${type}/${name}/v/${vers}
fi
exit $?
;;
(*)
echo >&2 "unrecognized type '$type'"
echo >&2 "USAGE: $0 $USAGE"
exit 1
;;
esac
exit 0
}
cmd_show() {
local USAGE="show (release|stemcell) NAME"
local type=$1 ; shift
local name=$1 ; shift
if [[ -z $type || -z $name || -n $1 ]]; then
echo >&2 "USAGE: $0 $USAGE"
exit 1
fi
case $type in
(release|stemcell)
curl --fail -Lsk -XGET ${GENESIS_INDEX}/v1/${type}/${name}
exit $?
;;
(*)
echo >&2 "unrecognized type '$type'"
echo >&2 "USAGE: $0 $USAGE"
exit 1
;;
esac
exit 0
}
cmd_list() {
local type=$1 ; shift
local USAGE="$type"
type=${type%%s}
if [[ -z $type || -n $1 ]]; then
echo >&2 "USAGE: $0 $USAGE"
exit 1
fi
case $type in
(release|stemcell)
curl --fail -Lsk -XGET ${GENESIS_INDEX}/v1/${type}
exit $?
;;
(*)
echo >&2 "unrecognized type '$type'"
echo >&2 "USAGE: $0 $USAGE"
exit 1
;;
esac
exit 0
}
main() {
local command=$1 ; shift
if [[ -z $command ]]; then
command="help"
fi
case $command in
(help|-h|--help)
cmd_help $*
exit 0
;;
(create|new|track)
cmd_create $*
;;
(remove|rm|delete)
cmd_remove $*
;;
(check)
cmd_check $*
;;
(version)
cmd_version $*
;;
(show)
cmd_show $*
;;
(stemcells|releases)
cmd_list $command $*
;;
(*)
echo "Invalid command '$command'"
cmd_help
exit 1
;;
esac
}
main "$@"

60
main.go Normal file
View File

@ -0,0 +1,60 @@
package main
import (
"fmt"
"net/http"
"os"
"github.com/jhunt/go-db"
_ "github.com/mattn/go-sqlite3"
"github.com/starkandwayne/goutils/log"
)
func main() {
level := os.Getenv("LOG_LEVEL")
if level == "" {
level = "warning"
}
log.SetupLogging(log.LogConfig{
Type: "console",
Level: level,
})
log.Infof("genesis-index starting up")
var d *db.DB
if dsn, err := ParseVcap(os.Getenv("VCAP_SERVICES"), "postgres", "uri"); err == nil {
d, err = Database("postgres", dsn)
if err != nil {
log.Infof("Unable to connect to database: %s", err)
return
}
} else if file := os.Getenv("SQLITE_DB"); file != "" {
d, err = Database("sqlite3", file)
if err != nil {
log.Infof("Unable to connect to database: %s", err)
return
}
} else {
log.Errorf("Unable to determine DSN for backing database")
log.Errorf("No service tagged 'postgres' is bound (per the VCAP_SERVICES environment variable)")
log.Errorf("and SQLITE_DB environment variable is not set.")
return
}
/* clean house */
d.Exec(`DELETE FROM release_versions WHERE valid = 0`)
/* set up the server */
mux := http.NewServeMux()
mux.Handle("/v1/release", ReleaseAPI{db: d})
mux.Handle("/v1/release/", ReleaseAPI{db: d})
mux.Handle("/v1/stemcell", StemcellAPI{db: d})
mux.Handle("/v1/stemcell/", StemcellAPI{db: d})
port := os.Getenv("PORT")
if port == "" {
port = "3000"
}
log.Infof("listening on *:%s", port)
http.ListenAndServe(fmt.Sprintf(":%s", port), mux)
}

7
manifest.yml Normal file
View File

@ -0,0 +1,7 @@
---
applications:
- name: genesis.starkandwayne.com
host: genesis
domain: starkandwayne.com
memory: 128M
command: genesis-index

197
release.go Normal file
View File

@ -0,0 +1,197 @@
package main
import (
"fmt"
"github.com/jhunt/go-db"
"github.com/starkandwayne/goutils/log"
)
type Release struct {
Name string `json:"name"`
Version string `json:"version,omitempty"`
SHA1 string `json:"sha1,omitempty"`
URL string `json:"url,omitempty"`
}
func CreateRelease(d *db.DB, name, url string) error {
return d.Exec(`INSERT INTO releases (name, url) VALUES ($1, $2)`, name, url)
}
func FindAllReleases(d *db.DB) ([]string, error) {
l := make([]string, 0)
r, err := d.Query(`SELECT name FROM releases`)
if err != nil {
return l, err
}
for r.Next() {
var o string
if err = r.Scan(&o); err != nil {
return l, err
}
l = append(l, o)
}
return l, nil
}
func FindRelease(d *db.DB, name string) (Release, error) {
var o Release
r, err := d.Query(`SELECT name, url FROM releases WHERE name = $1`, name)
if err != nil {
return o, err
}
if !r.Next() {
return o, fmt.Errorf("release '%s' not found", name)
}
if err = r.Scan(&o.Name, &o.URL); err != nil {
return o, err
}
if r.Next() {
return o, fmt.Errorf("duplicate releases found for '%s'", name)
}
return o, nil
}
func FindAllReleaseVersions(d *db.DB, name string) ([]Release, error) {
l := make([]Release, 0)
r, err := d.Query(`
SELECT
name,
version,
sha1,
url
FROM release_versions
WHERE name = $1
AND valid = 1
ORDER BY
version DESC`, name)
if err != nil {
return l, err
}
for r.Next() {
var o Release
if err = r.Scan(&o.Name, &o.Version, &o.SHA1, &o.URL); err != nil {
return l, err
}
l = append(l, o)
}
if len(l) == 0 {
n, err := d.Count("SELECT * FROM releases WHERE name = $1", name)
if err == nil && n != 0 {
return l, nil
}
return l, fmt.Errorf("release '%s' not found", name)
}
return l, nil
}
func FindReleaseVersion(d *db.DB, name, version string) (Release, error) {
var o Release
where := ""
args := make([]interface{}, 1)
args[0] = name
if version != "" {
where = "AND version = $2"
args = append(args, version)
}
r, err := d.Query(fmt.Sprintf(`
SELECT name, version, sha1, url
FROM release_versions
WHERE name = $1 AND valid = 1 %s`, where), args...)
if err != nil {
return o, err
}
if !r.Next() {
if version != "" {
return o, fmt.Errorf("version '%s' of release '%s' not found", version, name)
}
n, err := d.Count("SELECT * FROM releases WHERE name = $1", name)
if err == nil && n != 0 {
return o, fmt.Errorf("no known versions for release '%s'", name)
}
return o, fmt.Errorf("release '%s' not found", name)
}
if err = r.Scan(&o.Name, &o.Version, &o.SHA1, &o.URL); err != nil {
return o, err
}
if r.Next() {
return o, fmt.Errorf("duplicate releases found for '%s'", name)
}
return o, nil
}
func DeleteRelease(d *db.DB, name string) error {
err := d.Exec(`DELETE FROM release_versions WHERE name = $1`, name)
if err != nil {
return err
}
return d.Exec(`DELETE FROM releases WHERE name = $1`, name)
}
func DeleteReleaseVersion(d *db.DB, name, version string) error {
return d.Exec(`DELETE FROM release_versions WHERE name = $1 AND version = $2`, name, version)
}
func CheckReleaseVersion(d *db.DB, name, version string) {
release, err := FindRelease(d, name)
if err != nil {
log.Debugf("unable to find release '%s': %s", name, err)
return
}
/* generate the URL from the template */
url := urlify(release.URL, version)
log.Debugf("checking version '%s' of '%s' at '%s'", version, name, url)
recheck := true
n, _ := d.Count(`SELECT * FROM release_versions WHERE name = $1 AND version = $2`, name, version)
if n == 0 {
recheck = false
err = d.Exec(`INSERT INTO release_versions (name, version, valid) VALUES ($1, $2, 0)`,
name, version)
}
/* download and SHA1 the file */
sha1, err := sha1sum(url)
if err != nil {
log.Debugf("download/sha1sum failed: %s...", err)
if !recheck {
d.Exec(`DELETE FROM release_versions WHERE name = $1 AND version = $2`,
name, version)
}
return
}
err = d.Exec(`
UPDATE release_versions
SET valid = 1,
url = $3,
sha1 = $4
WHERE name = $1
AND version = $2`, name, version, url, sha1)
if err != nil {
log.Debugf("unable to check version '%s' of '%s': %s", version, name, err)
return
}
}

79
schema.go Normal file
View File

@ -0,0 +1,79 @@
package main
import (
"github.com/jhunt/go-db"
)
func Database(driver, dsn string) (*db.DB, error) {
d := &db.DB{
Driver: driver,
DSN: dsn,
}
err := d.Connect()
if err != nil {
return nil, err
}
s := db.NewSchema()
s.Version(1, func(d *db.DB) error {
err := d.Exec(`
CREATE TABLE releases (
name VARCHAR(40) NOT NULL PRIMARY KEY,
url TEXT NOT NULL
)
`)
if err != nil {
return err
}
err = d.Exec(`
CREATE TABLE stemcells (
name VARCHAR(40) NOT NULL PRIMARY KEY,
url TEXT NOT NULL
)
`)
if err != nil {
return err
}
err = d.Exec(`
CREATE TABLE release_versions (
name VARCHAR(40) NOT NULL,
version VARCHAR(20) NOT NULL,
sha1 VARCHAR(40) NOT NULL DEFAULT '',
url TEXT NOT NULL DEFAULT '',
valid INTEGER(1) NOT NULL DEFAULT 0,
UNIQUE (name, version)
)
`)
if err != nil {
return err
}
err = d.Exec(`
CREATE TABLE stemcell_versions (
name VARCHAR(40) NOT NULL,
version VARCHAR(20) NOT NULL,
sha1 VARCHAR(40) NOT NULL DEFAULT '',
url TEXT NOT NULL DEFAULT '',
valid INTEGER(1) NOT NULL DEFAULT 0,
UNIQUE (name, version)
)
`)
if err != nil {
return err
}
return nil
})
err = s.Migrate(d, db.Latest)
if err != nil {
return nil, err
}
return d, nil
}

53
seed Executable file
View File

@ -0,0 +1,53 @@
#!/bin/bash
GET() {
curl -Lk -XGET ${ENDPOINT}${1}
}
PUT() {
curl -Lk -XPUT ${ENDPOINT}${1} --data "${2}" -H 'Content-type: application/json'
}
POST() {
curl -Lk -XPOST ${ENDPOINT}${1} --data "${2}" -H 'Content-type: application/json'
}
DELETE() {
curl -Lk -XDELETE ${ENDPOINT}${1}
}
ENDPOINT=$1
if [[ -z $ENDPOINT ]]; then
echo >&2 "USAGE: $0 <url>"
exit 1
fi
for stemcell in \
bosh-aws-xen-hvm-ubuntu-trusty-go_agent \
bosh-aws-xen-ubuntu-trusty-go_agent \
bosh-azure-hyperv-ubuntu-trusty-go_agent \
bosh-openstack-kvm-ubuntu-trusty-go_agent \
bosh-openstack-kvm-ubuntu-trusty-go_agent-raw \
bosh-vcloud-esxi-ubuntu-trusty-go_agent \
bosh-vsphere-esxi-ubuntu-trusty-go_agent \
bosh-warden-boshlite-ubuntu-trusty-go_agent
do
POST '/v1/stemcell' '{"name":"'${stemcell}'","url":"https://bosh.io/d/stemcells/'${stemcell}'?v={{version}}"}'
done
for cpi in aws azure openstack rackhd softlayer vcloud vsphere
do
POST '/v1/release' '{"name":"bosh-'${cpi}'-cpi","url":"https://bosh.io/d/github.com/cloudfoundry-incubator/bosh-'${cpi}'-release?v={{version}}"}'
done
for release in \
bind9 \
cf-haproxy cf-subway consul \
docker-registry \
jumpbox \
route-registrar \
shell \
toolbelt \
vault
do
POST '/v1/release' '{"name":"'${release}'","url":"https://bosh.io/d/github.com/cloudfoundry-community/'${release}'-boshrelease?v={{version}}"}'
done
echo "DONE"

197
stemcell.go Normal file
View File

@ -0,0 +1,197 @@
package main
import (
"fmt"
"github.com/jhunt/go-db"
"github.com/starkandwayne/goutils/log"
)
type Stemcell struct {
Name string `json:"name"`
Version string `json:"version,omitempty"`
SHA1 string `json:"sha1,omitempty"`
URL string `json:"url,omitempty"`
}
func CreateStemcell(d *db.DB, name, url string) error {
return d.Exec(`INSERT INTO stemcells (name, url) VALUES ($1, $2)`, name, url)
}
func FindAllStemcells(d *db.DB) ([]string, error) {
l := make([]string, 0)
r, err := d.Query(`SELECT name FROM stemcells`)
if err != nil {
return l, err
}
for r.Next() {
var o string
if err = r.Scan(&o); err != nil {
return l, err
}
l = append(l, o)
}
return l, nil
}
func FindStemcell(d *db.DB, name string) (Stemcell, error) {
var o Stemcell
r, err := d.Query(`SELECT name, url FROM stemcells WHERE name = $1`, name)
if err != nil {
return o, err
}
if !r.Next() {
return o, fmt.Errorf("stemcell '%s' not found", name)
}
if err = r.Scan(&o.Name, &o.URL); err != nil {
return o, err
}
if r.Next() {
return o, fmt.Errorf("duplicate stemcells found for '%s'", name)
}
return o, nil
}
func FindAllStemcellVersions(d *db.DB, name string) ([]Stemcell, error) {
l := make([]Stemcell, 0)
r, err := d.Query(`
SELECT
name,
version,
sha1,
url
FROM stemcell_versions
WHERE name = $1
AND valid = 1
ORDER BY
version DESC`, name)
if err != nil {
return l, err
}
for r.Next() {
var o Stemcell
if err = r.Scan(&o.Name, &o.Version, &o.SHA1, &o.URL); err != nil {
return l, err
}
l = append(l, o)
}
if len(l) == 0 {
n, err := d.Count("SELECT * FROM stemcells WHERE name = $1", name)
if err == nil && n != 0 {
return l, nil
}
return l, fmt.Errorf("stemcell '%s' not found", name)
}
return l, nil
}
func FindStemcellVersion(d *db.DB, name, version string) (Stemcell, error) {
var o Stemcell
where := ""
args := make([]interface{}, 1)
args[0] = name
if version != "" {
where = "AND version = $2"
args = append(args, version)
}
r, err := d.Query(fmt.Sprintf(`
SELECT name, version, sha1, url
FROM stemcell_versions
WHERE name = $1 AND valid = 1 %s`, where), args...)
if err != nil {
return o, err
}
if !r.Next() {
if version != "" {
return o, fmt.Errorf("version '%s' of stemcell '%s' not found", version, name)
}
n, err := d.Count("SELECT * FROM stemcells WHERE name = $1", name)
if err == nil && n != 0 {
return o, fmt.Errorf("no known versions for stemcell '%s'", name)
}
return o, fmt.Errorf("stemcell '%s' not found", name)
}
if err = r.Scan(&o.Name, &o.Version, &o.SHA1, &o.URL); err != nil {
return o, err
}
if r.Next() {
return o, fmt.Errorf("duplicate stemcells found for '%s'", name)
}
return o, nil
}
func DeleteStemcell(d *db.DB, name string) error {
err := d.Exec(`DELETE FROM stemcell_versions WHERE name = $1`, name)
if err != nil {
return err
}
return d.Exec(`DELETE FROM stemcells WHERE name = $1`, name)
}
func DeleteStemcellVersion(d *db.DB, name, version string) error {
return d.Exec(`DELETE FROM stemcell_versions WHERE name = $1 AND version = $2`, name, version)
}
func CheckStemcellVersion(d *db.DB, name, version string) {
stemcell, err := FindStemcell(d, name)
if err != nil {
log.Debugf("unable to find stemcell '%s': %s", name, err)
return
}
/* generate the URL from the template */
url := urlify(stemcell.URL, version)
log.Debugf("checking version '%s' of '%s' at '%s'", version, name, url)
recheck := true
n, _ := d.Count(`SELECT * FROM stemcell_versions WHERE name = $1 AND version = $2`, name, version)
if n == 0 {
recheck = false
err = d.Exec(`INSERT INTO stemcell_versions (name, version, valid) VALUES ($1, $2, 0)`,
name, version)
}
/* download and SHA1 the file */
sha1, err := sha1sum(url)
if err != nil {
log.Debugf("download/sha1sum failed: %s...", err)
if !recheck {
d.Exec(`DELETE FROM stemcell_versions WHERE name = $1 AND version = $2`,
name, version)
}
return
}
err = d.Exec(`
UPDATE stemcell_versions
SET valid = 1,
url = $3,
sha1 = $4
WHERE name = $1
AND version = $2`, name, version, url, sha1)
if err != nil {
log.Debugf("unable to check version '%s' of '%s': %s", version, name, err)
return
}
}

106
util.go Normal file
View File

@ -0,0 +1,106 @@
package main
import (
"crypto/sha1"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"regexp"
"github.com/starkandwayne/goutils/log"
)
func urlify(template string, version string) string {
re := regexp.MustCompile("{{version}}")
return re.ReplaceAllLiteralString(template, version)
}
func sha1sum(url string) (string, error) {
r, err := http.Get(url)
if err != nil {
return "", err
}
h := sha1.New()
io.Copy(h, r.Body)
return fmt.Sprintf("%x", h.Sum(nil)), nil
}
func match(req *http.Request, pattern string) bool {
matched, _ := regexp.MatchString(
fmt.Sprintf("^%s$", pattern),
fmt.Sprintf("%s %s", req.Method, req.URL.Path))
return matched
}
func extract(r *http.Request, pattern string) string {
re := regexp.MustCompile(fmt.Sprintf("^%s$", pattern))
return re.FindStringSubmatch(r.URL.Path)[1]
}
func bail(w http.ResponseWriter, e error) {
w.WriteHeader(500)
fmt.Printf("responding with an error: [%s]\n", e)
x := struct {
E string `json:"e"`
}{E: e.Error()}
b, err := json.Marshal(x)
if err == nil {
fmt.Fprintf(w, "%s\n", string(b))
} else {
fmt.Fprintf(w, `{"e":"failed to prepare JSON response"}%s`, "\n")
}
}
func respond(w http.ResponseWriter, e error, status int, payload interface{}) {
w.Header().Set("Content-type", "application/json")
if e != nil {
bail(w, e)
return
}
if s, ok := payload.(string); ok {
fmt.Printf("SEND %d %s\n", status, s)
payload = struct {
M string `json:"m"`
}{M: s}
}
b, err := json.Marshal(payload)
if err == nil {
w.WriteHeader(status)
fmt.Fprintf(w, "%s\n", string(b))
} else {
bail(w, err)
}
return
}
func authed(w http.ResponseWriter, r *http.Request) bool {
auth_user := os.Getenv("AUTH_USERNAME")
auth_pass := os.Getenv("AUTH_PASSWORD")
if auth_user == "" {
log.Debugf("no AUTH_USERNAME set in environment; skipping auth checks")
return true
}
try_user, try_pass, provided := r.BasicAuth()
if !provided {
log.Debugf("no Authorization header provided. returning a 401")
w.WriteHeader(401)
return false
}
if try_user == auth_user && try_pass == auth_pass {
return true
}
log.Debugf("authorization failed for user '%s'", try_user)
w.WriteHeader(403)
return false
}

46
vcap.go Normal file
View File

@ -0,0 +1,46 @@
package main
import (
"encoding/json"
"fmt"
)
func ParseVcap(src, tag, subkey string) (string, error) {
var services map[string][]struct {
Credentials map[string]interface{} `json:"credentials"`
Label string `json:"label"`
Name string `json:"name"`
Plan string `json:"plan"`
Provider interface{} `json:"provider"`
SyslogDrainURL interface{} `json:"syslog_drain_url"`
Tags []string `json:"tags"`
}
err := json.Unmarshal([]byte(src), &services)
if err != nil {
return "", err
}
for _, l := range services {
for _, service := range l {
tagged := false
for _, actual := range service.Tags {
if tag == actual {
tagged = true
break
}
}
if !tagged {
continue
}
if v, ok := service.Credentials[subkey]; ok {
return fmt.Sprintf("%s", v), nil
}
}
}
return "", fmt.Errorf("no satisfactory service found")
}