Merge pull request #134 from allegro/develop

Release 1.0.0.

.goxc.json

@@ -2,7 +2,10 @@
 	"ArtifactsDest": "dist",
 	"TasksExclude": [
 		"go-vet",
-		"go-test"
+		"go-test",
+		"deb",
+		"deb-dev",
+		"deb-source"
 	],
 	"BuildConstraints": "linux,!arm darwin",
 	"PackageVersion": "0.4.2",
@@ -15,18 +18,11 @@
 			"repository": "deb",
 			"subject": "allegro"
 		},
-		"deb": {
-			"other-mapped-files": {
-				"/etc/init/marathon-consul.conf": "debian/marathon-consul.upstart",
-				"/etc/marathon-consul.d/config.json": "debian/config.json",
-				"/etc/systemd/system/marathon-consul.service": "debian/marathon-consul.service"
-			}
-		},
 		"publish-github": {
 			"body": "",
 			"owner": "allegro",
 			"repository": "marathon-consul"
 		}
 	},
 	"ConfigVersion": "0.9"
-}
+}

.travis.yml

@@ -2,21 +2,24 @@ language: go
 
 go:
   - 1.7
-  - tip
 before_install:
   - go get golang.org/x/tools/cmd/cover
   - go get github.com/axw/gocov/gocov
   - go get github.com/modocache/gover
   - go get github.com/mattn/goveralls
   - if ! go get github.com/golang/tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi
+  - go get github.com/alecthomas/gometalinter
 script:
   - make deps
+  - gometalinter --install
+  - gometalinter ./... --deadline 120s --vendor -D dupl -D gotype -D errcheck -D gas -D golint -E gofmt
   - go test -coverprofile=apps.coverprofile ./apps
   - go test -coverprofile=config.coverprofile ./config
   - go test -coverprofile=consul.coverprofile ./consul
   - go test -coverprofile=events.coverprofile ./events
   - go test -coverprofile=marathon.coverprofile ./marathon
   - go test -coverprofile=metrics.coverprofile ./metrics
+  - go test -coverprofile=service.coverprofile ./service
   - go test -coverprofile=sync.coverprofile ./sync
   - go test -coverprofile=utils.coverprofile ./utils
   - go test -coverprofile=web.coverprofile ./web

Makefile

@@ -3,6 +3,8 @@ TESTARGS?=-race
 CURRENT_DIR = $(shell pwd)
 SOURCEDIR = $(CURRENT_DIR)
 SOURCES := $(shell find $(SOURCEDIR) -name '*.go')
+VERSION=$(shell cat .goxc.json | python -c "import json,sys;obj=json.load(sys.stdin);print obj['PackageVersion'];")
+TEMPDIR := $(shell mktemp -d)
 
 all: deps build
 
@@ -24,10 +26,34 @@ test: deps $(SOURCES)
 	PATH=$(CURRENT_DIR)/bin:$(PATH) go test $(PACKAGES) $(TESTARGS)
 	go vet $(PACKAGES)
 
-release:
-	@rm -rf dist
+FPM-exists:
+	@fpm -v || \
+	(echo >&2 "FPM must be installed on the system. See https://github.com/jordansissel/fpm"; false)
+
+deb: FPM-exists build
+	mkdir -p dist/$(VERSION)/
+	cd dist/$(VERSION)/ && \
+	fpm -s dir \
+	    -t deb \
+        -n marathon-consul \
+        -v $(VERSION) \
+        --url="https://github.com/allegro/marathon-consul" \
+        --vendor=Allegro \
+        --maintainer="Allegro Group <[email protected]>" \
+        --description "Marathon-consul service (performs Marathon Tasks registration as Consul Services for service discovery) Marathon-consul takes information provided by the Marathon event bus and forwards it to Consul agents. It also re-syncs all the information from Marathon to Consul on startup and repeats it with given interval." \
+        --deb-priority optional \
+        --workdir $(TEMPDIR) \
+        --license "Apache License, version 2.0" \
+        ../../bin/marathon-consul=/usr/bin/marathon-consul \
+        ../../debian/marathon-consul.service=/etc/systemd/system/marathon-consul.service \
+        ../../debian/marathon-consul.upstart=/etc/init/marathon-consul.conf \
+        ../../debian/config.json=/etc/marathon-consul.d/config.json
+
+release: deb
 	@go get github.com/laher/goxc
 	goxc
+
+bump:
 	goxc bump
 	git add .goxc.json
 	git commit -m "Bumped version"

README.md

@@ -145,13 +145,17 @@ curl -X GET http://localhost:8500/v1/catalog/service/my-new-app
 "ServiceTags": [
   "marathon",
   "varnish",
-  "metrics"
+  "metrics",
+  "marathon-task:my-new-app.6a95bb03-6ad3-11e6-beaf-080027a7aca0"
 ],
 
 ```
+- Every service registration contains an additional tag `marathon-task` specifying the Marathon task id related to this registration.
 - If there are multiple ports in use for the same app, note that only the first one will be registered by marathon-consul in Consul.
 
-### Task heathchecks
+If you need to register your task under multiple ports, refer to *Advanced usage* section below.
+
+### Task healthchecks
 
 - At least one HTTP healthcheck should be defined for a task. The task is registered when Marathon marks it as alive.
 - The provided HTTP healthcheck will be transferred to Consul.
@@ -174,6 +178,7 @@ config-file                 |                 | Path to a JSON file to read conf
 consul-auth                 | `false`         | Use Consul with authentication
 consul-auth-password        |                 | The basic authentication password
 consul-auth-username        |                 | The basic authentication username
+consul-ignored-healthchecks |                 | A comma separated blacklist of Marathon health check types that will not be migrated to Consul, e.g. command,tcp
 consul-name-separator       | `.`             | Separator used to create default service name for Consul
 consul-get-services-retry   | `3`             | Number of retries on failure when performing requests to Consul. Each retry uses different cached agent
 consul-max-agent-failures   | `3`             | Max number of consecutive request failures for agent before removal from cache
@@ -213,7 +218,81 @@ Endpoint  | Description
 `/health` | healthcheck - returns `OK`
 `/events` | event sink - returns `OK` if all keys are set in an event, error message otherwise
 
-### Known limitations
+## Advanced usage
+
+### Register under multiple ports
+
+If you need to map your Marathon task into multiple service registrations in Consul, you can configure marathon-consul 
+via Marathon's `portDefinitions`:
+
+```
+  "id": "my-new-app",
+  "labels": {
+    "consul": "",
+    "common-tag": "tag"
+  },
+  "portDefinitions": [
+    {
+      "labels": {
+        "consul": "my-app-custom-name"
+      }
+    },
+    {
+      "labels": {
+        "consul": "my-app-other-name",
+        "specific-tag": "tag"
+      }
+    }
+  ]
+```
+
+This configuration would result in two service registrations:
+
+```
+curl -X GET http://localhost:8500/v1/catalog/service/my-app-custom-name
+...
+"ServiceName": "my-app-custom-name",
+"ServiceTags": [
+  "marathon",
+  "common-tag",
+  "marathon-task:my-new-app.6a95bb03-6ad3-11e6-beaf-080027a7aca0"
+],
+"ServicePort": 31292,
+...
+
+curl -X GET http://localhost:8500/v1/catalog/service/my-app-other-name
+...
+"ServiceName": "my-app-other-name",
+"ServiceTags": [
+  "marathon",
+  "common-tag",
+  "specific-tag",
+  "marathon-task:my-new-app.6a95bb03-6ad3-11e6-beaf-080027a7aca0"
+],
+"ServicePort": 31293,
+...
+``` 
+
+If any port definition contains the `consul` label, then advanced configuration mode is enabled. As a result, only the ports 
+containing this label are registered, under the name specified as the label's value – with empty value resolved to default name.
+Names don't have to be unique – you can have multiple registrations under the same name, but on different ports, 
+perhaps with different tags. Note that the `consul` label still needs to be present in the top-level application labels, even
+though its value won't have any effect.
+
+Tags configured in the top-level application labels will be added to all registrations. Tags configured in the port definition 
+labels will be added only to corresponding registrations.
+
+All registrations share the same `marathon-task` tag.
+
+## Migration to version 1.x.x
+
+Until 1.x.x marathon-consul would register services in Consul with registration id equal to related Marathon task id. Since 1.x.x registration ids are different and
+an additional tag, `marathon-task`, is added to each registration.
+
+If you update marathon-consul from version 0.x.x to 1.x.x, expect the synchronization phase during the first startup to 
+reregister all healthy services managed by marathon-consul to the new format. Unhealthy services will get deregistered in the process.
+
+## Known limitations
 
 The following section describes known limitations in `marathon-consul`.
 

apps/app.go

@@ -2,10 +2,13 @@ package apps
 
 import (
 	"encoding/json"
+	"strings"
+
+	log "github.com/Sirupsen/logrus"
 )
 
 // Only Marathon apps with this label will be registered in Consul
-const MARATHON_CONSUL_LABEL = "consul"
+const MarathonConsulLabel = "consul"
 
 type HealthCheck struct {
 	Path                   string `json:"path"`
@@ -21,37 +24,42 @@ type HealthCheck struct {
 	}
 }
 
-type AppWrapper struct {
+type PortDefinition struct {
+	Labels map[string]string `json:"labels"`
+}
+
+type appWrapper struct {
 	App App `json:"app"`
 }
 
-type AppsResponse struct {
+type Apps struct {
 	Apps []*App `json:"apps"`
 }
 
 type App struct {
-	Labels       map[string]string `json:"labels"`
-	HealthChecks []HealthCheck     `json:"healthChecks"`
-	ID           AppId             `json:"id"`
-	Tasks        []Task            `json:"tasks"`
+	Labels          map[string]string `json:"labels"`
+	HealthChecks    []HealthCheck     `json:"healthChecks"`
+	ID              AppID             `json:"id"`
+	Tasks           []Task            `json:"tasks"`
+	PortDefinitions []PortDefinition  `json:"portDefinitions"`
 }
 
 // Marathon Application Id (aka PathId)
 // Usually in the form of /rootGroup/subGroup/subSubGroup/name
 // allowed characters: lowercase letters, digits, hyphens, slash
-type AppId string
+type AppID string
 
-func (id AppId) String() string {
+func (id AppID) String() string {
 	return string(id)
 }
 
 func (app *App) IsConsulApp() bool {
-	_, ok := app.Labels[MARATHON_CONSUL_LABEL]
+	_, ok := app.Labels[MarathonConsulLabel]
 	return ok
 }
 
-func (app *App) ConsulName() string {
-	if value, ok := app.Labels[MARATHON_CONSUL_LABEL]; ok && !isSpecialConsulNameValue(value) {
+func (app *App) labelsToRawName(labels map[string]string) string {
+	if value, ok := labels[MarathonConsulLabel]; ok && !isSpecialConsulNameValue(value) {
 		return value
 	}
 	return app.ID.String()
@@ -62,15 +70,102 @@ func isSpecialConsulNameValue(name string) bool {
 }
 
 func ParseApps(jsonBlob []byte) ([]*App, error) {
-	apps := &AppsResponse{}
+	apps := &Apps{}
 	err := json.Unmarshal(jsonBlob, apps)
 
 	return apps.Apps, err
 }
 
 func ParseApp(jsonBlob []byte) (*App, error) {
-	wrapper := &AppWrapper{}
+	wrapper := &appWrapper{}
 	err := json.Unmarshal(jsonBlob, wrapper)
 
 	return &wrapper.App, err
 }
+
+type RegistrationIntent struct {
+	Name string
+	Port int
+	Tags []string
+}
+
+func (app *App) RegistrationIntentsNumber() int {
+	if !app.IsConsulApp() {
+		return 0
+	}
+
+	definitions := app.findConsulPortDefinitions()
+	if len(definitions) == 0 {
+		return 1
+	}
+
+	return len(definitions)
+}
+
+func (app *App) RegistrationIntents(task *Task, nameSeparator string) []*RegistrationIntent {
+	commonTags := labelsToTags(app.Labels)
+
+	definitions := app.findConsulPortDefinitions()
+	if len(definitions) == 0 {
+		return []*RegistrationIntent{
+			{
+				Name: app.labelsToName(app.Labels, nameSeparator),
+				Port: task.Ports[0],
+				Tags: commonTags,
+			},
+		}
+	}
+
+	var intents []*RegistrationIntent
+	for _, d := range definitions {
+		intents = append(intents, &RegistrationIntent{
+			Name: app.labelsToName(d.Labels, nameSeparator),
+			Port: task.Ports[d.Index],
+			Tags: append(commonTags, labelsToTags(d.Labels)...),
+		})
+	}
+	return intents
+}
+
+func marathonAppNameToServiceName(name string, nameSeparator string) string {
+	return strings.Replace(strings.Trim(strings.TrimSpace(name), "/"), "/", nameSeparator, -1)
+}
+
+func labelsToTags(labels map[string]string) []string {
+	tags := []string{}
+	for key, value := range labels {
+		if value == "tag" {
+			tags = append(tags, key)
+		}
+	}
+	return tags
+}
+
+func (app *App) labelsToName(labels map[string]string, nameSeparator string) string {
+	appConsulName := app.labelsToRawName(labels)
+	serviceName := marathonAppNameToServiceName(appConsulName, nameSeparator)
+	if serviceName == "" {
+		log.WithField("AppId", app.ID.String()).WithField("ConsulServiceName", appConsulName).
+			Warn("Warning! Invalid Consul service name provided for app. Will use default app name instead.")
+		return marathonAppNameToServiceName(app.ID.String(), nameSeparator)
+	}
+	return serviceName
+}
+
+type indexedPortDefinition struct {
+	Index  int
+	Labels map[string]string
+}
+
+func (app *App) findConsulPortDefinitions() []indexedPortDefinition {
+	var definitions []indexedPortDefinition
+	for i, d := range app.PortDefinitions {
+		if _, ok := d.Labels[MarathonConsulLabel]; ok {
+			definitions = append(definitions, indexedPortDefinition{
+				Index:  i,
+				Labels: d.Labels,
+			})
+		}
+	}
+	return definitions
+}