Skip to content

Commit 53e8e5e

Browse files
author
Bin Yi
authored
Merge pull request #7 from SumoLogic/ssong-SUMO-76740-metadata
Ssong sumo 76740 metadata
2 parents 45cb601 + 6c8863d commit 53e8e5e

File tree

8 files changed

+249
-51
lines changed

8 files changed

+249
-51
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,6 @@
1212

1313
# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
1414
.glide/
15+
16+
# plugin file system
17+
rootfs/

‎DEVELOPER.md‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Guide for developers
2+
This project is a plugin for the docker engine, which delivers logs to Sumo Logic by pushing log messages through an HTTP source.
3+
This is the guide for developers who want build and extend the plugin. If you just want use this plugin in your docker environment, please refer to the [readme file](README.md).
4+
5+
## Prerequisite
6+
* [Download](https://www.docker.com/get-docker) and install latest docker engine
7+
* [Download](https://golang.org/dl/) and install latest Go language distribution
8+
* Clone/Download this repository to a local directory, and
9+
* Get all dependencies with
10+
```bash
11+
$ go get -d ./...
12+
```
13+
14+
## Build and install plugin to docker
15+
In bash, run:
16+
```bash
17+
$ sudo bash ./plugin_install.sh
18+
```
19+
If everything goes fine, you can verify that the plugin is correctly installed with:
20+
```bash
21+
$ docker plugin ls
22+
ID NAME DESCRIPTION ENABLED
23+
2dcbb3a32956 sumologic:latest Sumo Logic logging driver true
24+
```
25+
26+
## Uninstall and cleanup the plugin
27+
In bash, run:
28+
```bash
29+
$ sudo bash ./plugin_uninstall.sh
30+
```
31+
32+
## Run sanity test
33+
* Make sure the plugin is installed and enabled
34+
* In bash, run:
35+
```bash
36+
$ docker run --log-driver=sumologic --log-opt sumo-url=<url> -i -t ubuntu bash
37+
```
38+
This will create a bash session in a docker container and send all console contents to a Sumo Logic HTTP source as log lines
39+
40+
## Run unit test
41+
The unit test is written in `XXX_test.go` which `XXX` is the module to be tested. You can launch all unit tests with:
42+
```bash
43+
$ go test -v
44+
```
45+
The unit test do not require docker environment to run. For details about unit test or test framework in Go language, click [here](https://golang.org/pkg/testing/).

‎README.md‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22

33
A Docker logging driver plugin to send logs to Sumo Logic.
44

5-
**Disclaimer:** This plugin is still being developed. We recommend using this plugin in non-production environments.
6-
75
**Note:** Docker plugins are not yet supported on Windows; see Docker's logging driver plugin [documentation].
86

97
[documentation]: https://github.com/docker/cli/blob/master/docs/extend/plugins_logging.md
@@ -22,7 +20,7 @@ ID NAME DESCRIPTION ENABLED
2220
cb0021522669 sumologic:latest SumoLogic logging driver true
2321
```
2422

25-
### Create an HTTP Metrics Source in Sumo Logic
23+
### Create HTTP Source in Sumo Logic
2624
Create a [Sumo Logic account](https://www.sumologic.com/) if you don't currently have one.
2725

2826
Follow these instructions for [setting up an HTTP Source](https://help.sumologic.com/Send-Data/Sources/02Sources-for-Hosted-Collectors/HTTP-Source/zGenerate-a-new-URL-for-an-HTTP-Source) in Sumo Logic. Be sure to obtain the URL endpoint after creating an HTTP Source.
@@ -37,18 +35,24 @@ $ docker run --log-driver=sumologic --log-opt sumo-url=https://<deployment>.sumo
3735
### Sumo Logic Options
3836
To specify additional logging driver options, you can use the `--log-opt NAME=VALUE` flag.
3937

40-
| Option | Required? | Default Value | Description
41-
| --------------------------- | :-------: | :-----------: | -------------------------------------- |
42-
| sumo-url | Yes | | HTTP Source URL
43-
| sumo-compress | No | true | Enable/disable gzip compression. Boolean.
44-
| sumo-compress-level | No | -1 | Set the gzip compression level. Valid values are -1 (default), 0 (no compression), 1 (best speed) ... 9 (best compression).
45-
| sumo-batch-size | No | 1000000 | The number of bytes of logs the driver should wait for before sending them in bulk. If the number of bytes never reaches `sumo-batch-size`, the driver will send the logs in smaller batches at predefined intervals; see `sumo-sending-interval`.
46-
| sumo-sending-interval | No | 2s | The maximum time the driver waits for number of logs to reach `sumo-batch-size` before sending the logs, even if the number of logs is less than the batch size. In the format 72h3m5s, valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
47-
| sumo-proxy-url | No | | Set a proxy URL.
48-
| sumo-insecure-skip-verify | No | false | Ignore server certificate validation. Boolean.
49-
| sumo-root-ca-path | No | | Set the path to a custom root certificate.
50-
| sumo-server-name | No | | Name used to validate the server certificate. By default, uses hostname of the `sumo-url`.
51-
| sumo-queue-size | No | 100 | The maximum number of log batches of size `sumo-batch-size` we can store in memory in the event of network failure, before we begin dropping batches. Thus in the worst case, the plugin will use `sumo-batch-size` * `sumo-queue-size` bytes of memory per container (default 100 MB).
38+
| Option | Required? | Default Value | Description
39+
| ------------------------- | :-------: | :------------------: | -------------------------------------- |
40+
| sumo-url | Yes | | HTTP Source URL
41+
| sumo-source-category | No | HTTP source category | Source category to appear when searching in Sumo Logic by `_sourceCategory`. Within the source category, the token `{{Tag}}` will be replaced with the value of the Docker tag option. If not specified, the default source category configured for the HTTP source will be used.
42+
| sumo-source-name | No | container's name | Source name to appear when searching in Sumo Logic by `_sourceName`. Within the source name, the token `{{Tag}}` will be replaced with the value of the Docker tag option. If not specified, the container's name will be used.
43+
| sumo-source-host | No | host name | Source host to appear when searching in Sumo Logic by `_sourceHost`. Within the source host, the token `{{Tag}}` will be replaced with the value of the Docker tag option. If not specified, the machine host name will be used.
44+
| sumo-compress | No | `true` | Enable/disable gzip compression. Boolean.
45+
| sumo-compress-level | No | `-1` | Set the gzip compression level. Valid values are -1 (default), 0 (no compression), 1 (best speed) ... 9 (best compression).
46+
| sumo-batch-size | No | `1000000` | The number of bytes of logs the driver should wait for before sending them in bulk. If the number of bytes never reaches `sumo-batch-size`, the driver will send the logs in smaller batches at predefined intervals; see `sumo-sending-interval`.
47+
| sumo-sending-interval | No | `2s` | The maximum time the driver waits for number of logs to reach `sumo-batch-size` before sending the logs, even if the number of logs is less than the batch size. In the format 72h3m5s, valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`.
48+
| sumo-proxy-url | No | | Set a proxy URL.
49+
| sumo-insecure-skip-verify | No | `false` | Ignore server certificate validation. Boolean.
50+
| sumo-root-ca-path | No | | Set the path to a custom root certificate.
51+
| sumo-server-name | No | | Name used to validate the server certificate. By default, uses hostname of the `sumo-url`.
52+
| sumo-queue-size | No | `100` | The maximum number of log batches of size `sumo-batch-size` we can store in memory in the event of network failure, before we begin dropping batches. Thus in the worst case, the plugin will use `sumo-batch-size` * `sumo-queue-size` bytes of memory per container (default 100 MB).
53+
| tag | No | `{{.ID}}` | Specifies a tag for messages, which can be used in the "source category", "source name", and "source host" fields. Certain tokens of the form {{X}} are supported. Default value is `{{.ID}}`, the first 12 characters of the container ID. Refer to the [tag log-opt documentation] for more information and a list of supported tokens.
54+
55+
[tag log-opt documentation]: https://docs.docker.com/engine/admin/logging/log_tags/
5256

5357
### Example
5458

@@ -59,7 +63,7 @@ $ docker run --log-driver=sumologic \
5963
--log-opt sumo-queue-size=400 \
6064
--log-opt sumo-sending-frequency=500ms \
6165
--log-opt sumo-compress=false \
62-
--log-opt ...
66+
--log-opt ... \
6367
your/container
6468
```
6569

‎driver.go‎

Lines changed: 80 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,15 @@ import (
1010
"io/ioutil"
1111
"net/http"
1212
"net/url"
13+
"regexp"
1314
"strconv"
15+
"strings"
1416
"sync"
1517
"syscall"
1618
"time"
1719

1820
"github.com/docker/docker/daemon/logger"
21+
"github.com/docker/docker/daemon/logger/loggerutils"
1922
"github.com/pkg/errors"
2023
"github.com/sirupsen/logrus"
2124
"github.com/tonistiigi/fifo"
@@ -52,12 +55,18 @@ const (
5255
If the number of bytes never reaches the batch size, the driver will send the logs in smaller
5356
batches at predefined intervals; see sending interval. */
5457
logOptBatchSize = "sumo-batch-size"
58+
/* The _sourceCategory. If empty, the category of HTTP source will be used */
59+
logOptSourceCategory = "sumo-source-category"
60+
/* The _sourceName. If empty, will be the container's name */
61+
logOptSourceName = "sumo-source-name"
62+
/* The _sourceHost. If empty, will be the machine host name */
63+
logOptSourceHost = "sumo-source-host"
5564

5665
defaultGzipCompression = true
5766
defaultGzipCompressionLevel = gzip.DefaultCompression
5867
defaultInsecureSkipVerify = false
5968

60-
defaultSendingIntervalMs = 2000 * time.Millisecond
69+
defaultSendingInterval = 2000 * time.Millisecond
6170
defaultQueueSizeItems = 100
6271
defaultBatchSizeBytes = 1000000
6372

@@ -93,6 +102,12 @@ type sumoLogger struct {
93102
logBatchQueue chan *sumoLogBatch
94103
sendingInterval time.Duration
95104
batchSize int
105+
106+
info logger.Info
107+
tag string
108+
sourceCategory string
109+
sourceName string
110+
sourceHost string
96111
}
97112

98113
func newSumoDriver() *sumoDriver {
@@ -120,6 +135,29 @@ func (sumoDriver *sumoDriver) NewSumoLogger(file string, info logger.Info) (*sum
120135
}
121136
sumoDriver.mu.Unlock()
122137

138+
sumoUrl := parseLogOptUrl(info, logOptUrl)
139+
if sumoUrl == nil {
140+
return nil, fmt.Errorf("%s: sumo-url must exist and be a valid URL", pluginName)
141+
}
142+
143+
hostname, err := info.Hostname()
144+
if err != nil {
145+
hostname = ""
146+
}
147+
148+
tag, err := loggerutils.ParseLogTag(info, loggerutils.DefaultTemplate)
149+
if err != nil {
150+
return nil, err
151+
}
152+
153+
dictionary := map[string]string {
154+
"tag": tag,
155+
}
156+
157+
sourceCategory := parseLogOptMetadata(info, logOptSourceCategory, "", dictionary)
158+
sourceName := parseLogOptMetadata(info, logOptSourceName, info.ContainerName[1:len(info.ContainerName)], dictionary) // trim leading "/"
159+
sourceHost := parseLogOptMetadata(info, logOptSourceHost, hostname, dictionary)
160+
123161
gzipCompression := parseLogOptBoolean(info, logOptGzipCompression, defaultGzipCompression)
124162
gzipCompressionLevel := parseLogOptGzipCompressionLevel(info, logOptGzipCompressionLevel, defaultGzipCompressionLevel)
125163

@@ -139,7 +177,7 @@ func (sumoDriver *sumoDriver) NewSumoLogger(file string, info logger.Info) (*sum
139177
}
140178

141179
transport := &http.Transport{}
142-
proxyUrl := parseLogOptProxyUrl(info, logOptProxyUrl, nil)
180+
proxyUrl := parseLogOptUrl(info, logOptProxyUrl)
143181
transport.Proxy = http.ProxyURL(proxyUrl)
144182
transport.TLSClientConfig = tlsConfig
145183

@@ -148,7 +186,7 @@ func (sumoDriver *sumoDriver) NewSumoLogger(file string, info logger.Info) (*sum
148186
Timeout: 30 * time.Second,
149187
}
150188

151-
sendingInterval := parseLogOptDuration(info, logOptSendingInterval, defaultSendingIntervalMs)
189+
sendingInterval := parseLogOptDuration(info, logOptSendingInterval, defaultSendingInterval)
152190
queueSize := parseLogOptIntPositive(info, logOptQueueSize, defaultQueueSizeItems)
153191
batchSize := parseLogOptIntPositive(info, logOptBatchSize, defaultBatchSizeBytes)
154192

@@ -159,7 +197,7 @@ func (sumoDriver *sumoDriver) NewSumoLogger(file string, info logger.Info) (*sum
159197
}
160198

161199
newSumoLogger := &sumoLogger{
162-
httpSourceUrl: info.Config[logOptUrl],
200+
httpSourceUrl: sumoUrl.String(),
163201
httpClient: httpClient,
164202
proxyUrl: proxyUrl,
165203
tlsConfig: tlsConfig,
@@ -170,6 +208,11 @@ func (sumoDriver *sumoDriver) NewSumoLogger(file string, info logger.Info) (*sum
170208
logBatchQueue: make(chan *sumoLogBatch, queueSize),
171209
sendingInterval: sendingInterval,
172210
batchSize: batchSize,
211+
info: info,
212+
tag: tag,
213+
sourceCategory: sourceCategory,
214+
sourceName: sourceName,
215+
sourceHost: sourceHost,
173216
}
174217

175218
sumoDriver.mu.Lock()
@@ -191,6 +234,31 @@ func (sumoDriver *sumoDriver) StopLogging(file string) error {
191234
return nil
192235
}
193236

237+
func interpretAll(re *regexp.Regexp, input string, dictionary map[string]string) string {
238+
result := ""
239+
lastIndex := 0
240+
241+
for _, v := range re.FindAllSubmatchIndex([]byte(input), -1) {
242+
groups := []string{}
243+
for i := 0; i < len(v); i += 2 {
244+
groups = append(groups, input[v[i]:v[i + 1]])
245+
}
246+
247+
result += input[lastIndex:v[0]] + dictionary[strings.ToLower(groups[1])] // groups[0] represents the whole pattern, groups[1] is the first capture group
248+
lastIndex = v[1]
249+
}
250+
251+
return result + input[lastIndex:]
252+
}
253+
254+
func parseLogOptMetadata(info logger.Info, logOptKey string, defaultValue string, dictionary map[string]string) string {
255+
if input, exists := info.Config[logOptKey]; exists {
256+
re := regexp.MustCompile(`(?i)\{\{(.*?)\}\}`) // needs to be a lazy match
257+
return interpretAll(re, input, dictionary)
258+
}
259+
return defaultValue
260+
}
261+
194262
func parseLogOptIntPositive(info logger.Info, logOptKey string, defaultValue int) int {
195263
if input, exists := info.Config[logOptKey]; exists {
196264
inputValue64, err := strconv.ParseInt(input, stringToIntBase, stringToIntBitSize)
@@ -201,7 +269,7 @@ func parseLogOptIntPositive(info logger.Info, logOptKey string, defaultValue int
201269
}
202270
inputValue := int(inputValue64)
203271
if inputValue <= 0 {
204-
logrus.Error(fmt.Errorf("%s: %s must be a positive value, got %d. Using default %d.",
272+
logrus.Error(fmt.Errorf("%s: %s must be a positive value, got %d. Using default %d",
205273
pluginName, logOptKey, inputValue, defaultValue))
206274
return defaultValue
207275
}
@@ -220,7 +288,7 @@ func parseLogOptDuration(info logger.Info, logOptKey string, defaultValue time.D
220288
}
221289
zeroSeconds, _ := time.ParseDuration("0s")
222290
if inputValue <= zeroSeconds {
223-
logrus.Error(fmt.Errorf("%s: %s must be a positive duration, got %s. Using default %s.",
291+
logrus.Error(fmt.Errorf("%s: %s must be a positive duration, got %s. Using default %s",
224292
pluginName, logOptKey, inputValue.String(), defaultValue.String()))
225293
return defaultValue
226294
}
@@ -242,17 +310,17 @@ func parseLogOptBoolean(info logger.Info, logOptKey string, defaultValue bool) b
242310
return defaultValue
243311
}
244312

245-
func parseLogOptProxyUrl(info logger.Info, logOptKey string, defaultValue *url.URL) *url.URL {
313+
func parseLogOptUrl(info logger.Info, logOptKey string) *url.URL {
246314
if input, exists := info.Config[logOptKey]; exists {
247315
inputValue, err := url.Parse(input)
248316
if err != nil {
249-
logrus.Error(fmt.Errorf("%s: Failed to parse value of %s as url. Initializing without proxy. %v",
250-
pluginName, logOptKey, defaultValue, err))
251-
return defaultValue
317+
logrus.Error(fmt.Errorf("%s: Failed to parse value of %s as url. %v",
318+
pluginName, logOptKey, err))
319+
return nil
252320
}
253321
return inputValue
254322
}
255-
return defaultValue
323+
return nil
256324
}
257325

258326
func parseLogOptGzipCompressionLevel(info logger.Info, logOptKey string, defaultValue int) int {
@@ -266,7 +334,7 @@ func parseLogOptGzipCompressionLevel(info logger.Info, logOptKey string, default
266334
inputValue := int(inputValue64)
267335
if inputValue < defaultValue || inputValue > gzip.BestCompression {
268336
logrus.Error(fmt.Errorf(
269-
"%s: Not supported level '%d' for %s (supported values between %d and %d). Using default compression.",
337+
"%s: Not supported level '%d' for %s (supported values between %d and %d). Using default compression",
270338
pluginName, inputValue, logOptKey, defaultValue, gzip.BestCompression))
271339
return defaultValue
272340
}

0 commit comments

Comments
 (0)