Skip to content

Commit cf66e2d

Browse files
committed
feat(open-telemetry#5408): add otlplogfile exporter
This commit adds a new experimental exporter `otlplogfile`, that outputs log records to a JSON line file. It is based on the following specification: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/file-exporter.md Signed-off-by: thomasgouveia <[email protected]>
1 parent 172cfb7 commit cf66e2d

File tree

19 files changed

+1749
-0
lines changed

19 files changed

+1749
-0
lines changed
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# OTLP Log File Exporter
2+
3+
[![PkgGoDev](https://pkg.go.dev/badge/go.opentelemetry.io/otel/exporters/otlp/otlplog/otlplogfile)](https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlplog/otlplogfile)
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
// Copyright The OpenTelemetry Authors
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package otlplogfile // import "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlplogfile"
5+
6+
import "time"
7+
8+
type fnOpt func(config) config
9+
10+
func (f fnOpt) applyOption(c config) config { return f(c) }
11+
12+
// Option sets the configuration value for an Exporter.
13+
type Option interface {
14+
applyOption(config) config
15+
}
16+
17+
// config contains options for the OTLP Log file exporter.
18+
type config struct {
19+
// Path to a file on disk where records must be appended.
20+
// This file is preferably a json line file as stated in the specification.
21+
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/file-exporter.md#json-lines-file
22+
// See: https://jsonlines.org
23+
path string
24+
// Duration represents the interval when the buffer should be flushed.
25+
flushInterval time.Duration
26+
}
27+
28+
func newConfig(options []Option) config {
29+
c := config{
30+
path: "/var/log/opentelemetry/logs.jsonl",
31+
flushInterval: 5 * time.Second,
32+
}
33+
for _, opt := range options {
34+
c = opt.applyOption(c)
35+
}
36+
return c
37+
}
38+
39+
// WithFlushInterval configures the duration after which the buffer is periodically flushed to the disk.
40+
func WithFlushInterval(flushInterval time.Duration) Option {
41+
return fnOpt(func(c config) config {
42+
c.flushInterval = flushInterval
43+
return c
44+
})
45+
}
46+
47+
// WithPath defines a path to a file where the log records will be written.
48+
// If not set, will default to /var/log/opentelemetry/logs.jsonl.
49+
func WithPath(path string) Option {
50+
return fnOpt(func(c config) config {
51+
c.path = path
52+
return c
53+
})
54+
}
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
// Copyright The OpenTelemetry Authors
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
/*
5+
Package otlplogfile provides an OTLP log exporter that outputs log records to a JSON line file. The exporter uses a buffered
6+
file writer to write log records to file to reduce I/O and improve performance.
7+
8+
All Exporters must be created with [New].
9+
10+
See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/file-exporter.md
11+
*/
12+
package otlplogfile // import "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlplogfile"
Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
// Copyright The OpenTelemetry Authors
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package otlplogfile // import "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlplogfile"
5+
6+
import (
7+
"context"
8+
"sync"
9+
10+
"google.golang.org/protobuf/encoding/protojson"
11+
12+
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlplogfile/internal/transform"
13+
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlplogfile/internal/writer"
14+
"go.opentelemetry.io/otel/sdk/log"
15+
lpb "go.opentelemetry.io/proto/otlp/logs/v1"
16+
)
17+
18+
// Exporter is an OpenTelemetry log exporter that outputs log records
19+
// into JSON files. The implementation is based on the specification
20+
// defined here: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/file-exporter.md
21+
type Exporter struct {
22+
mu sync.Mutex
23+
fw *writer.FileWriter
24+
stopped bool
25+
}
26+
27+
// Compile-time check that the implementation satisfies the interface.
28+
var _ log.Exporter = &Exporter{}
29+
30+
// New returns a new [Exporter].
31+
func New(options ...Option) (*Exporter, error) {
32+
cfg := newConfig(options)
33+
34+
fw, err := writer.NewFileWriter(cfg.path, cfg.flushInterval)
35+
if err != nil {
36+
return nil, err
37+
}
38+
39+
return &Exporter{
40+
fw: fw,
41+
stopped: false,
42+
}, nil
43+
}
44+
45+
// Export exports logs records to the file.
46+
func (e *Exporter) Export(ctx context.Context, records []log.Record) error {
47+
// Honor context cancellation
48+
if err := ctx.Err(); err != nil {
49+
return err
50+
}
51+
52+
e.mu.Lock()
53+
defer e.mu.Unlock()
54+
55+
if e.stopped {
56+
return nil
57+
}
58+
59+
data := &lpb.LogsData{
60+
ResourceLogs: transform.ResourceLogs(records),
61+
}
62+
63+
by, err := protojson.Marshal(data)
64+
if err != nil {
65+
return err
66+
}
67+
68+
return e.fw.Export(by)
69+
}
70+
71+
// ForceFlush flushes data to the file.
72+
func (e *Exporter) ForceFlush(_ context.Context) error {
73+
e.mu.Lock()
74+
defer e.mu.Unlock()
75+
76+
if e.stopped {
77+
return nil
78+
}
79+
80+
return e.fw.Flush()
81+
}
82+
83+
// Shutdown shuts down the exporter. Buffered data is written to disk,
84+
// and opened resources such as file will be closed.
85+
func (e *Exporter) Shutdown(_ context.Context) error {
86+
e.mu.Lock()
87+
defer e.mu.Unlock()
88+
89+
if e.stopped {
90+
return nil
91+
}
92+
93+
e.stopped = true
94+
return e.fw.Shutdown()
95+
}
Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
// Copyright The OpenTelemetry Authors
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package otlplogfile // import "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlplogfile"
5+
import (
6+
"context"
7+
"fmt"
8+
"os"
9+
"path"
10+
"runtime"
11+
"sync"
12+
"sync/atomic"
13+
"testing"
14+
"time"
15+
16+
"github.com/stretchr/testify/assert"
17+
"github.com/stretchr/testify/require"
18+
19+
"go.opentelemetry.io/otel/log"
20+
21+
sdklog "go.opentelemetry.io/otel/sdk/log"
22+
)
23+
24+
// tempFile creates a temporary file for the given test case and returns its path on disk.
25+
// The file is automatically cleaned up when the test ends.
26+
func tempFile(tb testing.TB) string {
27+
f, err := os.CreateTemp(tb.TempDir(), tb.Name())
28+
assert.NoError(tb, err, "must not error when creating temporary file")
29+
tb.Cleanup(func() {
30+
assert.NoError(tb, os.RemoveAll(path.Dir(f.Name())), "must clean up files after being written")
31+
})
32+
return f.Name()
33+
}
34+
35+
// makeRecords is a helper function to generate an array of log record with the desired size.
36+
func makeRecords(count int, message string) []sdklog.Record {
37+
var records []sdklog.Record
38+
for i := 0; i < count; i++ {
39+
r := sdklog.Record{}
40+
r.SetSeverityText("INFO")
41+
r.SetSeverity(log.SeverityInfo)
42+
r.SetBody(log.StringValue(message))
43+
r.SetTimestamp(time.Now())
44+
r.SetObservedTimestamp(time.Now())
45+
records = append(records, r)
46+
}
47+
return records
48+
}
49+
50+
func TestExporter(t *testing.T) {
51+
filepath := tempFile(t)
52+
records := makeRecords(1, "hello, world!")
53+
54+
exporter, err := New(WithPath(filepath))
55+
assert.NoError(t, err)
56+
t.Cleanup(func() {
57+
assert.NoError(t, exporter.Shutdown(context.TODO()))
58+
})
59+
60+
err = exporter.Export(context.TODO(), records)
61+
assert.NoError(t, err)
62+
err = exporter.ForceFlush(context.TODO())
63+
assert.NoError(t, err)
64+
}
65+
66+
func TestExporterConcurrentSafe(t *testing.T) {
67+
filepath := tempFile(t)
68+
exporter, err := New(WithPath(filepath))
69+
require.NoError(t, err, "New()")
70+
71+
const goroutines = 10
72+
73+
var wg sync.WaitGroup
74+
ctx, cancel := context.WithCancel(context.Background())
75+
runs := new(uint64)
76+
for i := 0; i < goroutines; i++ {
77+
wg.Add(1)
78+
i := i
79+
go func() {
80+
defer wg.Done()
81+
for {
82+
select {
83+
case <-ctx.Done():
84+
return
85+
default:
86+
_ = exporter.Export(ctx, makeRecords(1, fmt.Sprintf("log from goroutine %d", i)))
87+
_ = exporter.ForceFlush(ctx)
88+
atomic.AddUint64(runs, 1)
89+
}
90+
}
91+
}()
92+
}
93+
94+
for atomic.LoadUint64(runs) == 0 {
95+
runtime.Gosched()
96+
}
97+
98+
assert.NoError(t, exporter.Shutdown(ctx), "must not error when shutting down")
99+
cancel()
100+
wg.Wait()
101+
}
102+
103+
func BenchmarkExporter(b *testing.B) {
104+
for _, logCount := range []int{
105+
10,
106+
100,
107+
500,
108+
1000,
109+
} {
110+
records := makeRecords(logCount, "benchmark")
111+
112+
for name, interval := range map[string]time.Duration{
113+
"no-flush": 0,
114+
"flush-10ms": 10 * time.Millisecond,
115+
"flush-100ms": 100 * time.Millisecond,
116+
"flush-1s": time.Second,
117+
"flush-10s": 10 * time.Second,
118+
} {
119+
filepath := tempFile(b)
120+
exporter, err := New(WithPath(filepath), WithFlushInterval(interval))
121+
require.NoError(b, err, "must not error when calling New()")
122+
123+
b.Run(fmt.Sprintf("%s/%d-logs", name, logCount), func(b *testing.B) {
124+
b.ReportAllocs()
125+
b.ResetTimer()
126+
127+
for i := 0; i < b.N; i++ {
128+
if err := exporter.Export(context.Background(), records); err != nil {
129+
b.Fatalf("failed to export records: %v", err)
130+
}
131+
}
132+
})
133+
134+
if err := exporter.Shutdown(context.Background()); err != nil {
135+
b.Fatalf("failed to shutdown exporter: %v", err)
136+
}
137+
}
138+
}
139+
}
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
module go.opentelemetry.io/otel/exporters/otlp/otlplog/otlplogfile
2+
3+
go 1.21
4+
5+
require (
6+
github.com/stretchr/testify v1.9.0
7+
go.opentelemetry.io/otel v1.28.0
8+
go.opentelemetry.io/otel/log v0.4.0
9+
go.opentelemetry.io/otel/sdk v1.28.0
10+
go.opentelemetry.io/otel/sdk/log v0.4.0
11+
go.opentelemetry.io/otel/trace v1.28.0
12+
go.opentelemetry.io/proto/otlp v1.3.1
13+
google.golang.org/protobuf v1.34.1
14+
)
15+
16+
require (
17+
github.com/davecgh/go-spew v1.1.1 // indirect
18+
github.com/go-logr/logr v1.4.2 // indirect
19+
github.com/go-logr/stdr v1.2.2 // indirect
20+
github.com/google/uuid v1.6.0 // indirect
21+
github.com/pmezard/go-difflib v1.0.0 // indirect
22+
go.opentelemetry.io/otel/metric v1.28.0 // indirect
23+
golang.org/x/sys v0.22.0 // indirect
24+
gopkg.in/yaml.v3 v3.0.1 // indirect
25+
)
26+
27+
replace go.opentelemetry.io/otel => ../../../..
28+
29+
replace go.opentelemetry.io/otel/sdk/log => ../../../../sdk/log
30+
31+
replace go.opentelemetry.io/otel/sdk => ../../../../sdk
32+
33+
replace go.opentelemetry.io/otel/log => ../../../../log
34+
35+
replace go.opentelemetry.io/otel/trace => ../../../../trace
36+
37+
replace go.opentelemetry.io/otel/metric => ../../../../metric
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
2+
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
3+
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
4+
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
5+
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
6+
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
7+
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
8+
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
9+
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
10+
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
11+
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
12+
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
13+
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
14+
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
15+
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
16+
go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=
17+
go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
18+
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
19+
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
20+
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
21+
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
22+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
23+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
24+
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
25+
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
// Copyright The OpenTelemetry Authors
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package internal // import "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlplogfile/internal"
5+
6+
//go:generate gotmpl --body=../../../../../internal/shared/otlp/otlplog/transform/attr_test.go.tmpl "--data={}" --out=transform/attr_test.go
7+
//go:generate gotmpl --body=../../../../../internal/shared/otlp/otlplog/transform/log.go.tmpl "--data={}" --out=transform/log.go
8+
//go:generate gotmpl --body=../../../../../internal/shared/otlp/otlplog/transform/log_attr_test.go.tmpl "--data={}" --out=transform/log_attr_test.go
9+
//go:generate gotmpl --body=../../../../../internal/shared/otlp/otlplog/transform/log_test.go.tmpl "--data={}" --out=transform/log_test.go

0 commit comments

Comments
 (0)