Skip to content

Commit cc47a83

Browse files
ikstewaikstewa
committed
Feat: Add ability to generate files in folders
Adding a new feature flag `prefix_schema_files_with_package` which results in the schema files being generated in folders based on the package of the message.
1 parent e06a9c2 commit cc47a83

File tree

11 files changed

+633
-112
lines changed

11 files changed

+633
-112
lines changed

‎README.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,19 @@ message MyRecord {
146146
}
147147
```
148148

149+
* `prefix_schema_files_with_package` - if set to true, files will be generated into folders matching the proto package. E.g. :
150+
if set to true, will remove the prefixes from enum values. E.g. if you have an enum like:
151+
```protobuf
152+
package my.test.data;
153+
message Yowza {
154+
float hoo_boy = 1;
155+
}
156+
```
157+
158+
...with this option on, it will be generated as:
159+
160+
`./my.test.data/Yowza.avsc`
161+
149162
---
150163

151164
To Do List:

‎input/params.go‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,20 @@
11
package input
22

33
import (
4-
"google.golang.org/protobuf/proto"
5-
"google.golang.org/protobuf/types/pluginpb"
6-
"io"
7-
"os"
8-
"strings"
4+
"google.golang.org/protobuf/proto"
5+
"google.golang.org/protobuf/types/pluginpb"
6+
"io"
7+
"os"
8+
"strings"
99
)
1010

1111
type Params struct {
12-
EmitOnly []string
13-
NamespaceMap map[string]string
14-
CollapseFields []string
15-
RemoveEnumPrefixes bool
16-
PreserveNonStringMaps bool
12+
EmitOnly []string
13+
NamespaceMap map[string]string
14+
CollapseFields []string
15+
RemoveEnumPrefixes bool
16+
PreserveNonStringMaps bool
17+
PrefixSchemaFilesWithPackage bool
1718
}
1819

1920
func ReadRequest() (*pluginpb.CodeGeneratorRequest, error) {
@@ -40,6 +41,8 @@ func parseRawParams(req *pluginpb.CodeGeneratorRequest) map[string]string {
4041
paramStrings := strings.Split(token, "=")
4142
if len(paramStrings) == 2 {
4243
paramMap[paramStrings[0]] = paramStrings[1]
44+
} else {
45+
paramMap[paramStrings[0]] = "true"
4346
}
4447
}
4548
return paramMap
@@ -63,6 +66,8 @@ func ParseParams(req *pluginpb.CodeGeneratorRequest) Params {
6366
params.RemoveEnumPrefixes = v == "true"
6467
} else if k == "preserve_non_string_maps" {
6568
params.PreserveNonStringMaps = true
69+
} else if k == "prefix_schema_files_with_package" {
70+
params.PrefixSchemaFilesWithPackage = true
6671
}
6772
}
6873
return params

‎main.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,16 @@ func processEnum(proto *descriptorpb.EnumDescriptorProto, protoPackage string) {
2828
typeRepo.AddType(enum)
2929
}
3030

31+
func generateSchemaFilename(record avro.Record) string {
32+
if params.PrefixSchemaFilesWithPackage {
33+
return fmt.Sprintf("%s/%s.avsc", record.Namespace, record.Name)
34+
}
35+
return fmt.Sprintf("%s.avsc", record.Name)
36+
}
37+
3138
func generateFileResponse(record avro.Record) (*pluginpb.CodeGeneratorResponse_File, error) {
3239
typeRepo.Start()
33-
fileName := fmt.Sprintf("%s.avsc", record.Name)
40+
fileName := generateSchemaFilename(record)
3441
jsonObj, err := record.ToJSON(typeRepo)
3542
if err != nil {
3643
return nil, fmt.Errorf("error parsing record %s: %w", record.Name, err)

‎main_test.go‎

Lines changed: 133 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
package main
22

33
import (
4-
"fmt"
5-
"github.com/stretchr/testify/assert"
6-
"os"
7-
"os/exec"
8-
"path/filepath"
9-
"strings"
10-
"testing"
4+
"fmt"
5+
"github.com/stretchr/testify/assert"
6+
"os"
7+
"os/exec"
8+
"path/filepath"
9+
"strings"
10+
"testing"
1111
)
1212

1313
// Overall approach taken from https://github.com/mix-php/mix/blob/master/src/grpc/protoc-gen-mix/plugin_test.go
@@ -16,124 +16,156 @@ import (
1616
// tests and instead act as protoc-gen-avro. This allows the test binary to
1717
// pass itself to protoc.
1818
func init() {
19-
if os.Getenv("RUN_AS_PROTOC_GEN_AVRO") != "" {
20-
main()
21-
os.Exit(0)
22-
}
19+
if os.Getenv("RUN_AS_PROTOC_GEN_AVRO") != "" {
20+
main()
21+
os.Exit(0)
22+
}
2323
}
2424

25-
func fileNames(directory string, appendDirectory bool) ([]string, error) {
26-
files, err := os.ReadDir(directory)
27-
if err != nil {
28-
return nil, fmt.Errorf("can't read %s directory: %w", directory, err)
29-
}
30-
var names []string
31-
for _, file := range files {
32-
if file.IsDir() {
33-
continue
34-
}
35-
if appendDirectory {
36-
names = append(names, filepath.Base(directory) + "/" + file.Name())
37-
} else {
38-
names = append(names, file.Name())
39-
}
40-
}
41-
return names, nil
25+
func fileNames(directory string, appendDirectory bool, recurse bool) ([]string, error) {
26+
// var rootPath string
27+
// var names []string
28+
// err := filepath.Walk(directory,
29+
// func(path string, info os.FileInfo, err error) error {
30+
// if err != nil {
31+
// return err
32+
// }
33+
//
34+
// if info.Name() == directory
35+
//
36+
// if appendDirectory {
37+
// names = append(names, filepath.Base(directory)+"/"+path)
38+
// } else {
39+
// names = append(names, path)
40+
// }
41+
// return nil
42+
// })
43+
// if err != nil {
44+
// return nil, fmt.Errorf("can't read %s directory: %w", directory, err)
45+
// }
46+
47+
files, err := os.ReadDir(directory)
48+
if err != nil {
49+
return nil, fmt.Errorf("can't read %s directory: %w", directory, err)
50+
}
51+
var names []string
52+
for _, file := range files {
53+
if file.IsDir() {
54+
if recurse {
55+
r_names, err := fileNames(directory+"/"+file.Name(), true, true)
56+
if err != nil {
57+
return nil, err
58+
}
59+
names = append(names, r_names...)
60+
}
61+
continue
62+
}
63+
if appendDirectory {
64+
names = append(names, filepath.Base(directory)+"/"+file.Name())
65+
} else {
66+
names = append(names, file.Name())
67+
}
68+
}
69+
return names, nil
4270
}
4371

4472
func runTest(t *testing.T, directory string, options map[string]string) {
45-
workdir, _ := os.Getwd()
46-
tmpdir, err := os.MkdirTemp(workdir, "proto-test.")
47-
if err != nil {
48-
t.Fatal(err)
49-
}
50-
defer os.RemoveAll(tmpdir)
51-
52-
args := []string{
53-
"-I.",
54-
"--avro_out=" + tmpdir,
55-
}
56-
names, err := fileNames(workdir + "/testdata", true)
57-
if err != nil {
58-
t.Fatal(fmt.Errorf("testData fileNames %w", err))
59-
}
60-
for _, name := range names {
61-
args = append(args, name)
62-
}
63-
for k, v := range options {
64-
args = append(args, "--avro_opt=" + k + "=" + v)
65-
}
66-
protoc(t, args)
67-
68-
testDir := workdir + "/testdata/" + directory
69-
if os.Getenv("UPDATE_SNAPSHOTS") != "" {
70-
cmd := exec.Command("/bin/sh", "-c", fmt.Sprintf("cp %v/* %v", tmpdir, testDir))
71-
cmd.Run()
72-
} else {
73-
assertEqualFiles(t, testDir, tmpdir)
74-
}
73+
workdir, _ := os.Getwd()
74+
tmpdir, err := os.MkdirTemp(workdir, "proto-test.")
75+
if err != nil {
76+
t.Fatal(err)
77+
}
78+
defer os.RemoveAll(tmpdir)
79+
80+
args := []string{
81+
"-I.",
82+
"--avro_out=" + tmpdir,
83+
}
84+
names, err := fileNames(workdir+"/testdata", true, false)
85+
if err != nil {
86+
t.Fatal(fmt.Errorf("testData fileNames %w", err))
87+
}
88+
for _, name := range names {
89+
args = append(args, name)
90+
}
91+
for k, v := range options {
92+
args = append(args, "--avro_opt="+k+"="+v)
93+
}
94+
protoc(t, args)
95+
96+
testDir := workdir + "/testdata/" + directory
97+
if os.Getenv("UPDATE_SNAPSHOTS") != "" {
98+
cmd := exec.Command("/bin/sh", "-c", fmt.Sprintf("cp -r %v/* %v", tmpdir, testDir))
99+
cmd.Run()
100+
} else {
101+
assertEqualFiles(t, testDir, tmpdir)
102+
}
75103
}
76104

77105
func Test_Base(t *testing.T) {
78-
runTest(t, "base", map[string]string{})
106+
runTest(t, "base", map[string]string{})
79107
}
80108

81109
func Test_CollapseFields(t *testing.T) {
82-
runTest(t, "collapse_fields", map[string]string{"collapse_fields": "StringList"})
110+
runTest(t, "collapse_fields", map[string]string{"collapse_fields": "StringList"})
83111
}
84112

85113
func Test_EmitOnly(t *testing.T) {
86-
runTest(t, "emit_only", map[string]string{"emit_only": "Widget"})
114+
runTest(t, "emit_only", map[string]string{"emit_only": "Widget"})
87115
}
88116

89117
func Test_NamespaceMap(t *testing.T) {
90-
runTest(t, "namespace_map", map[string]string{"namespace_map": "testdata:mynamespace"})
118+
runTest(t, "namespace_map", map[string]string{"namespace_map": "testdata:mynamespace"})
91119
}
92120

93121
func Test_PreserveNonStringMaps(t *testing.T) {
94-
runTest(t, "preserve_non_string_maps", map[string]string{"preserve_non_string_maps": "true"})
122+
runTest(t, "preserve_non_string_maps", map[string]string{"preserve_non_string_maps": "true"})
123+
}
124+
125+
func Test_PrefixSchemaFilesWithPackage(t *testing.T) {
126+
runTest(t, "prefix_schema_files_with_package", map[string]string{"prefix_schema_files_with_package": "true"})
95127
}
96128

97129
func assertEqualFiles(t *testing.T, original, generated string) {
98-
names, err := fileNames(original, false)
99-
if err != nil {
100-
t.Fatal(fmt.Errorf("original fileNames %w", err))
101-
}
102-
generatedNames, err := fileNames(generated, false)
103-
if err != nil {
104-
t.Fatal(fmt.Errorf("generated fileNames %w", err))
105-
}
106-
assert.Equal(t, names, generatedNames)
107-
for i, name := range names {
108-
originalData, err := os.ReadFile(original + "/" + name)
109-
if err != nil {
110-
t.Fatal("Can't find original file for comparison")
111-
}
112-
113-
generatedData, err := os.ReadFile(generated + "/" + generatedNames[i])
114-
if err != nil {
115-
t.Fatal("Can't find generated file for comparison")
116-
}
117-
r := strings.NewReplacer("\r\n", "", "\n", "")
118-
assert.Equal(t, r.Replace(string(originalData)), r.Replace(string(generatedData)))
119-
}
130+
names, err := fileNames(original, false, true)
131+
if err != nil {
132+
t.Fatal(fmt.Errorf("original fileNames %w", err))
133+
}
134+
generatedNames, err := fileNames(generated, false, true)
135+
if err != nil {
136+
t.Fatal(fmt.Errorf("generated fileNames %w", err))
137+
}
138+
assert.Equal(t, names, generatedNames)
139+
for i, name := range names {
140+
originalData, err := os.ReadFile(original + "/" + name)
141+
if err != nil {
142+
t.Fatal("Can't find original file for comparison")
143+
}
144+
145+
generatedData, err := os.ReadFile(generated + "/" + generatedNames[i])
146+
if err != nil {
147+
t.Fatal("Can't find generated file for comparison")
148+
}
149+
r := strings.NewReplacer("\r\n", "", "\n", "")
150+
assert.Equal(t, r.Replace(string(originalData)), r.Replace(string(generatedData)))
151+
}
120152
}
121153

122154
func protoc(t *testing.T, args []string) {
123-
cmd := exec.Command("protoc", "--plugin=protoc-gen-avro=" + os.Args[0])
124-
cmd.Args = append(cmd.Args, args...)
125-
cmd.Env = append(os.Environ(), "RUN_AS_PROTOC_GEN_AVRO=1")
126-
out, err := cmd.CombinedOutput()
127-
128-
if len(out) > 0 || err != nil {
129-
t.Log("RUNNING: ", strings.Join(cmd.Args, " "))
130-
}
131-
132-
if len(out) > 0 {
133-
t.Log(string(out))
134-
}
135-
136-
if err != nil {
137-
t.Fatalf("protoc: %v", err)
138-
}
155+
cmd := exec.Command("protoc", "--plugin=protoc-gen-avro="+os.Args[0])
156+
cmd.Args = append(cmd.Args, args...)
157+
cmd.Env = append(os.Environ(), "RUN_AS_PROTOC_GEN_AVRO=1")
158+
out, err := cmd.CombinedOutput()
159+
160+
if len(out) > 0 || err != nil {
161+
t.Log("RUNNING: ", strings.Join(cmd.Args, " "))
162+
}
163+
164+
if len(out) > 0 {
165+
t.Log(string(out))
166+
}
167+
168+
if err != nil {
169+
t.Fatalf("protoc: %v", err)
170+
}
139171
}

‎testdata/prefix_schema_files_with_package/testdata/AOneOf.avsc‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
{
2+
"type": "record",
3+
"name": "AOneOf",
4+
"namespace": "testdata",
5+
"fields": [
6+
{
7+
"name": "oneof_types",
8+
"type": [
9+
{
10+
"type": "record",
11+
"name": "TypeA",
12+
"namespace": "testdata",
13+
"fields": [
14+
{
15+
"name": "foo",
16+
"type": "string",
17+
"default": ""
18+
}
19+
]
20+
},
21+
{
22+
"type": "record",
23+
"name": "TypeB",
24+
"namespace": "testdata",
25+
"fields": [
26+
{
27+
"name": "bar",
28+
"type": "string",
29+
"default": ""
30+
}
31+
]
32+
}
33+
],
34+
"default": null
35+
}
36+
]
37+
}

0 commit comments

Comments
 (0)