Merge pull request #703 from murgatroid99/proto-loader_messages

Add message and enum type information to package definition output.

Author: murgatroid99

gulpfile.ts

@@ -99,7 +99,7 @@ gulp.task('native.test', 'Run tests of native code', (callback) => {
 });
 
 gulp.task('test.only', 'Run tests without rebuilding anything',
-          ['js.core.test', 'native.test.only']);
+          ['js.core.test', 'native.test.only', 'protobuf.test']);
 
 gulp.task('test', 'Run all tests', (callback) => {
   runSequence('build', 'test.only', 'internal.test.test', callback);

packages/proto-loader/gulpfile.ts

@@ -22,6 +22,7 @@ import * as fs from 'fs';
 import * as mocha from 'gulp-mocha';
 import * as path from 'path';
 import * as execa from 'execa';
+import * as semver from 'semver';
 
 // gulp-help monkeypatches tasks to have an additional description parameter
 const gulp = help(_gulp);
@@ -54,7 +55,21 @@ gulp.task('clean', 'Deletes transpiled code.', ['install'],
 gulp.task('clean.all', 'Deletes all files added by targets', ['clean']);
 
 /**
- * Transpiles TypeScript files in src/ to JavaScript according to the settings
+ * Transpiles TypeScript files in src/ and test/ to JavaScript according to the settings
  * found in tsconfig.json.
  */
-gulp.task('compile', 'Transpiles src/.', () => execNpmCommand('compile'));
+gulp.task('compile', 'Transpiles src/ and test/.', () => execNpmCommand('compile'));
+
+/**
+ * Transpiles src/ and test/, and then runs all tests.
+ */
+gulp.task('test', 'Runs all tests.', () => {
+  if (semver.satisfies(process.version, ">=6")) {
+    return gulp.src(`${outDir}/test/**/*.js`)
+      .pipe(mocha({reporter: 'mocha-jenkins-reporter',
+                    require: ['ts-node/register']}));
+  } else {
+    console.log(`Skipping proto-loader tests for Node ${process.version}`);
+    return Promise.resolve(null);
+  }
+});

packages/proto-loader/src/index.ts

@@ -16,10 +16,25 @@
  *
  */
 import * as Protobuf from 'protobufjs';
+import * as descriptor from 'protobufjs/ext/descriptor';
 import * as fs from 'fs';
 import * as path from 'path';
 import camelCase = require('lodash.camelcase');
 
+declare module 'protobufjs' {
+  interface Type {
+    toDescriptor(protoVersion: string): Protobuf.Message<descriptor.IDescriptorProto> & descriptor.IDescriptorProto;
+  }
+
+  interface Root {
+    toDescriptor(protoVersion: string): Protobuf.Message<descriptor.IFileDescriptorSet> & descriptor.IFileDescriptorSet;
+  }
+
+  interface Enum {
+    toDescriptor(protoVersion: string): Protobuf.Message<descriptor.IEnumDescriptorProto> & descriptor.IEnumDescriptorProto;
+  }
+}
+
 export interface Serialize<T> {
   (value: T): Buffer;
 }
@@ -28,6 +43,20 @@ export interface Deserialize<T> {
   (bytes: Buffer): T;
 }
 
+export interface ProtobufTypeDefinition {
+  format: string;
+  type: object;
+  fileDescriptorProtos: Buffer[];
+}
+
+export interface MessageTypeDefinition extends ProtobufTypeDefinition {
+  format: 'Protocol Buffer 3 DescriptorProto';
+}
+
+export interface EnumTypeDefinition extends ProtobufTypeDefinition {
+  format: 'Protocol Buffer 3 EnumDescriptorProto';
+}
+
 export interface MethodDefinition<RequestType, ResponseType> {
   path: string;
   requestStream: boolean;
@@ -37,20 +66,33 @@ export interface MethodDefinition<RequestType, ResponseType> {
   requestDeserialize: Deserialize<RequestType>;
   responseDeserialize: Deserialize<ResponseType>;
   originalName?: string;
+  requestType: MessageTypeDefinition;
+  responseType: MessageTypeDefinition;
 }
 
 export interface ServiceDefinition {
   [index: string]: MethodDefinition<object, object>;
 }
 
+export type AnyDefinition = ServiceDefinition | MessageTypeDefinition | EnumTypeDefinition;
+
 export interface PackageDefinition {
-  [index: string]: ServiceDefinition;
+  [index: string]: AnyDefinition;
 }
 
 export type Options = Protobuf.IParseOptions & Protobuf.IConversionOptions & {
   includeDirs?: string[];
 };
 
+const descriptorOptions: Protobuf.IConversionOptions = {
+  longs: String,
+  enums: String,
+  bytes: String,
+  defaults: true,
+  oneofs: true,
+  json: true
+};
+
 function joinName(baseName: string, name: string): string {
   if (baseName === '') {
     return name;
@@ -59,19 +101,28 @@ function joinName(baseName: string, name: string): string {
   }
 }
 
-function getAllServices(obj: Protobuf.NamespaceBase, parentName: string): Array<[string, Protobuf.Service]> {
+type HandledReflectionObject = Protobuf.Service | Protobuf.Type | Protobuf.Enum;
+
+function isHandledReflectionObject(obj: Protobuf.ReflectionObject): obj is HandledReflectionObject {
+  return obj instanceof Protobuf.Service || obj instanceof Protobuf.Type || obj instanceof Protobuf.Enum;
+}
+
+function isNamespaceBase(obj: Protobuf.ReflectionObject): obj is Protobuf.NamespaceBase {
+  return obj instanceof Protobuf.Namespace || obj instanceof Protobuf.Root;
+}
+
+function getAllHandledReflectionObjects(obj: Protobuf.ReflectionObject, parentName: string): Array<[string, HandledReflectionObject]> {
   const objName = joinName(parentName, obj.name);
-  if (obj.hasOwnProperty('methods')) {
-    return [[objName, obj as Protobuf.Service]];
+  if (isHandledReflectionObject(obj)) {
+    return [[objName, obj]];
   } else {
-    return obj.nestedArray.map((child) => {
-      if (child.hasOwnProperty('nested')) {
-        return getAllServices(child as Protobuf.NamespaceBase, objName);
-      } else {
-        return [];
-      }
-    }).reduce((accumulator, currentValue) => accumulator.concat(currentValue), []);
+    if (isNamespaceBase(obj) && typeof obj.nested !== undefined) {
+      return Object.keys(obj.nested!).map((name) => {
+        return getAllHandledReflectionObjects(obj.nested![name], objName);
+      }).reduce((accumulator, currentValue) => accumulator.concat(currentValue), []);
+    }
   }
+  return [];
 }
 
 function createDeserializer(cls: Protobuf.Type, options: Options): Deserialize<object> {
@@ -88,16 +139,22 @@ function createSerializer(cls: Protobuf.Type): Serialize<object> {
 }
 
 function createMethodDefinition(method: Protobuf.Method, serviceName: string, options: Options): MethodDefinition<object, object> {
+  /* This is only ever called after the corresponding root.resolveAll(), so we
+   * can assume that the resolved request and response types are non-null */
+  const requestType: Protobuf.Type = method.resolvedRequestType!;
+  const responseType: Protobuf.Type = method.resolvedResponseType!;
   return {
     path: '/' + serviceName + '/' + method.name,
     requestStream: !!method.requestStream,
     responseStream: !!method.responseStream,
-    requestSerialize: createSerializer(method.resolvedRequestType as Protobuf.Type),
-    requestDeserialize: createDeserializer(method.resolvedRequestType as Protobuf.Type, options),
-    responseSerialize: createSerializer(method.resolvedResponseType as Protobuf.Type),
-    responseDeserialize: createDeserializer(method.resolvedResponseType as Protobuf.Type, options),
+    requestSerialize: createSerializer(requestType),
+    requestDeserialize: createDeserializer(requestType, options),
+    responseSerialize: createSerializer(responseType),
+    responseDeserialize: createDeserializer(responseType, options),
     // TODO(murgatroid99): Find a better way to handle this
-    originalName: camelCase(method.name)
+    originalName: camelCase(method.name),
+    requestType: createMessageDefinition(requestType),
+    responseType: createMessageDefinition(responseType)
   };
 }
 
@@ -109,10 +166,58 @@ function createServiceDefinition(service: Protobuf.Service, name: string, option
   return def;
 }
 
+const fileDescriptorCache: Map<Protobuf.Root, Buffer[]> = new Map<Protobuf.Root, Buffer[]>();
+function getFileDescriptors(root: Protobuf.Root): Buffer[] {
+  if (fileDescriptorCache.has(root)) {
+    return fileDescriptorCache.get(root)!;
+  } else {
+    const descriptorList: descriptor.IFileDescriptorProto[] = root.toDescriptor('proto3').file;
+    const bufferList: Buffer[] = descriptorList.map(value => Buffer.from(descriptor.FileDescriptorProto.encode(value).finish()));
+    fileDescriptorCache.set(root, bufferList);
+    return bufferList;
+  }
+}
+
+function createMessageDefinition(message: Protobuf.Type): MessageTypeDefinition {
+  const messageDescriptor: protobuf.Message<descriptor.IDescriptorProto> = message.toDescriptor('proto3');
+  return {
+    format: 'Protocol Buffer 3 DescriptorProto',
+    type: messageDescriptor.$type.toObject(messageDescriptor, descriptorOptions),
+    fileDescriptorProtos: getFileDescriptors(message.root)
+  };
+}
+
+function createEnumDefinition(enumType: Protobuf.Enum): EnumTypeDefinition {
+  const enumDescriptor: protobuf.Message<descriptor.IEnumDescriptorProto> = enumType.toDescriptor('proto3');
+  return {
+    format: 'Protocol Buffer 3 EnumDescriptorProto',
+    type: enumDescriptor.$type.toObject(enumDescriptor, descriptorOptions),
+    fileDescriptorProtos: getFileDescriptors(enumType.root)
+  };
+}
+
+/**
+ * function createDefinition(obj: Protobuf.Service, name: string, options: Options): ServiceDefinition;
+ * function createDefinition(obj: Protobuf.Type, name: string, options: Options): MessageTypeDefinition;
+ * function createDefinition(obj: Protobuf.Enum, name: string, options: Options): EnumTypeDefinition;
+ */
+function createDefinition(obj: HandledReflectionObject, name: string, options: Options): AnyDefinition {
+  if (obj instanceof Protobuf.Service) {
+    return createServiceDefinition(obj, name, options);
+  } else if (obj instanceof Protobuf.Type) {
+    return createMessageDefinition(obj);
+  } else if (obj instanceof Protobuf.Enum) {
+    return createEnumDefinition(obj);
+  } else {
+    throw new Error('Type mismatch in reflection object handling');
+  }
+}
+
 function createPackageDefinition(root: Protobuf.Root, options: Options): PackageDefinition {
   const def: PackageDefinition = {};
-  for (const [name, service] of getAllServices(root, '')) {
-    def[name] = createServiceDefinition(service, name, options);
+  root.resolveAll();
+  for (const [name, obj] of getAllHandledReflectionObjects(root, '')) {
+    def[name] = createDefinition(obj, name, options);
   }
   return def;
 }

packages/proto-loader/test/descriptor_type_test.ts

@@ -0,0 +1,53 @@
+import * as assert from 'assert';
+
+import * as proto_loader from '../src/index';
+
+// Relative path from build output directory to test_protos directory
+const TEST_PROTO_DIR = `${__dirname}/../../test_protos/`;
+
+type TypeDefinition = proto_loader.EnumTypeDefinition | proto_loader.MessageTypeDefinition;
+
+function isTypeObject(obj: proto_loader.AnyDefinition): obj is TypeDefinition {
+  return 'format' in obj;
+}
+
+describe('Descriptor types', () => {
+  it('Should be output for each enum', (done) => {
+    proto_loader.load(`${TEST_PROTO_DIR}/enums.proto`).then((packageDefinition) => {
+      assert('Enum1' in packageDefinition);
+      assert(isTypeObject(packageDefinition.Enum1));
+      // Need additional check because compiler doesn't understand asserts
+      if(isTypeObject(packageDefinition.Enum1)) {
+        const enum1Def: TypeDefinition = packageDefinition.Enum1;
+        assert.strictEqual(enum1Def.format, 'Protocol Buffer 3 EnumDescriptorProto');
+      }
+
+      assert('Enum2' in packageDefinition);
+      assert(isTypeObject(packageDefinition.Enum2));
+      // Need additional check because compiler doesn't understand asserts
+      if(isTypeObject(packageDefinition.Enum2)) {
+        const enum2Def: TypeDefinition = packageDefinition.Enum2;
+        assert.strictEqual(enum2Def.format, 'Protocol Buffer 3 EnumDescriptorProto');
+      }
+      done();
+    }, (error) => {done(error);});
+  });
+  it('Should be output for each message', (done) => {
+    proto_loader.load(`${TEST_PROTO_DIR}/messages.proto`).then((packageDefinition) => {
+      assert('LongValues' in packageDefinition);
+      assert(isTypeObject(packageDefinition.LongValues));
+      if(isTypeObject(packageDefinition.LongValues)) {
+        const longValuesDef: TypeDefinition = packageDefinition.LongValues;
+        assert.strictEqual(longValuesDef.format, 'Protocol Buffer 3 DescriptorProto');
+      }
+
+      assert('SequenceValues' in packageDefinition);
+      assert(isTypeObject(packageDefinition.SequenceValues));
+      if(isTypeObject(packageDefinition.SequenceValues)) {
+        const sequenceValuesDef: TypeDefinition = packageDefinition.SequenceValues;
+        assert.strictEqual(sequenceValuesDef.format, 'Protocol Buffer 3 DescriptorProto');
+      }
+      done();
+    }, (error) => {done(error);});
+  });
+});

packages/proto-loader/test_protos/enums.proto

@@ -0,0 +1,27 @@
+// Copyright 2019 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+enum Enum1 {
+  DEFAULT = 0;
+  VALUE1 = 1;
+  VALUE2 = 2;
+}
+
+enum Enum2 {
+  DEFAULT = 0;
+  ABC = 5;
+  DEF = 10;
+}

packages/proto-loader/test_protos/messages.proto

@@ -0,0 +1,28 @@
+// Copyright 2019 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+message LongValues {
+  int64 int_64 = 1;
+  uint64 uint_64 = 2;
+  sint64 sint_64 = 3;
+  fixed64 fixed_64 = 4;
+  sfixed64 sfixed_64 = 5;
+}
+
+message SequenceValues {
+  bytes bytes_field = 1;
+  repeated int32 repeated_field = 2;
+}

packages/proto-loader/tsconfig.json

@@ -5,6 +5,7 @@
     "outDir": "build"
   },
   "include": [
-    "src/*.ts"
+    "src/*.ts",
+    "test/*.ts"
   ]
 }