Merge pull request #54 from stephenoken/master

Enums are now treated as type INTEGER in BigQuery

Author: mescanne

.github/workflows/go.yml

@@ -43,6 +43,7 @@ jobs:
          protoc -I. -Iexamples --plugin=./protoc-gen-bq-schema --bq-schema_out=examples examples/foo.proto
          protoc -I. -Iexamples --plugin=./protoc-gen-bq-schema --bq-schema_out=examples examples/foo-proto3.proto
          protoc -I. -Iexamples --plugin=./protoc-gen-bq-schema --bq-schema_out=examples --bq-schema_opt=single-message examples/single_message.proto
+         protoc -I. -Iexamples --plugin=./protoc-gen-bq-schema --bq-schema_out=examples --bq-schema_opt=enum-as-string examples/enum_as_string.proto
     - name: Verify examples are working
       run: |
         if [ -n "$(git status --porcelain)" ]; then

README.md

@@ -8,7 +8,7 @@ So you can reuse existing data definitions in .proto for BigQuery with this plug
 ## Installation
 
 ```sh
-go install github.com/GoogleCloudPlatform/protoc-gen-bq-schema/v2@latest
+go install github.com/GoogleCloudPlatform/protoc-gen-bq-schema/v3@latest
 ```
 
 ## Usage
@@ -72,6 +72,8 @@ The message `foo.Baz` is ignored because it doesn't have option `gen_bq_schema.b
 `protoc --bq-schema_out=. --bq-schema_opt=single-message single_message.proto` will generate a file named `foo/single_message.schema`.
 The message `foo.Baz` is also ignored because it is not the first message in the file.
 
+`protoc --bq-schema_out=. --bq-schema_opt=enum-as-string foo.proto` will generate a file named `foo/bar_table.schema`.
+Fields that are of type enum will be converted to strings in the generated BigQuery schema.
 
 ### Support for PolicyTags
 `protoc-gen-bq-schema` now supports [policyTags](https://cloud.google.com/bigquery/docs/column-level-security-intro).

examples/enum_as_string.proto

@@ -0,0 +1,12 @@
+syntax = "proto2";
+package foo;
+import "bq_table.proto";
+
+message Bar {
+  option (gen_bq_schema.bigquery_opts).table_name = "enum_as_string_table";
+  optional Baz f = 6;
+
+  enum Baz {
+    foo = 1;
+  }
+}

examples/foo.proto

@@ -32,6 +32,12 @@ message Bar {
     }
   ];
 
+  optional Qux f = 6;
+
+  enum Qux {
+    foo = 1;
+  }
+
   optional google.protobuf.Int32Value wkt1 = 11;
   optional google.protobuf.Timestamp wkt2 = 12;
 }

examples/foo/bar_table.schema

@@ -30,6 +30,11 @@
   "mode": "REQUIRED",
   "description": "TIMESTAMP (uint64 in proto) - required in BigQuery"
  },
+ {
+  "name": "f",
+  "type": "INTEGER",
+  "mode": "NULLABLE"
+ },
  {
   "name": "wkt1",
   "type": "INTEGER",

examples/foo/enum_as_string_table.schema

@@ -0,0 +1,7 @@
+[
+ {
+  "name": "f",
+  "type": "STRING",
+  "mode": "NULLABLE"
+ }
+]

go.mod

@@ -1,8 +1,8 @@
-module github.com/GoogleCloudPlatform/protoc-gen-bq-schema/v2
+module github.com/GoogleCloudPlatform/protoc-gen-bq-schema/v3
 
 go 1.21
 
 require (
-	github.com/golang/glog v1.2.3
-	google.golang.org/protobuf v1.35.2
+	github.com/golang/glog v1.2.4
+	google.golang.org/protobuf v1.36.3
 )

go.sum

@@ -1,6 +1,6 @@
-github.com/golang/glog v1.2.3 h1:oDTdz9f5VGVVNGu/Q7UXKWYsD0873HXLHdJUNBsSEKM=
-github.com/golang/glog v1.2.3/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
+github.com/golang/glog v1.2.4 h1:CNNw5U8lSiiBk7druxtSHHTsRWcxKoac6kZKm2peBBc=
+github.com/golang/glog v1.2.4/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
-google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
+google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU=
+google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=

main.go

@@ -29,7 +29,7 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/GoogleCloudPlatform/protoc-gen-bq-schema/v2/pkg/converter"
+	"github.com/GoogleCloudPlatform/protoc-gen-bq-schema/v3/pkg/converter"
 	"github.com/golang/glog"
 	"google.golang.org/protobuf/proto"
 	plugin "google.golang.org/protobuf/types/pluginpb"

pkg/converter/convert.go

@@ -8,7 +8,7 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/GoogleCloudPlatform/protoc-gen-bq-schema/v2/protos"
+	"github.com/GoogleCloudPlatform/protoc-gen-bq-schema/v3/protos"
 	"github.com/golang/glog"
 	"google.golang.org/protobuf/encoding/prototext"
 	"google.golang.org/protobuf/proto"
@@ -47,7 +47,7 @@ var (
 
 		descriptor.FieldDescriptorProto_TYPE_STRING: "STRING",
 		descriptor.FieldDescriptorProto_TYPE_BYTES:  "BYTES",
-		descriptor.FieldDescriptorProto_TYPE_ENUM:   "STRING",
+		descriptor.FieldDescriptorProto_TYPE_ENUM:   "INTEGER",
 
 		descriptor.FieldDescriptorProto_TYPE_BOOL: "BOOLEAN",
 
@@ -401,6 +401,15 @@ func handleSingleMessageOpt(file *descriptor.FileDescriptorProto, requestParam s
 	})
 }
 
+// enumAsStringOpt handles --bq-schema_opt=enum-as-string in protoc params.
+// providing that param tesll protoc-gen-bq-schema to treat enums as strings.
+func enumAsStringOpt(requestParam string) {
+	if !strings.Contains(requestParam, "enum-as-string") {
+		return
+	}
+	typeFromFieldType[descriptor.FieldDescriptorProto_TYPE_ENUM] = "STRING"
+}
+
 func Convert(req *plugin.CodeGeneratorRequest) (*plugin.CodeGeneratorResponse, error) {
 	generateTargets := make(map[string]bool)
 	for _, file := range req.GetFileToGenerate() {
@@ -412,6 +421,7 @@ func Convert(req *plugin.CodeGeneratorRequest) (*plugin.CodeGeneratorResponse, e
 		MinimumEdition:    proto.Int32(int32(descriptor.Edition_EDITION_PROTO2)),
 		MaximumEdition:    proto.Int32(int32(descriptor.Edition_EDITION_MAX)),
 	}
+	enumAsStringOpt(req.GetParameter())
 	for _, file := range req.GetProtoFile() {
 		for msgIndex, msg := range file.GetMessageType() {
 			glog.V(1).Infof("Loading a message type %s from package %s", msg.GetName(), file.GetPackage())

pkg/converter/plugin_test.go

@@ -309,8 +309,8 @@ func TestTypes(t *testing.T) {
 				{ "name": "bool", "type": "BOOLEAN", "mode": "NULLABLE" },
 				{ "name": "str", "type": "STRING", "mode": "NULLABLE" },
 				{ "name": "bytes", "type": "BYTES", "mode": "NULLABLE" },
-				{ "name": "enum1", "type": "STRING", "mode": "NULLABLE" },
-				{ "name": "enum2", "type": "STRING", "mode": "NULLABLE" },
+				{ "name": "enum1", "type": "INTEGER", "mode": "NULLABLE" },
+				{ "name": "enum2", "type": "INTEGER", "mode": "NULLABLE" },
 				{
 					"name": "grp1", "type": "RECORD", "mode": "NULLABLE",
 					"fields": [{ "name": "i1", "type": "INTEGER", "mode": "NULLABLE" }]