From a2f15176e9abc26830e256bc1e337d330b88bfae Mon Sep 17 00:00:00 2001 From: Yiru Tang Date: Fri, 2 Dec 2022 20:19:21 +0000 Subject: [PATCH 1/8] . --- .../cloud/bigquery/storage/v1/JsonStreamWriter.java | 1 + .../google/cloud/bigquery/storage/v1/SimpleWriter.java | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java index 6380af4fc6..733a5bcac7 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java @@ -170,6 +170,7 @@ private Message buildMessage(JSONObject json) } } } + /** * Writes a JSONArray that contains JSONObjects to the BigQuery table by first converting the JSON * data to protobuf messages, then using StreamWriter's append() to write the data at the diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java new file mode 100644 index 0000000000..681617a4ca --- /dev/null +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java @@ -0,0 +1,8 @@ +package com.google.cloud.bigquery.storage.v1; + +/** + * A SimpleWriter provides a way to write to BigQuery without a Stream. + */ +public class SimpleWriter { + +} From 4bccdd756261bac17df99ead1be5a3ef9810f56d Mon Sep 17 00:00:00 2001 From: Yiru Tang Date: Sat, 3 Dec 2022 00:21:30 +0000 Subject: [PATCH 2/8] feat: Add a simple unary API for append data --- .../bigquery/storage/v1/SimpleWriter.java | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java index 681617a4ca..070bb85f2f 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java @@ -1,8 +1,67 @@ package com.google.cloud.bigquery.storage.v1; +import com.google.api.core.ApiFuture; +import com.google.common.base.Preconditions; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; +import com.google.protobuf.Descriptors.DescriptorValidationException; +import java.io.IOException; +import com.google.common.cache.LoadingCache; +import com.google.common.cache.CacheBuilder; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.json.JSONArray; + /** * A SimpleWriter provides a way to write to BigQuery without a Stream. */ public class SimpleWriter { + BigQueryWriteClient client; + LoadingCache writerCache; + + /** + * Creates a SimpleWriter instance with provided BigQueryWriteClient. This object should be + * reused to do as many append calls as possible. + * @param client + * @throws IOException + */ + public SimpleWriter(BigQueryWriteClient client) throws IOException { + new SimpleWriter(client, "maximumSize=100,expireAfterWrite=60m"); + } + + /** + * Creates a SimpleWriter with custom caching policy for JsonStreamWriter. + * @param client + * @param cacheSpec + * @throws IOException + */ + public SimpleWriter(BigQueryWriteClient client, String cacheSpec) throws IOException { + this.client = Preconditions.checkNotNull(client); + LoadingCache writerCache = CacheBuilder.from(cacheSpec) + .removalListener(new RemovalListener(){ + public void onRemoval(RemovalNotification notification) { + notification.getValue().close(); + }}) + .build( + new CacheLoader() { + public JsonStreamWriter load(String key) + throws DescriptorValidationException, IOException, InterruptedException { + return JsonStreamWriter.newBuilder(key, client) + .setEnableConnectionPool(true).build(); + } + }); + } + /** + * Appends data to a BigQuery Table. + * + * @param rows the rows in serialized format to write to BigQuery. + * @return the append response wrapped in a future. + */ + public ApiFuture append(String tableName, JSONArray data) + throws ExecutionException, DescriptorValidationException, IOException { + return writerCache.get(tableName).append(data); + } } From bfa956f089efae75e518c3601d42eb95428b56ad Mon Sep 17 00:00:00 2001 From: Yiru Tang Date: Sat, 3 Dec 2022 00:23:25 +0000 Subject: [PATCH 3/8] . --- .../java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java index 070bb85f2f..135fe9ed85 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java @@ -49,7 +49,7 @@ public void onRemoval(RemovalNotification notification public JsonStreamWriter load(String key) throws DescriptorValidationException, IOException, InterruptedException { return JsonStreamWriter.newBuilder(key, client) - .setEnableConnectionPool(true).build(); + .setEnableConnectionPool(true).setTraceId("SimpleWrite").build(); } }); } From dc44442f7a48b8afc8a9588c3ba0825845e425f5 Mon Sep 17 00:00:00 2001 From: Yiru Tang Date: Sat, 3 Dec 2022 00:51:15 +0000 Subject: [PATCH 4/8] . --- .../bigquery/storage/v1/SimpleWriter.java | 84 +++++++++++-------- 1 file changed, 51 insertions(+), 33 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java index 135fe9ed85..2021175cf3 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java @@ -2,56 +2,74 @@ import com.google.api.core.ApiFuture; import com.google.common.base.Preconditions; +import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import com.google.common.cache.RemovalListener; import com.google.common.cache.RemovalNotification; import com.google.protobuf.Descriptors.DescriptorValidationException; import java.io.IOException; -import com.google.common.cache.LoadingCache; -import com.google.common.cache.CacheBuilder; - import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; import org.json.JSONArray; /** - * A SimpleWriter provides a way to write to BigQuery without a Stream. + * A SimpleWriter provides a way to write to BigQuery without a Stream. Underneath, it maintains a + * cache of JsonStreamWriter based on talbe name. */ public class SimpleWriter { BigQueryWriteClient client; LoadingCache writerCache; + String traceId; - /** - * Creates a SimpleWriter instance with provided BigQueryWriteClient. This object should be - * reused to do as many append calls as possible. - * @param client - * @throws IOException - */ - public SimpleWriter(BigQueryWriteClient client) throws IOException { - new SimpleWriter(client, "maximumSize=100,expireAfterWrite=60m"); + /** Constructs a new {@link SimpleWriter.Builder} using the given bigquery write client. */ + public static SimpleWriter.Builder newBuilder(BigQueryWriteClient client) { + return new SimpleWriter.Builder(client); } - /** - * Creates a SimpleWriter with custom caching policy for JsonStreamWriter. - * @param client - * @param cacheSpec - * @throws IOException - */ - public SimpleWriter(BigQueryWriteClient client, String cacheSpec) throws IOException { + private SimpleWriter(SimpleWriter.Builder builder) throws IOException { + this.client = builder.client; + this.traceId = builder.traceId; + this.writerCache = + CacheBuilder.from(builder.cacheSpec) + .removalListener( + new RemovalListener() { + public void onRemoval( + RemovalNotification notification) { + notification.getValue().close(); + } + }) + .build( + new CacheLoader() { + public JsonStreamWriter load(String key) + throws DescriptorValidationException, IOException, InterruptedException { + return JsonStreamWriter.newBuilder(key, client) + .setEnableConnectionPool(true) + .setTraceId(traceId) + .build(); + } + }); + } + + public static final class Builder { + BigQueryWriteClient client; + String cacheSpec = "maximumSize=100,expireAfterWrite=60m"; + String traceId = "SimpleWriter:null"; + + private Builder(BigQueryWriteClient client) { this.client = Preconditions.checkNotNull(client); - LoadingCache writerCache = CacheBuilder.from(cacheSpec) - .removalListener(new RemovalListener(){ - public void onRemoval(RemovalNotification notification) { - notification.getValue().close(); - }}) - .build( - new CacheLoader() { - public JsonStreamWriter load(String key) - throws DescriptorValidationException, IOException, InterruptedException { - return JsonStreamWriter.newBuilder(key, client) - .setEnableConnectionPool(true).setTraceId("SimpleWrite").build(); - } - }); + } + + /** CacheSpec for the JsonStreamWriter cache. */ + public SimpleWriter.Builder setCacheSpec(String cacheSpec) { + this.cacheSpec = cacheSpec; + return this; + } + + /** One time trace id to apply to all writes */ + public SimpleWriter.Builder setTraceId(String traceId) { + this.traceId = "SimpleWriter_" + traceId; + return this; + } } /** From f94dc20bd2e4a13aae1a23c71166dab03994cf5b Mon Sep 17 00:00:00 2001 From: Yiru Tang Date: Sat, 3 Dec 2022 00:53:52 +0000 Subject: [PATCH 5/8] . --- .../java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java index 2021175cf3..e2e7b3b04b 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java @@ -74,6 +74,7 @@ public SimpleWriter.Builder setTraceId(String traceId) { /** * Appends data to a BigQuery Table. + * Rows will appear AT_LEAST_ONCE in BigQuery. * * @param rows the rows in serialized format to write to BigQuery. * @return the append response wrapped in a future. From 3b2df0ac6311c804515f1f352d03e5e7d60be452 Mon Sep 17 00:00:00 2001 From: Yiru Tang Date: Sat, 3 Dec 2022 10:01:35 +0000 Subject: [PATCH 6/8] . --- .../bigquery/storage/v1/SimpleWriter.java | 58 +++++++++++--- .../bigquery/storage/v1/SimpleWriterTest.java | 78 +++++++++++++++++++ .../it/ITBigQueryWriteManualClientTest.java | 48 ++++++++++++ 3 files changed, 175 insertions(+), 9 deletions(-) create mode 100644 google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/SimpleWriterTest.java diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java index e2e7b3b04b..228164f0d7 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java @@ -1,6 +1,8 @@ package com.google.cloud.bigquery.storage.v1; import com.google.api.core.ApiFuture; +import com.google.api.gax.batching.FlowControlSettings; +import com.google.api.gax.batching.FlowController.LimitExceededBehavior; import com.google.common.base.Preconditions; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -10,25 +12,38 @@ import com.google.protobuf.Descriptors.DescriptorValidationException; import java.io.IOException; import java.util.concurrent.ExecutionException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.json.JSONArray; /** - * A SimpleWriter provides a way to write to BigQuery without a Stream. Underneath, it maintains a - * cache of JsonStreamWriter based on talbe name. + * A SimpleWriter provides a way to write to BigQuery in a unary fashion. Underneath, it still uses + * the streaming API through JsonStreamWriter. There is a cache that manages different table writes. + * Default max table destination cached is 100, you can adjust it through the Builder. + * + *

Currently, it only supports AT_LEAST_ONCE writes. + * + *

This class is still in development, DO NO USE IT. + * + *

TODOS: 1. Make the class thread safe 2. Handle failed writer case */ public class SimpleWriter { - BigQueryWriteClient client; - LoadingCache writerCache; - String traceId; + private BigQueryWriteClient client; + private LoadingCache writerCache; + private String traceId; + private static String tableNamePatternString = "projects/[^/]+/datasets/[^/]+/tables/[^/]+"; + private static Pattern tableNamePattern = Pattern.compile(tableNamePatternString); + private boolean ignoreUnkownFields; /** Constructs a new {@link SimpleWriter.Builder} using the given bigquery write client. */ public static SimpleWriter.Builder newBuilder(BigQueryWriteClient client) { return new SimpleWriter.Builder(client); } - private SimpleWriter(SimpleWriter.Builder builder) throws IOException { + private SimpleWriter(SimpleWriter.Builder builder) { this.client = builder.client; this.traceId = builder.traceId; + this.ignoreUnkownFields = builder.ignoreUnknownField; this.writerCache = CacheBuilder.from(builder.cacheSpec) .removalListener( @@ -45,6 +60,11 @@ public JsonStreamWriter load(String key) return JsonStreamWriter.newBuilder(key, client) .setEnableConnectionPool(true) .setTraceId(traceId) + .setIgnoreUnknownFields(ignoreUnkownFields) + .setFlowControlSettings( + FlowControlSettings.newBuilder() + .setLimitExceededBehavior(LimitExceededBehavior.ThrowException) + .build()) .build(); } }); @@ -52,8 +72,12 @@ public JsonStreamWriter load(String key) public static final class Builder { BigQueryWriteClient client; - String cacheSpec = "maximumSize=100,expireAfterWrite=60m"; + // Cache expriration time is set to the same as connection timeout time. After a connection + // is cut, we might be missing schema updates to the object, so we will just let the cache + // expire so that a fresh table schema will be retrieved. + String cacheSpec = "maximumSize=100,expireAfterWrite=10m"; String traceId = "SimpleWriter:null"; + boolean ignoreUnknownField = false; private Builder(BigQueryWriteClient client) { this.client = Preconditions.checkNotNull(client); @@ -70,17 +94,33 @@ public SimpleWriter.Builder setTraceId(String traceId) { this.traceId = "SimpleWriter_" + traceId; return this; } + + /** + * One time set ignoreUnknown field. If you want different behavior, you will need to build a + * different simple writer. + */ + public SimpleWriter.Builder setIgnoreUnknownField(boolean ignoreUnknownField) { + this.ignoreUnknownField = ignoreUnknownField; + return this; + } + + public SimpleWriter build() { + return new SimpleWriter(this); + } } /** - * Appends data to a BigQuery Table. - * Rows will appear AT_LEAST_ONCE in BigQuery. + * Appends data to a BigQuery Table. Rows will appear AT_LEAST_ONCE in BigQuery. * * @param rows the rows in serialized format to write to BigQuery. * @return the append response wrapped in a future. */ public ApiFuture append(String tableName, JSONArray data) throws ExecutionException, DescriptorValidationException, IOException { + Matcher tableNameMatcher = tableNamePattern.matcher(tableName); + if (!tableNameMatcher.matches()) { + throw new IllegalArgumentException("Invalid table name: " + tableName); + } return writerCache.get(tableName).append(data); } } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/SimpleWriterTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/SimpleWriterTest.java new file mode 100644 index 0000000000..112c351554 --- /dev/null +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/SimpleWriterTest.java @@ -0,0 +1,78 @@ +package com.google.cloud.bigquery.storage.v1; + + +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.grpc.testing.MockGrpcService; +import com.google.api.gax.grpc.testing.MockServiceHelper; +import com.google.protobuf.Descriptors.DescriptorValidationException; +import java.util.Arrays; +import java.util.UUID; +import java.util.logging.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class SimpleWriterTest { + private static final Logger log = + Logger.getLogger(com.google.cloud.bigquery.storage.v1.StreamWriterTest.class.getName()); + private static final String TEST_STREAM_1 = "projects/p/datasets/d1/tables/t1/streams/s1"; + private static final String TEST_STREAM_2 = "projects/p/datasets/d2/tables/t2/streams/s2"; + private static final String TEST_TRACE_ID = "DATAFLOW:job_id"; + private FakeScheduledExecutorService fakeExecutor; + private FakeBigQueryWrite testBigQueryWrite; + private static MockServiceHelper serviceHelper; + private BigQueryWriteClient client; + private final TableFieldSchema FOO = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.STRING) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("foo") + .build(); + private final TableFieldSchema BAR = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.STRING) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("bar") + .build(); + + public SimpleWriterTest() throws DescriptorValidationException {} + + @Before + public void setUp() throws Exception { + testBigQueryWrite = new FakeBigQueryWrite(); + serviceHelper = + new MockServiceHelper( + UUID.randomUUID().toString(), Arrays.asList(testBigQueryWrite)); + serviceHelper.start(); + fakeExecutor = new FakeScheduledExecutorService(); + testBigQueryWrite.setExecutor(fakeExecutor); + client = + BigQueryWriteClient.create( + BigQueryWriteSettings.newBuilder() + .setCredentialsProvider(NoCredentialsProvider.create()) + .setTransportChannelProvider(serviceHelper.createChannelProvider()) + .build()); + } + + @After + public void tearDown() throws Exception { + log.info("tearDown called"); + client.close(); + serviceHelper.stop(); + } + + @Test + public void testGoodWrites() throws Exception {} + + @Test + public void testBadWrites() throws Exception {} + + @Test + public void testWriterCacheExpired() throws Exception {} + + @Test + public void testBuilderParams() throws Exception {} +} diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryWriteManualClientTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryWriteManualClientTest.java index 6972813e20..fe46e0e989 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryWriteManualClientTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryWriteManualClientTest.java @@ -1296,4 +1296,52 @@ public void testMultiplexingMixedLocation() assertEquals("us", streamWriter2.getLocation()); assertEquals("eu", streamWriter3.getLocation()); } + + @Test + public void testSimpleWriter() + throws IOException, InterruptedException, ExecutionException, DescriptorValidationException { + String table1 = + String.format( + "projects/%s/datasets/%s/tables/%s", + ServiceOptions.getDefaultProjectId(), DATASET, TABLE); + String table2 = + String.format( + "projects/%s/datasets/%s/tables/%s", + ServiceOptions.getDefaultProjectId(), DATASET_EU, TABLE); + + SimpleWriter simpleWriter = SimpleWriter.newBuilder(BigQueryWriteClient.create()).build(); + JSONObject foo = new JSONObject(); + foo.put("foo", "aaa"); + JSONArray jsonArr = new JSONArray(); + jsonArr.put(foo); + ApiFuture r1 = simpleWriter.append(table1, jsonArr); + foo = new JSONObject(); + foo.put("foo", "bbb"); + jsonArr = new JSONArray(); + jsonArr.put(foo); + ApiFuture r2 = simpleWriter.append(table2, jsonArr); + foo = new JSONObject(); + foo.put("foo", "ccc"); + jsonArr = new JSONArray(); + jsonArr.put(foo); + ApiFuture r3 = simpleWriter.append(table1, jsonArr); + r1.get(); + r2.get(); + r3.get(); + TableResult result = + bigquery.listTableData(tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); + Iterator iter = result.getValues().iterator(); + FieldValueList currentRow = iter.next(); + assertEquals("aaa", currentRow.get(0).getStringValue()); + currentRow = iter.next(); + assertEquals("ccc", currentRow.get(0).getStringValue()); + assertFalse(iter.hasNext()); + result = + bigquery.listTableData( + tableInfoEU.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); + iter = result.getValues().iterator(); + currentRow = iter.next(); + assertEquals("bbb", currentRow.get(0).getStringValue()); + assertFalse(iter.hasNext()); + } } From f8e28b2b4f6e881957f643c34a77bfa29d5247df Mon Sep 17 00:00:00 2001 From: Yiru Tang Date: Sat, 3 Dec 2022 10:03:42 +0000 Subject: [PATCH 7/8] . --- .../com/google/cloud/bigquery/storage/v1/SimpleWriter.java | 4 ++-- .../google/cloud/bigquery/storage/v1/SimpleWriterTest.java | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java index 228164f0d7..807fec10d2 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/SimpleWriter.java @@ -96,8 +96,8 @@ public SimpleWriter.Builder setTraceId(String traceId) { } /** - * One time set ignoreUnknown field. If you want different behavior, you will need to build a - * different simple writer. + * One time set ignoreUnknown field. If true, then if the input has unknown fields to bigquery + * table, the append will not fail. By default, the setting is false. */ public SimpleWriter.Builder setIgnoreUnknownField(boolean ignoreUnknownField) { this.ignoreUnknownField = ignoreUnknownField; diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/SimpleWriterTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/SimpleWriterTest.java index 112c351554..ba8d5fb0bf 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/SimpleWriterTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/SimpleWriterTest.java @@ -1,6 +1,5 @@ package com.google.cloud.bigquery.storage.v1; - import com.google.api.gax.core.NoCredentialsProvider; import com.google.api.gax.grpc.testing.MockGrpcService; import com.google.api.gax.grpc.testing.MockServiceHelper; From 58645ee2ccfb13ec91fef10f365e98007312edbd Mon Sep 17 00:00:00 2001 From: Yiru Tang Date: Sat, 3 Dec 2022 10:05:58 +0000 Subject: [PATCH 8/8] . --- .../com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java index 733a5bcac7..6380af4fc6 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java @@ -170,7 +170,6 @@ private Message buildMessage(JSONObject json) } } } - /** * Writes a JSONArray that contains JSONObjects to the BigQuery table by first converting the JSON * data to protobuf messages, then using StreamWriter's append() to write the data at the