feat: Add support for ObjectMetadata (#3217)

anoopj · gcf-owl-bot[bot] · web-flow · commit 975df05b95b7 · 2024-04-17T13:03:55.000-04:00
* feat: Add support for ObjectMetadata Fixes #3216 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java
@@ -180,6 +180,12 @@ public Builder setHivePartitioningOptions(HivePartitioningOptions hivePartitioni
     abstract Builder setHivePartitioningOptionsInner(
         HivePartitioningOptions hivePartitioningOptions);
 
+    public Builder setObjectMetadata(String objectMetadata) {
+      return setObjectMetadataInner(objectMetadata);
+    }
+
+    abstract Builder setObjectMetadataInner(String objectMetadata);
+
     /** Creates an {@code ExternalTableDefinition} object. */
     @Override
     public abstract ExternalTableDefinition build();
@@ -255,6 +261,21 @@ public String getFileSetSpecType() {
   @Nullable
   public abstract ImmutableList<String> getSourceUrisImmut();
 
+  /**
+   * Returns the object metadata.
+   *
+   * @see <a
+   *     href="https://cloud.google.com/bigquery/docs/reference/v2/tables#externalDataConfiguration">
+   *     ObjectMetadata</a>
+   */
+  @Nullable
+  public String getObjectMetadata() {
+    return getObjectMetadataInner();
+  }
+
+  @Nullable
+  abstract String getObjectMetadataInner();
+
   /**
    * Returns the source format, and possibly some parsing options, of the external data. Supported
    * formats are {@code CSV} and {@code NEWLINE_DELIMITED_JSON}.
@@ -362,6 +383,10 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC
       externalConfigurationPb.setFileSetSpecType(getFileSetSpecType());
     }
 
+    if (getObjectMetadata() != null) {
+      externalConfigurationPb.setObjectMetadata(getObjectMetadata());
+    }
+
     return externalConfigurationPb;
   }
 
@@ -426,6 +451,24 @@ public static Builder newBuilder(String sourceUri, FormatOptions format) {
     return newBuilder().setSourceUris(ImmutableList.of(sourceUri)).setFormatOptions(format);
   }
 
+  /**
+   * Creates a builder for an ExternalTableDefinition object.
+   *
+   * @param sourceUri the fully-qualified URIs that point to your data in Google Cloud. For Google
+   *     Cloud Bigtable URIs: Exactly one URI can be specified and it has be a fully specified and
+   *     valid HTTPS URL for a Google Cloud Bigtable table. Size limits related to load jobs apply
+   *     to external data sources, plus an additional limit of 10 GB maximum size across all URIs.
+   * @return a builder for an ExternalTableDefinition object given source URIs and format
+   * @see <a href="https://cloud.google.com/bigquery/loading-data-into-bigquery#quota">Quota</a>
+   * @see <a
+   *     href="https://cloud.google.com/bigquery/docs/reference/v2/tables#externalDataConfiguration.sourceFormat">
+   *     Source Format</a>
+   */
+  public static Builder newBuilder(String sourceUri) {
+    checkArgument(!isNullOrEmpty(sourceUri), "Provided sourceUri is null or empty");
+    return newBuilder().setSourceUris(ImmutableList.of(sourceUri));
+  }
+
   /**
    * Creates an ExternalTableDefinition object.
    *
@@ -534,6 +577,9 @@ static ExternalTableDefinition fromPb(Table tablePb) {
       if (externalDataConfiguration.getFileSetSpecType() != null) {
         builder.setFileSetSpecType(externalDataConfiguration.getFileSetSpecType());
       }
+      if (externalDataConfiguration.getObjectMetadata() != null) {
+        builder.setObjectMetadata(externalDataConfiguration.getObjectMetadata());
+      }
     }
     return builder.build();
   }
@@ -597,6 +643,10 @@ static ExternalTableDefinition fromExternalDataConfiguration(
       builder.setFileSetSpecType(externalDataConfiguration.getFileSetSpecType());
     }
 
+    if (externalDataConfiguration.getObjectMetadata() != null) {
+      builder.setObjectMetadata(externalDataConfiguration.getObjectMetadata());
+    }
+
     return builder.build();
   }
 }
diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java
@@ -58,6 +58,7 @@ public class ExternalTableDefinitionTest {
           .setMode("AUTO")
           .setSourceUriPrefix(SOURCE_URIS.get(0))
           .build();
+  private static final String OBJECT_METADATA = "SIMPLE";
   private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION =
       ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS)
           .setFileSetSpecType("FILE_SET_SPEC_TYPE_FILE_SYSTEM_MATCH")
@@ -68,6 +69,7 @@ public class ExternalTableDefinitionTest {
           .setMaxBadRecords(MAX_BAD_RECORDS)
           .setAutodetect(AUTODETECT)
           .setHivePartitioningOptions(HIVE_PARTITIONING_OPTIONS)
+          .setObjectMetadata(OBJECT_METADATA)
           .build();
 
   private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION_AVRO =
@@ -167,5 +169,6 @@ private void compareExternalTableDefinition(
     assertEquals(expected.hashCode(), value.hashCode());
     assertEquals(expected.getAutodetect(), value.getAutodetect());
     assertEquals(expected.getHivePartitioningOptions(), value.getHivePartitioningOptions());
+    assertEquals(expected.getObjectMetadata(), value.getObjectMetadata());
   }
 }
diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java
@@ -6579,6 +6579,49 @@ public void testExternalTableMetadataCachingNotEnable() throws InterruptedExcept
     assertTrue(remoteTable.delete());
   }
 
+  @Test
+  public void testObjectTable() throws InterruptedException {
+    String tableName = "test_object_table";
+    TableId tableId = TableId.of(DATASET, tableName);
+
+    String sourceUri = "gs://" + BUCKET + "/" + JSON_LOAD_FILE;
+    ExternalTableDefinition externalTableDefinition =
+        ExternalTableDefinition.newBuilder(sourceUri)
+            .setConnectionId(
+                "projects/java-docs-samples-testing/locations/us/connections/DEVREL_TEST_CONNECTION")
+            .setObjectMetadata("SIMPLE")
+            .build();
+    TableInfo tableInfo = TableInfo.of(tableId, externalTableDefinition);
+    Table createdTable = bigquery.create(tableInfo);
+    assertNotNull(createdTable);
+    assertEquals(DATASET, createdTable.getTableId().getDataset());
+    assertEquals(tableName, createdTable.getTableId().getTable());
+    Table remoteTable = bigquery.getTable(DATASET, tableName);
+    assertNotNull(remoteTable);
+
+    try {
+      assertTrue(remoteTable.getDefinition() instanceof ExternalTableDefinition);
+      assertEquals(createdTable.getTableId(), remoteTable.getTableId());
+      assertEquals(
+          "SIMPLE", ((ExternalTableDefinition) remoteTable.getDefinition()).getObjectMetadata());
+      assertNotNull(remoteTable.getDefinition().getSchema().getFields().get("uri"));
+
+      String query = String.format("SELECT * FROM  %s.%s", DATASET, tableName);
+      QueryJobConfiguration config = QueryJobConfiguration.newBuilder(query).build();
+
+      Job remoteJob = bigquery.create(JobInfo.of(config));
+      remoteJob = remoteJob.waitFor();
+      assertNull(remoteJob.getStatus().getError());
+
+      Job queryJob = bigquery.getJob(remoteJob.getJobId());
+      JobStatistics.QueryStatistics statistics = queryJob.getStatistics();
+      assertNotNull(statistics);
+      assertThat(statistics.getTotalBytesProcessed()).isGreaterThan(0);
+    } finally {
+      assertTrue(remoteTable.delete());
+    }
+  }
+
   static GoogleCredentials loadCredentials(String credentialFile) {
     try {
       InputStream keyStream = new ByteArrayInputStream(credentialFile.getBytes());
