Sitelet https://github.com/googleapis/java-bigquery/pull/3217/files
Skip to content
This repository was archived by the owner on Aug 30, 2026. It is now read-only.

feat: Add support for ObjectMetadata #3217

Merged
PhongChuong merged 2 commits into from
Apr 17, 2024
Merged

feat: Add support for ObjectMetadata #3217

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f075341
feat: Add support for ObjectMetadata
Mar 28, 2024
7f96513
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Apr 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ If you are using Maven without the BOM, add this to your dependencies:
If you are using Gradle 5.x or later, add this to your dependencies:

```Groovy
implementation platform('com.google.cloud:libraries-bom:26.34.0')
implementation platform('com.google.cloud:libraries-bom:26.37.0')

implementation 'com.google.cloud:google-cloud-bigquery'
```
Expand Down
50 changes: 50 additions & 0 deletions google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,12 @@ public Builder setHivePartitioningOptions(HivePartitioningOptions hivePartitioni
abstract Builder setHivePartitioningOptionsInner(
HivePartitioningOptions hivePartitioningOptions);

public Builder setObjectMetadata(String objectMetadata) {
return setObjectMetadataInner(objectMetadata);
}

abstract Builder setObjectMetadataInner(String objectMetadata);

/** Creates an {@code ExternalTableDefinition} object. */
@Override
public abstract ExternalTableDefinition build();
Expand Down Expand Up @@ -255,6 +261,21 @@ public String getFileSetSpecType() {
@Nullable
public abstract ImmutableList<String> getSourceUrisImmut();

/**
* Returns the object metadata.
*
* @see <a
* href="https://cloud.google.com/bigquery/docs/reference/v2/tables#externalDataConfiguration">
* ObjectMetadata</a>
*/
@Nullable
public String getObjectMetadata() {
return getObjectMetadataInner();
}

@Nullable
abstract String getObjectMetadataInner();

/**
* Returns the source format, and possibly some parsing options, of the external data. Supported
* formats are {@code CSV} and {@code NEWLINE_DELIMITED_JSON}.
Expand Down Expand Up @@ -362,6 +383,10 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC
externalConfigurationPb.setFileSetSpecType(getFileSetSpecType());
}

if (getObjectMetadata() != null) {
externalConfigurationPb.setObjectMetadata(getObjectMetadata());
}

return externalConfigurationPb;
}

Expand Down Expand Up @@ -426,6 +451,24 @@ public static Builder newBuilder(String sourceUri, FormatOptions format) {
return newBuilder().setSourceUris(ImmutableList.of(sourceUri)).setFormatOptions(format);
}

/**
* Creates a builder for an ExternalTableDefinition object.
*
* @param sourceUri the fully-qualified URIs that point to your data in Google Cloud. For Google
* Cloud Bigtable URIs: Exactly one URI can be specified and it has be a fully specified and
* valid HTTPS URL for a Google Cloud Bigtable table. Size limits related to load jobs apply
* to external data sources, plus an additional limit of 10 GB maximum size across all URIs.
* @return a builder for an ExternalTableDefinition object given source URIs and format
* @see <a href="https://cloud.google.com/bigquery/loading-data-into-bigquery#quota">Quota</a>
* @see <a
* href="https://cloud.google.com/bigquery/docs/reference/v2/tables#externalDataConfiguration.sourceFormat">
* Source Format</a>
*/
public static Builder newBuilder(String sourceUri) {
checkArgument(!isNullOrEmpty(sourceUri), "Provided sourceUri is null or empty");
return newBuilder().setSourceUris(ImmutableList.of(sourceUri));
}

/**
* Creates an ExternalTableDefinition object.
*
Expand Down Expand Up @@ -534,6 +577,9 @@ static ExternalTableDefinition fromPb(Table tablePb) {
if (externalDataConfiguration.getFileSetSpecType() != null) {
builder.setFileSetSpecType(externalDataConfiguration.getFileSetSpecType());
}
if (externalDataConfiguration.getObjectMetadata() != null) {
builder.setObjectMetadata(externalDataConfiguration.getObjectMetadata());
}
}
return builder.build();
}
Expand Down Expand Up @@ -597,6 +643,10 @@ static ExternalTableDefinition fromExternalDataConfiguration(
builder.setFileSetSpecType(externalDataConfiguration.getFileSetSpecType());
}

if (externalDataConfiguration.getObjectMetadata() != null) {
builder.setObjectMetadata(externalDataConfiguration.getObjectMetadata());
}

return builder.build();
}
}
3 changes: 3 additions & 0 deletions ...e-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ public class ExternalTableDefinitionTest {
.setMode("AUTO")
.setSourceUriPrefix(SOURCE_URIS.get(0))
.build();
private static final String OBJECT_METADATA = "SIMPLE";
private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION =
ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS)
.setFileSetSpecType("FILE_SET_SPEC_TYPE_FILE_SYSTEM_MATCH")
Expand All @@ -68,6 +69,7 @@ public class ExternalTableDefinitionTest {
.setMaxBadRecords(MAX_BAD_RECORDS)
.setAutodetect(AUTODETECT)
.setHivePartitioningOptions(HIVE_PARTITIONING_OPTIONS)
.setObjectMetadata(OBJECT_METADATA)
.build();

private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION_AVRO =
Expand Down Expand Up @@ -167,5 +169,6 @@ private void compareExternalTableDefinition(
assertEquals(expected.hashCode(), value.hashCode());
assertEquals(expected.getAutodetect(), value.getAutodetect());
assertEquals(expected.getHivePartitioningOptions(), value.getHivePartitioningOptions());
assertEquals(expected.getObjectMetadata(), value.getObjectMetadata());
}
}
43 changes: 43 additions & 0 deletions google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6581,6 +6581,49 @@ public void testExternalTableMetadataCachingNotEnable() throws InterruptedExcept
assertTrue(remoteTable.delete());
}

@Test
public void testObjectTable() throws InterruptedException {
String tableName = "test_object_table";
TableId tableId = TableId.of(DATASET, tableName);

String sourceUri = "gs://" + BUCKET + "/" + JSON_LOAD_FILE;
ExternalTableDefinition externalTableDefinition =
ExternalTableDefinition.newBuilder(sourceUri)
.setConnectionId(
"projects/java-docs-samples-testing/locations/us/connections/DEVREL_TEST_CONNECTION")
.setObjectMetadata("SIMPLE")
.build();
TableInfo tableInfo = TableInfo.of(tableId, externalTableDefinition);
Table createdTable = bigquery.create(tableInfo);
assertNotNull(createdTable);
assertEquals(DATASET, createdTable.getTableId().getDataset());
assertEquals(tableName, createdTable.getTableId().getTable());
Table remoteTable = bigquery.getTable(DATASET, tableName);
assertNotNull(remoteTable);

try {
assertTrue(remoteTable.getDefinition() instanceof ExternalTableDefinition);
assertEquals(createdTable.getTableId(), remoteTable.getTableId());
assertEquals(
"SIMPLE", ((ExternalTableDefinition) remoteTable.getDefinition()).getObjectMetadata());
assertNotNull(remoteTable.getDefinition().getSchema().getFields().get("uri"));

String query = String.format("SELECT * FROM %s.%s", DATASET, tableName);
QueryJobConfiguration config = QueryJobConfiguration.newBuilder(query).build();

Job remoteJob = bigquery.create(JobInfo.of(config));
remoteJob = remoteJob.waitFor();
assertNull(remoteJob.getStatus().getError());

Job queryJob = bigquery.getJob(remoteJob.getJobId());
JobStatistics.QueryStatistics statistics = queryJob.getStatistics();
assertNotNull(statistics);
assertThat(statistics.getTotalBytesProcessed()).isGreaterThan(0);
} finally {
assertTrue(remoteTable.delete());
}
}

static GoogleCredentials loadCredentials(String credentialFile) {
try {
InputStream keyStream = new ByteArrayInputStream(credentialFile.getBytes());
Expand Down