Content-Length: 960821 | pFad | https://www.github.com/googleapis/java-bigquery/commit/d33b64594f9b3217b0fc89d255618b558a9d45e8

8CC feat: add materialize view (#174) · googleapis/java-bigquery@d33b645 · GitHub
Skip to content

Commit d33b645

Browse files
author
Praful Makani
authored
feat: add materialize view (#174)
* feat: add materialize view * feat: add unit test case * feat: modified comment * feat: add additional fields and code coverage * feat: add integration test * feat: run formatter
1 parent 6213ea9 commit d33b645

File tree

4 files changed

+303
-12
lines changed

4 files changed

+303
-12
lines changed

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/MaterializedViewDefinition.java

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.bigquery;
18+
19+
import com.google.api.services.bigquery.model.Table;
20+
import com.google.auto.value.AutoValue;
21+
import javax.annotation.Nullable;
22+
23+
@AutoValue
24+
public abstract class MaterializedViewDefinition extends TableDefinition {
25+
26+
private static final long serialVersionUID = 5898696389126164276L;
27+
28+
@AutoValue.Builder
29+
public abstract static class Builder
30+
extends TableDefinition.Builder<MaterializedViewDefinition, Builder> {
31+
32+
/**
33+
* [Output-only] The time when this materialized view was last modified, in milliseconds since
34+
* the epoch.
35+
*/
36+
abstract Builder setLastRefreshTime(Long lastRefreshTime);
37+
38+
/** Sets the query whose result is persisted. */
39+
public abstract Builder setQuery(String query);
40+
41+
/**
42+
* Set enable automatic refresh of the materialized view when the base table is updated. The
43+
* default value is "true".
44+
*/
45+
public abstract Builder setEnableRefresh(Boolean enableRefresh);
46+
47+
/**
48+
* Set a maximum frequency at which this materialized view will be refreshed. The default value
49+
* is "1800000" (30 minutes).
50+
*/
51+
public abstract Builder setRefreshIntervalMs(Long refreshIntervalMs);
52+
53+
/** Sets the table schema. */
54+
@Override
55+
public abstract Builder setSchema(Schema schema);
56+
57+
@Override
58+
public abstract Builder setType(Type type);
59+
60+
/** Creates a {@code MaterializedViewDefinition} object. */
61+
@Override
62+
public abstract MaterializedViewDefinition build();
63+
}
64+
65+
/**
66+
* Returns time when this materialized view was last modified, in milliseconds since the epoch.
67+
*/
68+
@Nullable
69+
public abstract Long getLastRefreshTime();
70+
71+
/** Returns a query whose result is persisted. */
72+
@Nullable
73+
public abstract String getQuery();
74+
75+
/**
76+
* Returns enable automatic refresh of the materialized view when the base table is updated. The
77+
* default value is "true".
78+
*/
79+
@Nullable
80+
public abstract Boolean getEnableRefresh();
81+
82+
/**
83+
* Returns a maximum frequency at which this materialized view will be refreshed. The default
84+
* value is "1800000" (30 minutes).
85+
*/
86+
@Nullable
87+
public abstract Long getRefreshIntervalMs();
88+
89+
/** Returns a builder for the {@code MaterializedViewDefinition} object. */
90+
public abstract Builder toBuilder();
91+
92+
@Override
93+
Table toPb() {
94+
Table tablePb = super.toPb();
95+
com.google.api.services.bigquery.model.MaterializedViewDefinition materializedViewDefinition =
96+
new com.google.api.services.bigquery.model.MaterializedViewDefinition();
97+
if (getQuery() != null) {
98+
materializedViewDefinition.setQuery(getQuery());
99+
}
100+
if (getLastRefreshTime() != null) {
101+
materializedViewDefinition.setLastRefreshTime(getLastRefreshTime());
102+
}
103+
if (getEnableRefresh() != null) {
104+
materializedViewDefinition.setEnableRefresh(getEnableRefresh());
105+
}
106+
if (getRefreshIntervalMs() != null) {
107+
materializedViewDefinition.setRefreshIntervalMs(getRefreshIntervalMs());
108+
}
109+
tablePb.setMaterializedView(materializedViewDefinition);
110+
return tablePb;
111+
}
112+
113+
static Builder newBuilder() {
114+
return new AutoValue_MaterializedViewDefinition.Builder().setType(Type.MATERIALIZED_VIEW);
115+
}
116+
117+
/**
118+
* Returns a builder for a BigQuery materialized view definition.
119+
*
120+
* @param query the query used to generate the materialized view
121+
*/
122+
public static Builder newBuilder(String query) {
123+
return newBuilder().setQuery(query);
124+
}
125+
126+
/**
127+
* Returns a builder for a BigQuery materialized view definition.
128+
*
129+
* @param query the query used to generate the materialized view
130+
*/
131+
public static MaterializedViewDefinition of(String query) {
132+
return newBuilder(query).build();
133+
}
134+
135+
static MaterializedViewDefinition fromPb(Table tablePb) {
136+
Builder builder = newBuilder().table(tablePb);
137+
if (tablePb.getMaterializedView() != null) {
138+
com.google.api.services.bigquery.model.MaterializedViewDefinition materializedViewDefinition =
139+
tablePb.getMaterializedView();
140+
if (materializedViewDefinition.getQuery() != null) {
141+
builder.setQuery(materializedViewDefinition.getQuery());
142+
}
143+
if (materializedViewDefinition.getLastRefreshTime() != null) {
144+
builder.setLastRefreshTime(materializedViewDefinition.getLastRefreshTime());
145+
}
146+
if (materializedViewDefinition.getEnableRefresh() != null) {
147+
builder.setEnableRefresh(materializedViewDefinition.getEnableRefresh());
148+
}
149+
if (materializedViewDefinition.getRefreshIntervalMs() != null) {
150+
builder.setRefreshIntervalMs(materializedViewDefinition.getRefreshIntervalMs());
151+
}
152+
}
153+
return builder.build();
154+
}
155+
}

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableDefinition.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,14 @@ public Type apply(String constant) {
5757
*/
5858
public static final Type VIEW = type.createAndRegister("VIEW");
5959

60+
/**
61+
* SQL query whose result is persisted. Instances of {@code MaterializedViewDefinition} for this
62+
* type are implemented by {@link MaterializedViewDefinition}.
63+
*
64+
* @see <a href="https://cloud.google.com/bigquery/querying-data#views">Views</a>
65+
*/
66+
public static final Type MATERIALIZED_VIEW = type.createAndRegister("MATERIALIZED_VIEW");
67+
6068
/**
6169
* A BigQuery table backed by external data. Instances of {@code TableDefinition} for this type
6270
* are implemented by {@link ExternalTableDefinition}.
@@ -151,6 +159,8 @@ static <T extends TableDefinition> T fromPb(Table tablePb) {
151159
return (T) StandardTableDefinition.fromPb(tablePb);
152160
case "VIEW":
153161
return (T) ViewDefinition.fromPb(tablePb);
162+
case "MATERIALIZED_VIEW":
163+
return (T) MaterializedViewDefinition.fromPb(tablePb);
154164
case "EXTERNAL":
155165
return (T) ExternalTableDefinition.fromPb(tablePb);
156166
case "MODEL":

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/MaterializedViewDefinitionTest.java

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.bigquery;
18+
19+
import static org.junit.Assert.assertEquals;
20+
import static org.junit.Assert.assertTrue;
21+
22+
import org.junit.Test;
23+
24+
public class MaterializedViewDefinitionTest {
25+
26+
private static final String MATERIALIZED_VIEW_QUERY = "MATERIALIZED_VIEW_QUERY";
27+
private static final Long LAST_REFRESH_TIME = 1580302008L;
28+
private static final Boolean ENABLE_REFRESH = false;
29+
private static final Long REFRESH_INTERVAL_MS = 60000L;
30+
private static final Schema SCHEMA = Schema.of();
31+
private static final MaterializedViewDefinition MATERIALIZED_VIEW_DEFINITION =
32+
MaterializedViewDefinition.newBuilder()
33+
.setSchema(SCHEMA)
34+
.setQuery(MATERIALIZED_VIEW_QUERY)
35+
.setLastRefreshTime(LAST_REFRESH_TIME)
36+
.setEnableRefresh(ENABLE_REFRESH)
37+
.setRefreshIntervalMs(REFRESH_INTERVAL_MS)
38+
.build();
39+
40+
@Test
41+
public void testToBuilder() {
42+
compareMaterializedView(
43+
MATERIALIZED_VIEW_DEFINITION, MATERIALIZED_VIEW_DEFINITION.toBuilder().build());
44+
MaterializedViewDefinition materializedViewDefinition =
45+
MATERIALIZED_VIEW_DEFINITION.toBuilder().setQuery("NEW QUERY").build();
46+
assertEquals("NEW QUERY", materializedViewDefinition.getQuery());
47+
materializedViewDefinition =
48+
materializedViewDefinition.toBuilder().setQuery(MATERIALIZED_VIEW_QUERY).build();
49+
compareMaterializedView(MATERIALIZED_VIEW_DEFINITION, materializedViewDefinition);
50+
}
51+
52+
@Test
53+
public void testToBuilderIncomplete() {
54+
TableDefinition materializedViewDefinition =
55+
MaterializedViewDefinition.of(MATERIALIZED_VIEW_QUERY);
56+
assertEquals(materializedViewDefinition, materializedViewDefinition.toBuilder().build());
57+
}
58+
59+
@Test
60+
public void testBuilder() {
61+
assertEquals(MATERIALIZED_VIEW_QUERY, MATERIALIZED_VIEW_DEFINITION.getQuery());
62+
assertEquals(TableDefinition.Type.MATERIALIZED_VIEW, MATERIALIZED_VIEW_DEFINITION.getType());
63+
assertEquals(LAST_REFRESH_TIME, MATERIALIZED_VIEW_DEFINITION.getLastRefreshTime());
64+
MaterializedViewDefinition materializedViewDefinition =
65+
MaterializedViewDefinition.newBuilder()
66+
.setSchema(SCHEMA)
67+
.setQuery(MATERIALIZED_VIEW_QUERY)
68+
.setLastRefreshTime(LAST_REFRESH_TIME)
69+
.setEnableRefresh(ENABLE_REFRESH)
70+
.setRefreshIntervalMs(REFRESH_INTERVAL_MS)
71+
.build();
72+
assertEquals(MATERIALIZED_VIEW_DEFINITION, materializedViewDefinition);
73+
}
74+
75+
@Test
76+
public void testToAndFromPb() {
77+
MaterializedViewDefinition materializedViewDefinition =
78+
MATERIALIZED_VIEW_DEFINITION.toBuilder().build();
79+
assertTrue(
80+
TableDefinition.fromPb(materializedViewDefinition.toPb())
81+
instanceof MaterializedViewDefinition);
82+
compareMaterializedView(
83+
materializedViewDefinition,
84+
TableDefinition.<MaterializedViewDefinition>fromPb(materializedViewDefinition.toPb()));
85+
}
86+
87+
private void compareMaterializedView(
88+
MaterializedViewDefinition expected, MaterializedViewDefinition actual) {
89+
assertEquals(expected.getType(), actual.getType());
90+
assertEquals(expected.getSchema(), actual.getSchema());
91+
assertEquals(expected.getQuery(), actual.getQuery());
92+
assertEquals(expected.getLastRefreshTime(), actual.getLastRefreshTime());
93+
assertEquals(expected.getEnableRefresh(), actual.getEnableRefresh());
94+
assertEquals(expected.getRefreshIntervalMs(), actual.getRefreshIntervalMs());
95+
assertEquals(expected.toString(), actual.toString());
96+
assertEquals(expected.hashCode(), actual.hashCode());
97+
assertEquals(expected, actual);
98+
}
99+
}

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java

Lines changed: 39 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import com.google.auth.oauth2.ServiceAccountCredentials;
3333
import com.google.cloud.Date;
3434
import com.google.cloud.RetryOption;
35+
import com.google.cloud.ServiceOptions;
3536
import com.google.cloud.bigquery.Acl;
3637
import com.google.cloud.bigquery.BigQuery;
3738
import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption;
@@ -65,6 +66,7 @@
6566
import com.google.cloud.bigquery.JobStatistics.LoadStatistics;
6667
import com.google.cloud.bigquery.LegacySQLTypeName;
6768
import com.google.cloud.bigquery.LoadJobConfiguration;
69+
import com.google.cloud.bigquery.MaterializedViewDefinition;
6870
import com.google.cloud.bigquery.Model;
6971
import com.google.cloud.bigquery.ModelId;
7072
import com.google.cloud.bigquery.ModelInfo;
@@ -134,6 +136,7 @@ public class ITBigQueryTest {
134136
private static final String OTHER_DATASET = RemoteBigQueryHelper.generateDatasetName();
135137
private static final String MODEL_DATASET = RemoteBigQueryHelper.generateDatasetName();
136138
private static final String ROUTINE_DATASET = RemoteBigQueryHelper.generateDatasetName();
139+
private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId();
137140
private static final Map<String, String> LABELS =
138141
ImmutableMap.of(
139142
"example-label1", "example-value1",
@@ -219,6 +222,17 @@ public class ITBigQueryTest {
219222
Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN)
220223
.setMode(Field.Mode.NULLABLE)
221224
.build());
225+
private static final Schema VIEW_SCHEMA =
226+
Schema.of(
227+
Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP)
228+
.setMode(Field.Mode.NULLABLE)
229+
.build(),
230+
Field.newBuilder("StringField", LegacySQLTypeName.STRING)
231+
.setMode(Field.Mode.NULLABLE)
232+
.build(),
233+
Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN)
234+
.setMode(Field.Mode.NULLABLE)
235+
.build());
222236
private static final RangePartitioning.Range RANGE =
223237
RangePartitioning.Range.newBuilder().setStart(1L).setInterval(2L).setEnd(20L).build();
224238
private static final RangePartitioning RANGE_PARTITIONING =
@@ -640,18 +654,7 @@ public void testCreateViewTable() throws InterruptedException {
640654
assertNotNull(remoteTable);
641655
assertEquals(createdTable.getTableId(), remoteTable.getTableId());
642656
assertTrue(remoteTable.getDefinition() instanceof ViewDefinition);
643-
Schema expectedSchema =
644-
Schema.of(
645-
Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP)
646-
.setMode(Field.Mode.NULLABLE)
647-
.build(),
648-
Field.newBuilder("StringField", LegacySQLTypeName.STRING)
649-
.setMode(Field.Mode.NULLABLE)
650-
.build(),
651-
Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN)
652-
.setMode(Field.Mode.NULLABLE)
653-
.build());
654-
assertEquals(expectedSchema, remoteTable.getDefinition().getSchema());
657+
assertEquals(VIEW_SCHEMA, remoteTable.getDefinition().getSchema());
655658
QueryJobConfiguration config =
656659
QueryJobConfiguration.newBuilder("SELECT * FROM " + tableName)
657660
.setDefaultDataset(DatasetId.of(DATASET))
@@ -678,6 +681,30 @@ public void testCreateViewTable() throws InterruptedException {
678681
assertTrue(remoteTable.delete());
679682
}
680683

684+
@Test
685+
public void testCreateMaterializedViewTable() {
686+
String tableName = "test_materialized_view_table";
687+
TableId tableId = TableId.of(DATASET, tableName);
688+
MaterializedViewDefinition viewDefinition =
689+
MaterializedViewDefinition.newBuilder(
690+
String.format(
691+
"SELECT MAX(TimestampField) AS TimestampField,StringField, MAX(BooleanField) AS BooleanField FROM %s.%s.%s GROUP BY StringField",
692+
PROJECT_ID, DATASET, TABLE_ID.getTable()))
693+
.build();
694+
TableInfo tableInfo = TableInfo.of(tableId, viewDefinition);
695+
Table createdTable = bigquery.create(tableInfo);
696+
assertNotNull(createdTable);
697+
assertEquals(DATASET, createdTable.getTableId().getDataset());
698+
assertEquals(tableName, createdTable.getTableId().getTable());
699+
Table remoteTable = bigquery.getTable(DATASET, tableName);
700+
assertNotNull(remoteTable);
701+
assertEquals(createdTable.getTableId(), remoteTable.getTableId());
702+
assertEquals(createdTable.getTableId(), remoteTable.getTableId());
703+
assertTrue(remoteTable.getDefinition() instanceof MaterializedViewDefinition);
704+
assertEquals(VIEW_SCHEMA, remoteTable.getDefinition().getSchema());
705+
assertTrue(remoteTable.delete());
706+
}
707+
681708
@Test
682709
public void testListTables() {
683710
String tableName = "test_list_tables";

0 commit comments

Comments
 (0)








ApplySandwichStrip

pFad - (p)hone/(F)rame/(a)nonymizer/(d)eclutterfier!      Saves Data!


--- a PPN by Garber Painting Akron. With Image Size Reduction included!

Fetched URL: https://www.github.com/googleapis/java-bigquery/commit/d33b64594f9b3217b0fc89d255618b558a9d45e8

Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy