org.apache.beam.sdk.io.gcp.bigquery

Class BigQueryStorageTableSource<T>

java.lang.Object

org.apache.beam.sdk.io.Source<T>

org.apache.beam.sdk.io.BoundedSource<T>

org.apache.beam.sdk.io.gcp.bigquery.BigQueryStorageTableSource<T>

All Implemented Interfaces:

java.io.Serializable, org.apache.beam.sdk.transforms.display.HasDisplayData

public class BigQueryStorageTableSource<T>
extends org.apache.beam.sdk.io.BoundedSource<T>

A Source representing reading from a table.

See Also:

Serialized Form

Nested Class Summary

Nested classes/interfaces inherited from class org.apache.beam.sdk.io.BoundedSource

org.apache.beam.sdk.io.BoundedSource.BoundedReader<T>

Nested classes/interfaces inherited from class org.apache.beam.sdk.io.Source

org.apache.beam.sdk.io.Source.Reader<T>

Field Summary

Fields

Modifier and Type	Field and Description
protected BigQueryServices	bqServices
protected @Nullable com.google.cloud.bigquery.storage.v1.DataFormat	format
protected org.apache.beam.sdk.coders.Coder<T>	outputCoder
protected org.apache.beam.sdk.transforms.SerializableFunction<SchemaAndRecord,T>	parseFn
protected @Nullable org.apache.beam.sdk.options.ValueProvider<java.lang.String>	rowRestrictionProvider
protected @Nullable org.apache.beam.sdk.options.ValueProvider<java.util.List<java.lang.String>>	selectedFieldsProvider

Method Summary

Modifier and Type	Method and Description
static <T> BigQueryStorageTableSource<T>	create(org.apache.beam.sdk.options.ValueProvider<com.google.api.services.bigquery.model.TableReference> tableRefProvider, com.google.cloud.bigquery.storage.v1.DataFormat format, @Nullable org.apache.beam.sdk.options.ValueProvider<java.util.List<java.lang.String>> selectedFields, @Nullable org.apache.beam.sdk.options.ValueProvider<java.lang.String> rowRestriction, org.apache.beam.sdk.transforms.SerializableFunction<SchemaAndRecord,T> parseFn, org.apache.beam.sdk.coders.Coder<T> outputCoder, BigQueryServices bqServices, boolean projectionPushdownApplied)
static <T> BigQueryStorageTableSource<T>	create(org.apache.beam.sdk.options.ValueProvider<com.google.api.services.bigquery.model.TableReference> tableRefProvider, @Nullable org.apache.beam.sdk.options.ValueProvider<java.util.List<java.lang.String>> selectedFields, @Nullable org.apache.beam.sdk.options.ValueProvider<java.lang.String> rowRestriction, org.apache.beam.sdk.transforms.SerializableFunction<SchemaAndRecord,T> parseFn, org.apache.beam.sdk.coders.Coder<T> outputCoder, BigQueryServices bqServices)
org.apache.beam.sdk.io.BoundedSource.BoundedReader<T>	createReader(org.apache.beam.sdk.options.PipelineOptions options)
long	getEstimatedSizeBytes(org.apache.beam.sdk.options.PipelineOptions options)
org.apache.beam.sdk.coders.Coder<T>	getOutputCoder()
protected com.google.api.services.bigquery.model.Table	getTargetTable(BigQueryOptions options) Returns the table to read from at split time.
protected java.lang.String	getTargetTableId(BigQueryOptions options)
void	populateDisplayData(org.apache.beam.sdk.transforms.display.DisplayData.Builder builder)
java.util.List<? extends org.apache.beam.sdk.io.BoundedSource<T>>	split(long desiredBundleSizeBytes, org.apache.beam.sdk.options.PipelineOptions options)

Methods inherited from class org.apache.beam.sdk.io.Source

getDefaultOutputCoder, validate

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

format

protected final @Nullable com.google.cloud.bigquery.storage.v1.DataFormat format

selectedFieldsProvider

protected final @Nullable org.apache.beam.sdk.options.ValueProvider<java.util.List<java.lang.String>> selectedFieldsProvider

rowRestrictionProvider

protected final @Nullable org.apache.beam.sdk.options.ValueProvider<java.lang.String> rowRestrictionProvider

parseFn

protected final org.apache.beam.sdk.transforms.SerializableFunction<SchemaAndRecord,T> parseFn

outputCoder

protected final org.apache.beam.sdk.coders.Coder<T> outputCoder

bqServices

protected final BigQueryServices bqServices

Method Detail

create

public static <T> BigQueryStorageTableSource<T> create(org.apache.beam.sdk.options.ValueProvider<com.google.api.services.bigquery.model.TableReference> tableRefProvider,
                                                       com.google.cloud.bigquery.storage.v1.DataFormat format,
                                                       @Nullable org.apache.beam.sdk.options.ValueProvider<java.util.List<java.lang.String>> selectedFields,
                                                       @Nullable org.apache.beam.sdk.options.ValueProvider<java.lang.String> rowRestriction,
                                                       org.apache.beam.sdk.transforms.SerializableFunction<SchemaAndRecord,T> parseFn,
                                                       org.apache.beam.sdk.coders.Coder<T> outputCoder,
                                                       BigQueryServices bqServices,
                                                       boolean projectionPushdownApplied)

create

public static <T> BigQueryStorageTableSource<T> create(org.apache.beam.sdk.options.ValueProvider<com.google.api.services.bigquery.model.TableReference> tableRefProvider,
                                                       @Nullable org.apache.beam.sdk.options.ValueProvider<java.util.List<java.lang.String>> selectedFields,
                                                       @Nullable org.apache.beam.sdk.options.ValueProvider<java.lang.String> rowRestriction,
                                                       org.apache.beam.sdk.transforms.SerializableFunction<SchemaAndRecord,T> parseFn,
                                                       org.apache.beam.sdk.coders.Coder<T> outputCoder,
                                                       BigQueryServices bqServices)

populateDisplayData

public void populateDisplayData(org.apache.beam.sdk.transforms.display.DisplayData.Builder builder)

Specified by:

populateDisplayData in interface org.apache.beam.sdk.transforms.display.HasDisplayData

Overrides:

populateDisplayData in class org.apache.beam.sdk.io.Source<T>

getEstimatedSizeBytes

public long getEstimatedSizeBytes(org.apache.beam.sdk.options.PipelineOptions options)
                           throws java.lang.Exception

Specified by:

getEstimatedSizeBytes in class org.apache.beam.sdk.io.BoundedSource<T>

Throws:

java.lang.Exception

getTargetTableId

protected java.lang.String getTargetTableId(BigQueryOptions options)
                                     throws java.lang.Exception

Throws:

java.lang.Exception

getTargetTable

protected com.google.api.services.bigquery.model.Table getTargetTable(BigQueryOptions options)
                                                               throws java.lang.Exception

Returns the table to read from at split time. This is currently never an anonymous table, but it can be a named table which was created to hold the results of a query.

Throws:

java.lang.Exception

getOutputCoder

public org.apache.beam.sdk.coders.Coder<T> getOutputCoder()

Overrides:

getOutputCoder in class org.apache.beam.sdk.io.Source<T>

split

public java.util.List<? extends org.apache.beam.sdk.io.BoundedSource<T>> split(long desiredBundleSizeBytes,
                                                                               org.apache.beam.sdk.options.PipelineOptions options)
                                                                        throws java.lang.Exception

Specified by:

split in class org.apache.beam.sdk.io.BoundedSource<T>

Throws:

java.lang.Exception

createReader

public org.apache.beam.sdk.io.BoundedSource.BoundedReader<T> createReader(org.apache.beam.sdk.options.PipelineOptions options)
                                                                   throws java.io.IOException

Specified by:

createReader in class org.apache.beam.sdk.io.BoundedSource<T>

Throws:

java.io.IOException