Package org.apache.beam.sdk.io.gcp.bigquery

Class StorageApiLoads<DestinationT,ElementT>

java.lang.Object

org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>>,WriteResult>

org.apache.beam.sdk.io.gcp.bigquery.StorageApiLoads<DestinationT,ElementT>

All Implemented Interfaces:

java.io.Serializable, org.apache.beam.sdk.transforms.display.HasDisplayData

public class StorageApiLoads<DestinationT,ElementT>
extends org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>>,WriteResult>

This PTransform manages loads into BigQuery using the Storage API.

See Also:

Serialized Form

Field Summary

Fields inherited from class org.apache.beam.sdk.transforms.PTransform

annotations, displayData, name, resourceHints

Constructor Summary

Constructors

Constructor

Description

StorageApiLoads(org.apache.beam.sdk.coders.Coder<DestinationT> destinationCoder, org.apache.beam.sdk.io.gcp.bigquery.StorageApiDynamicDestinations<ElementT,DestinationT> dynamicDestinations, org.apache.beam.sdk.transforms.SerializableFunction<ElementT,RowMutationInformation> rowUpdateFn, BigQueryIO.Write.CreateDisposition createDisposition, java.lang.String kmsKey, org.joda.time.Duration triggeringFrequency, BigQueryServices bqServices, int numShards, boolean allowInconsistentWrites, boolean allowAutosharding, boolean autoUpdateSchema, boolean ignoreUnknownValues, boolean propagateSuccessfulStorageApiWrites, java.util.function.Predicate<java.lang.String> propagateSuccessfulStorageApiWritesPredicate, boolean usesCdc, com.google.cloud.bigquery.storage.v1.AppendRowsRequest.MissingValueInterpretation defaultMissingValueInterpretation, java.util.Map<java.lang.String,java.lang.String> bigLakeConfiguration, org.apache.beam.sdk.transforms.errorhandling.BadRecordRouter badRecordRouter, org.apache.beam.sdk.transforms.errorhandling.ErrorHandler<org.apache.beam.sdk.transforms.errorhandling.BadRecord,?> badRecordErrorHandler)

Method Summary

Modifier and Type	Method	Description
WriteResult	expand(org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>> input)
WriteResult	expandInconsistent(org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>> input, org.apache.beam.sdk.coders.Coder<org.apache.beam.sdk.values.KV<DestinationT,StorageApiWritePayload>> successCoder)
WriteResult	expandTriggered(org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>> input, org.apache.beam.sdk.coders.Coder<org.apache.beam.sdk.values.KV<DestinationT,StorageApiWritePayload>> successCoder, org.apache.beam.sdk.coders.Coder<StorageApiWritePayload> payloadCoder)
WriteResult	expandUntriggered(org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>> input, org.apache.beam.sdk.coders.Coder<org.apache.beam.sdk.values.KV<DestinationT,StorageApiWritePayload>> successCoder)
org.apache.beam.sdk.values.TupleTag<BigQueryStorageApiInsertError>	getFailedRowsTag()
boolean	usesErrorHandler()

Methods inherited from class org.apache.beam.sdk.transforms.PTransform

addAnnotation, compose, compose, getAdditionalInputs, getAnnotations, getDefaultOutputCoder, getDefaultOutputCoder, getDefaultOutputCoder, getKindString, getName, getResourceHints, populateDisplayData, setDisplayData, setResourceHints, toString, validate, validate

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Constructor Detail

StorageApiLoads

public StorageApiLoads(org.apache.beam.sdk.coders.Coder<DestinationT> destinationCoder,
                       org.apache.beam.sdk.io.gcp.bigquery.StorageApiDynamicDestinations<ElementT,DestinationT> dynamicDestinations,
                       @Nullable
                       org.apache.beam.sdk.transforms.SerializableFunction<ElementT,RowMutationInformation> rowUpdateFn,
                       BigQueryIO.Write.CreateDisposition createDisposition,
                       java.lang.String kmsKey,
                       org.joda.time.Duration triggeringFrequency,
                       BigQueryServices bqServices,
                       int numShards,
                       boolean allowInconsistentWrites,
                       boolean allowAutosharding,
                       boolean autoUpdateSchema,
                       boolean ignoreUnknownValues,
                       boolean propagateSuccessfulStorageApiWrites,
                       java.util.function.Predicate<java.lang.String> propagateSuccessfulStorageApiWritesPredicate,
                       boolean usesCdc,
                       com.google.cloud.bigquery.storage.v1.AppendRowsRequest.MissingValueInterpretation defaultMissingValueInterpretation,
                       java.util.Map<java.lang.String,java.lang.String> bigLakeConfiguration,
                       org.apache.beam.sdk.transforms.errorhandling.BadRecordRouter badRecordRouter,
                       org.apache.beam.sdk.transforms.errorhandling.ErrorHandler<org.apache.beam.sdk.transforms.errorhandling.BadRecord,?> badRecordErrorHandler)

Method Detail

getFailedRowsTag

public org.apache.beam.sdk.values.TupleTag<BigQueryStorageApiInsertError> getFailedRowsTag()

usesErrorHandler

public boolean usesErrorHandler()

expand

public WriteResult expand(org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>> input)

Specified by:

expand in class org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>>,WriteResult>

expandInconsistent

public WriteResult expandInconsistent(org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>> input,
                                      org.apache.beam.sdk.coders.Coder<org.apache.beam.sdk.values.KV<DestinationT,StorageApiWritePayload>> successCoder)

expandTriggered

public WriteResult expandTriggered(org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>> input,
                                   org.apache.beam.sdk.coders.Coder<org.apache.beam.sdk.values.KV<DestinationT,StorageApiWritePayload>> successCoder,
                                   org.apache.beam.sdk.coders.Coder<StorageApiWritePayload> payloadCoder)

expandUntriggered

public WriteResult expandUntriggered(org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<DestinationT,ElementT>> input,
                                     org.apache.beam.sdk.coders.Coder<org.apache.beam.sdk.values.KV<DestinationT,StorageApiWritePayload>> successCoder)