Package org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn

Class ReadChangeStreamPartitionDoFn

java.lang.Object

org.apache.beam.sdk.transforms.DoFn<PartitionRecord,org.apache.beam.sdk.values.KV<com.google.protobuf.ByteString,com.google.cloud.bigtable.data.v2.models.ChangeStreamRecord>>

org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn.ReadChangeStreamPartitionDoFn

All Implemented Interfaces:

java.io.Serializable, org.apache.beam.sdk.transforms.display.HasDisplayData

@Internal
@UnboundedPerElement
public class ReadChangeStreamPartitionDoFn
extends org.apache.beam.sdk.transforms.DoFn<PartitionRecord,org.apache.beam.sdk.values.KV<com.google.protobuf.ByteString,com.google.cloud.bigtable.data.v2.models.ChangeStreamRecord>>

See Also:

Serialized Form

Nested Class Summary

Nested classes/interfaces inherited from class org.apache.beam.sdk.transforms.DoFn

org.apache.beam.sdk.transforms.DoFn.AlwaysFetched, org.apache.beam.sdk.transforms.DoFn.BoundedPerElement, org.apache.beam.sdk.transforms.DoFn.BundleFinalizer, org.apache.beam.sdk.transforms.DoFn.Element, org.apache.beam.sdk.transforms.DoFn.FieldAccess, org.apache.beam.sdk.transforms.DoFn.FinishBundle, org.apache.beam.sdk.transforms.DoFn.FinishBundleContext, org.apache.beam.sdk.transforms.DoFn.GetInitialRestriction, org.apache.beam.sdk.transforms.DoFn.GetInitialWatermarkEstimatorState, org.apache.beam.sdk.transforms.DoFn.GetRestrictionCoder, org.apache.beam.sdk.transforms.DoFn.GetSize, org.apache.beam.sdk.transforms.DoFn.GetWatermarkEstimatorStateCoder, org.apache.beam.sdk.transforms.DoFn.Key, org.apache.beam.sdk.transforms.DoFn.MultiOutputReceiver, org.apache.beam.sdk.transforms.DoFn.NewTracker, org.apache.beam.sdk.transforms.DoFn.NewWatermarkEstimator, org.apache.beam.sdk.transforms.DoFn.OnTimer, org.apache.beam.sdk.transforms.DoFn.OnTimerContext, org.apache.beam.sdk.transforms.DoFn.OnTimerFamily, org.apache.beam.sdk.transforms.DoFn.OnWindowExpiration, org.apache.beam.sdk.transforms.DoFn.OnWindowExpirationContext, org.apache.beam.sdk.transforms.DoFn.OutputReceiver<T extends java.lang.Object>, org.apache.beam.sdk.transforms.DoFn.ProcessContext, org.apache.beam.sdk.transforms.DoFn.ProcessContinuation, org.apache.beam.sdk.transforms.DoFn.ProcessElement, org.apache.beam.sdk.transforms.DoFn.RequiresStableInput, org.apache.beam.sdk.transforms.DoFn.RequiresTimeSortedInput, org.apache.beam.sdk.transforms.DoFn.Restriction, org.apache.beam.sdk.transforms.DoFn.Setup, org.apache.beam.sdk.transforms.DoFn.SideInput, org.apache.beam.sdk.transforms.DoFn.SplitRestriction, org.apache.beam.sdk.transforms.DoFn.StartBundle, org.apache.beam.sdk.transforms.DoFn.StartBundleContext, org.apache.beam.sdk.transforms.DoFn.StateId, org.apache.beam.sdk.transforms.DoFn.Teardown, org.apache.beam.sdk.transforms.DoFn.TimerFamily, org.apache.beam.sdk.transforms.DoFn.TimerId, org.apache.beam.sdk.transforms.DoFn.Timestamp, org.apache.beam.sdk.transforms.DoFn.TruncateRestriction, org.apache.beam.sdk.transforms.DoFn.UnboundedPerElement, org.apache.beam.sdk.transforms.DoFn.WatermarkEstimatorState, org.apache.beam.sdk.transforms.DoFn.WindowedContext

Constructor Summary

Constructors

Constructor	Description
ReadChangeStreamPartitionDoFn(DaoFactory daoFactory, ActionFactory actionFactory, ChangeStreamMetrics metrics, org.joda.time.Duration backlogReplicationAdjustment)

Method Summary

Modifier and Type	Method	Description
org.joda.time.Instant	getInitialWatermarkEstimatorState(PartitionRecord partitionRecord)
double	getSize(StreamProgress streamProgress)
StreamProgress	initialRestriction()
org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator<org.joda.time.Instant>	newWatermarkEstimator(org.joda.time.Instant watermarkEstimatorState)
org.apache.beam.sdk.transforms.DoFn.ProcessContinuation	processElement(PartitionRecord partitionRecord, org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker<StreamProgress,StreamProgress> tracker, org.apache.beam.sdk.transforms.DoFn.OutputReceiver<org.apache.beam.sdk.values.KV<com.google.protobuf.ByteString,com.google.cloud.bigtable.data.v2.models.ChangeStreamRecord>> receiver, org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator<org.joda.time.Instant> watermarkEstimator)
ReadChangeStreamPartitionProgressTracker	restrictionTracker(StreamProgress restriction)
void	setSizeEstimator(CoderSizeEstimator<org.apache.beam.sdk.values.KV<com.google.protobuf.ByteString,com.google.cloud.bigtable.data.v2.models.ChangeStreamRecord>> sizeEstimator)	Sets the estimator to track throughput for each DoFn instance.
void	setup()

Methods inherited from class org.apache.beam.sdk.transforms.DoFn

getAllowedTimestampSkew, getInputTypeDescriptor, getOutputTypeDescriptor, populateDisplayData, prepareForProcessing

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

ReadChangeStreamPartitionDoFn

public ReadChangeStreamPartitionDoFn(DaoFactory daoFactory,
                                     ActionFactory actionFactory,
                                     ChangeStreamMetrics metrics,
                                     org.joda.time.Duration backlogReplicationAdjustment)

Method Detail

getInitialWatermarkEstimatorState

@GetInitialWatermarkEstimatorState
public org.joda.time.Instant getInitialWatermarkEstimatorState(@Element
                                                               PartitionRecord partitionRecord)

newWatermarkEstimator

@NewWatermarkEstimator
public org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator<org.joda.time.Instant> newWatermarkEstimator(@WatermarkEstimatorState
                                                                                                                           org.joda.time.Instant watermarkEstimatorState)

initialRestriction

@GetInitialRestriction
public StreamProgress initialRestriction()

restrictionTracker

@NewTracker
public ReadChangeStreamPartitionProgressTracker restrictionTracker(@Restriction
                                                                   StreamProgress restriction)

getSize

@GetSize
public double getSize(@Restriction
                      StreamProgress streamProgress)

setup

@Setup
public void setup()
           throws java.io.IOException

Throws:

java.io.IOException

processElement

@ProcessElement
public org.apache.beam.sdk.transforms.DoFn.ProcessContinuation processElement(@Element
                                                                              PartitionRecord partitionRecord,
                                                                              org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker<StreamProgress,StreamProgress> tracker,
                                                                              org.apache.beam.sdk.transforms.DoFn.OutputReceiver<org.apache.beam.sdk.values.KV<com.google.protobuf.ByteString,com.google.cloud.bigtable.data.v2.models.ChangeStreamRecord>> receiver,
                                                                              org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator<org.joda.time.Instant> watermarkEstimator)
                                                                       throws java.lang.InterruptedException,
                                                                              java.io.IOException

Throws:

java.lang.InterruptedException

java.io.IOException

setSizeEstimator

public void setSizeEstimator(CoderSizeEstimator<org.apache.beam.sdk.values.KV<com.google.protobuf.ByteString,com.google.cloud.bigtable.data.v2.models.ChangeStreamRecord>> sizeEstimator)

Sets the estimator to track throughput for each DoFn instance.

Parameters:

sizeEstimator - an estimator to calculate the size of records for throughput estimates