org.apache.flink.runtime.scheduler

Class SchedulerBase

java.lang.Object

org.apache.flink.runtime.scheduler.SchedulerBase

All Implemented Interfaces:

SchedulerNG

Direct Known Subclasses:

DefaultScheduler, LegacyScheduler

public abstract class SchedulerBase
extends Object
implements SchedulerNG

Base class which can be used to implement SchedulerNG.

Field Summary

Fields

Modifier and Type	Field and Description
protected ExecutionVertexVersioner	executionVertexVersioner

Constructor Summary

Constructors

Constructor and Description

SchedulerBase(org.slf4j.Logger log, JobGraph jobGraph, BackPressureStatsTracker backPressureStatsTracker, Executor ioExecutor, org.apache.flink.configuration.Configuration jobMasterConfiguration, SlotProvider slotProvider, ScheduledExecutorService futureExecutor, ClassLoader userCodeLoader, CheckpointRecoveryFactory checkpointRecoveryFactory, org.apache.flink.api.common.time.Time rpcTimeout, RestartStrategyFactory restartStrategyFactory, BlobWriter blobWriter, JobManagerJobMetricGroup jobManagerJobMetricGroup, org.apache.flink.api.common.time.Time slotRequestTimeout, ShuffleMaster<?> shuffleMaster, JobMasterPartitionTracker partitionTracker, ExecutionVertexVersioner executionVertexVersioner, boolean legacyScheduling)

Method Summary

Modifier and Type	Method and Description
void	acknowledgeCheckpoint(org.apache.flink.api.common.JobID jobID, ExecutionAttemptID executionAttemptID, long checkpointId, CheckpointMetrics checkpointMetrics, TaskStateSnapshot checkpointState)
void	cancel()
void	declineCheckpoint(DeclineCheckpoint decline)
protected void	failJob(Throwable cause)
protected ExecutionGraph	getExecutionGraph() Deprecated. Direct access to the execution graph by scheduler implementations is discouraged because currently the execution graph has various features and responsibilities that a scheduler should not be concerned about. The following specialized abstractions to the execution graph and accessors should be preferred over direct access: getSchedulingTopology() getFailoverTopology() getInputsLocationsRetriever() getExecutionVertex(ExecutionVertexID) getExecutionVertexId(ExecutionAttemptID) getExecutionVertexIdOrThrow(ExecutionAttemptID) Currently, only LegacyScheduler requires direct access to the execution graph.
protected ExecutionVertex	getExecutionVertex(ExecutionVertexID executionVertexId)
protected Optional<ExecutionVertexID>	getExecutionVertexId(ExecutionAttemptID executionAttemptId)
protected ExecutionVertexID	getExecutionVertexIdOrThrow(ExecutionAttemptID executionAttemptId)
protected FailoverTopology<?,?>	getFailoverTopology()
protected InputsLocationsRetriever	getInputsLocationsRetriever()
protected JobGraph	getJobGraph()
protected ComponentMainThreadExecutor	getMainThreadExecutor()
protected abstract long	getNumberOfRestarts()
protected ResultPartitionAvailabilityChecker	getResultPartitionAvailabilityChecker()
protected SchedulingTopology<?,?>	getSchedulingTopology()
CompletableFuture<Void>	getTerminationFuture()
void	notifyKvStateRegistered(org.apache.flink.api.common.JobID jobId, JobVertexID jobVertexId, KeyGroupRange keyGroupRange, String registrationName, org.apache.flink.queryablestate.KvStateID kvStateId, InetSocketAddress kvStateServerAddress)
void	notifyKvStateUnregistered(org.apache.flink.api.common.JobID jobId, JobVertexID jobVertexId, KeyGroupRange keyGroupRange, String registrationName)
protected void	prepareExecutionGraphForNgScheduling()
void	registerJobStatusListener(JobStatusListener jobStatusListener)
ArchivedExecutionGraph	requestJob()
JobDetails	requestJobDetails()
org.apache.flink.api.common.JobStatus	requestJobStatus()
KvStateLocation	requestKvStateLocation(org.apache.flink.api.common.JobID jobId, String registrationName)
SerializedInputSplit	requestNextInputSplit(JobVertexID vertexID, ExecutionAttemptID executionAttempt)
Optional<OperatorBackPressureStats>	requestOperatorBackPressureStats(JobVertexID jobVertexId)
ExecutionState	requestPartitionState(IntermediateDataSetID intermediateResultId, ResultPartitionID resultPartitionId)
protected void	resetForNewExecutions(Collection<ExecutionVertexID> vertices)
protected void	restoreState(Set<ExecutionVertexID> vertices)
void	scheduleOrUpdateConsumers(ResultPartitionID partitionId)
protected void	scheduleOrUpdateConsumersInternal(ExecutionVertexID producerVertexId, ResultPartitionID resultPartitionId)
protected void	setGlobalFailureCause(Throwable cause)
void	setMainThreadExecutor(ComponentMainThreadExecutor mainThreadExecutor)
void	startScheduling()
protected abstract void	startSchedulingInternal()
CompletableFuture<String>	stopWithSavepoint(String targetDirectory, boolean advanceToEndOfEventTime)
void	suspend(Throwable cause)
protected void	transitionExecutionGraphState(org.apache.flink.api.common.JobStatus current, org.apache.flink.api.common.JobStatus newState)
protected void	transitionToScheduled(List<ExecutionVertexID> verticesToDeploy)
CompletableFuture<String>	triggerSavepoint(String targetDirectory, boolean cancelJob)
void	updateAccumulators(AccumulatorSnapshot accumulatorSnapshot)
boolean	updateTaskExecutionState(TaskExecutionState taskExecutionState)
protected void	updateTaskExecutionStateInternal(ExecutionVertexID executionVertexId, TaskExecutionState taskExecutionState)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.flink.runtime.scheduler.SchedulerNG

handleGlobalFailure

Field Detail

executionVertexVersioner

protected final ExecutionVertexVersioner executionVertexVersioner

Constructor Detail

SchedulerBase

public SchedulerBase(org.slf4j.Logger log,
                     JobGraph jobGraph,
                     BackPressureStatsTracker backPressureStatsTracker,
                     Executor ioExecutor,
                     org.apache.flink.configuration.Configuration jobMasterConfiguration,
                     SlotProvider slotProvider,
                     ScheduledExecutorService futureExecutor,
                     ClassLoader userCodeLoader,
                     CheckpointRecoveryFactory checkpointRecoveryFactory,
                     org.apache.flink.api.common.time.Time rpcTimeout,
                     RestartStrategyFactory restartStrategyFactory,
                     BlobWriter blobWriter,
                     JobManagerJobMetricGroup jobManagerJobMetricGroup,
                     org.apache.flink.api.common.time.Time slotRequestTimeout,
                     ShuffleMaster<?> shuffleMaster,
                     JobMasterPartitionTracker partitionTracker,
                     ExecutionVertexVersioner executionVertexVersioner,
                     boolean legacyScheduling)
              throws Exception

Throws:

Exception

Method Detail

getExecutionGraph

@Deprecated
protected ExecutionGraph getExecutionGraph()

Deprecated. Direct access to the execution graph by scheduler implementations is discouraged because currently the execution graph has various features and responsibilities that a scheduler should not be concerned about. The following specialized abstractions to the execution graph and accessors should be preferred over direct access:

• getSchedulingTopology()
• getFailoverTopology()
• getInputsLocationsRetriever()
• getExecutionVertex(ExecutionVertexID)
• getExecutionVertexId(ExecutionAttemptID)
• getExecutionVertexIdOrThrow(ExecutionAttemptID)

Currently, only LegacyScheduler requires direct access to the execution graph.

resetForNewExecutions

protected void resetForNewExecutions(Collection<ExecutionVertexID> vertices)

restoreState

protected void restoreState(Set<ExecutionVertexID> vertices)
                     throws Exception

Throws:

Exception

transitionToScheduled

protected void transitionToScheduled(List<ExecutionVertexID> verticesToDeploy)

setGlobalFailureCause

protected void setGlobalFailureCause(@Nullable
                                     Throwable cause)

getMainThreadExecutor

protected ComponentMainThreadExecutor getMainThreadExecutor()

failJob

protected void failJob(Throwable cause)

getFailoverTopology

protected final FailoverTopology<?,?> getFailoverTopology()

getSchedulingTopology

protected final SchedulingTopology<?,?> getSchedulingTopology()

getResultPartitionAvailabilityChecker

protected final ResultPartitionAvailabilityChecker getResultPartitionAvailabilityChecker()

getInputsLocationsRetriever

protected final InputsLocationsRetriever getInputsLocationsRetriever()

prepareExecutionGraphForNgScheduling

protected final void prepareExecutionGraphForNgScheduling()

getExecutionVertexId

protected Optional<ExecutionVertexID> getExecutionVertexId(ExecutionAttemptID executionAttemptId)

getExecutionVertexIdOrThrow

protected ExecutionVertexID getExecutionVertexIdOrThrow(ExecutionAttemptID executionAttemptId)

getExecutionVertex

protected ExecutionVertex getExecutionVertex(ExecutionVertexID executionVertexId)

getJobGraph

protected JobGraph getJobGraph()

getNumberOfRestarts

protected abstract long getNumberOfRestarts()

transitionExecutionGraphState

protected void transitionExecutionGraphState(org.apache.flink.api.common.JobStatus current,
                                             org.apache.flink.api.common.JobStatus newState)

setMainThreadExecutor

public void setMainThreadExecutor(ComponentMainThreadExecutor mainThreadExecutor)

Specified by:

setMainThreadExecutor in interface SchedulerNG

registerJobStatusListener

public void registerJobStatusListener(JobStatusListener jobStatusListener)

Specified by:

registerJobStatusListener in interface SchedulerNG

startScheduling

public final void startScheduling()

Specified by:

startScheduling in interface SchedulerNG

startSchedulingInternal

protected abstract void startSchedulingInternal()

suspend

public void suspend(Throwable cause)

Specified by:

suspend in interface SchedulerNG

cancel

public void cancel()

Specified by:

cancel in interface SchedulerNG

getTerminationFuture

public CompletableFuture<Void> getTerminationFuture()

Specified by:

getTerminationFuture in interface SchedulerNG

updateTaskExecutionState

public final boolean updateTaskExecutionState(TaskExecutionState taskExecutionState)

Specified by:

updateTaskExecutionState in interface SchedulerNG

updateTaskExecutionStateInternal

protected void updateTaskExecutionStateInternal(ExecutionVertexID executionVertexId,
                                                TaskExecutionState taskExecutionState)

requestNextInputSplit

public SerializedInputSplit requestNextInputSplit(JobVertexID vertexID,
                                                  ExecutionAttemptID executionAttempt)
                                           throws IOException

Specified by:

requestNextInputSplit in interface SchedulerNG

Throws:

IOException

requestPartitionState

public ExecutionState requestPartitionState(IntermediateDataSetID intermediateResultId,
                                            ResultPartitionID resultPartitionId)
                                     throws PartitionProducerDisposedException

Specified by:

requestPartitionState in interface SchedulerNG

Throws:

PartitionProducerDisposedException

scheduleOrUpdateConsumers

public final void scheduleOrUpdateConsumers(ResultPartitionID partitionId)

Specified by:

scheduleOrUpdateConsumers in interface SchedulerNG

scheduleOrUpdateConsumersInternal

protected void scheduleOrUpdateConsumersInternal(ExecutionVertexID producerVertexId,
                                                 ResultPartitionID resultPartitionId)

requestJob

public ArchivedExecutionGraph requestJob()

Specified by:

requestJob in interface SchedulerNG

requestJobStatus

public org.apache.flink.api.common.JobStatus requestJobStatus()

Specified by:

requestJobStatus in interface SchedulerNG

requestJobDetails

public JobDetails requestJobDetails()

Specified by:

requestJobDetails in interface SchedulerNG

requestKvStateLocation

public KvStateLocation requestKvStateLocation(org.apache.flink.api.common.JobID jobId,
                                              String registrationName)
                                       throws UnknownKvStateLocation,
                                              FlinkJobNotFoundException

Specified by:

requestKvStateLocation in interface SchedulerNG

Throws:

UnknownKvStateLocation

FlinkJobNotFoundException

notifyKvStateRegistered

public void notifyKvStateRegistered(org.apache.flink.api.common.JobID jobId,
                                    JobVertexID jobVertexId,
                                    KeyGroupRange keyGroupRange,
                                    String registrationName,
                                    org.apache.flink.queryablestate.KvStateID kvStateId,
                                    InetSocketAddress kvStateServerAddress)
                             throws FlinkJobNotFoundException

Specified by:

notifyKvStateRegistered in interface SchedulerNG

Throws:

FlinkJobNotFoundException

notifyKvStateUnregistered

public void notifyKvStateUnregistered(org.apache.flink.api.common.JobID jobId,
                                      JobVertexID jobVertexId,
                                      KeyGroupRange keyGroupRange,
                                      String registrationName)
                               throws FlinkJobNotFoundException

Specified by:

notifyKvStateUnregistered in interface SchedulerNG

Throws:

FlinkJobNotFoundException

updateAccumulators

public void updateAccumulators(AccumulatorSnapshot accumulatorSnapshot)

Specified by:

updateAccumulators in interface SchedulerNG

requestOperatorBackPressureStats

public Optional<OperatorBackPressureStats> requestOperatorBackPressureStats(JobVertexID jobVertexId)
                                                                     throws org.apache.flink.util.FlinkException

Specified by:

requestOperatorBackPressureStats in interface SchedulerNG

Throws:

org.apache.flink.util.FlinkException

triggerSavepoint

public CompletableFuture<String> triggerSavepoint(String targetDirectory,
                                                  boolean cancelJob)

Specified by:

triggerSavepoint in interface SchedulerNG

acknowledgeCheckpoint

public void acknowledgeCheckpoint(org.apache.flink.api.common.JobID jobID,
                                  ExecutionAttemptID executionAttemptID,
                                  long checkpointId,
                                  CheckpointMetrics checkpointMetrics,
                                  TaskStateSnapshot checkpointState)

Specified by:

acknowledgeCheckpoint in interface SchedulerNG

declineCheckpoint

public void declineCheckpoint(DeclineCheckpoint decline)

Specified by:

declineCheckpoint in interface SchedulerNG

stopWithSavepoint

public CompletableFuture<String> stopWithSavepoint(String targetDirectory,
                                                   boolean advanceToEndOfEventTime)

Specified by:

stopWithSavepoint in interface SchedulerNG