com.amazon.titan.diskstorage.dynamodb

Class DynamoDBDelegate

java.lang.Object

com.amazon.titan.diskstorage.dynamodb.DynamoDBDelegate

public class DynamoDBDelegate
extends java.lang.Object

A wrapper on top of the DynamoDB client API that self-throttles using metric-based and context-aware estimates for all Read APIs and for DeleteItem, and that self-throttles using accurate upper-bound estimates of item size for PutItem and UpdateItem. Has a thread pool and is able to do parallel UpdateItem / DeleteItem, Query, and Scan calls.

Author:

Alexander Patrikalakis

Field Summary

Fields

Modifier and Type	Field and Description
static int	BASE_LOGICAL_SIZE_OF_NESTED_TYPES
static java.lang.String	BATCH_WRITE_ITEM
static java.lang.String	CONNECTION_RESET
static java.lang.String	CREATE_TABLE
static java.lang.String	DELETE_ITEM
static java.lang.String	DELETE_TABLE
static java.lang.String	DESCRIBE_TABLE
static java.lang.String	GET_ITEM
static java.lang.String	HASH_RANGE_KEY_SIZE_LIMIT
protected static int	LOGICAL_SIZE_OF_EMPTY_DOCUMENT
static int	MAX_NUMBER_OF_BYTES_FOR_NUMBER
static java.lang.String	MUTATE_ITEM
static int	ONE_KILOBYTE
static int	ONE_KILOBYTE_MINUS_ONE
static java.lang.String	PAGES
static java.lang.String	PUT_ITEM
static java.lang.String	QUERY
static java.lang.String	SCAN
static java.lang.String	SUBSCRIBER_THROUGHPUT_LIMIT
static java.lang.String	TITAN_USER_AGENT
static java.lang.String	UPDATE_ITEM
static java.lang.String	UPDATE_ITEM_SIZE_LIMIT
static java.lang.String	USER_AGENT
static java.lang.String	VALIDATION_EXCEPTION

Constructor Summary

Constructors

Constructor and Description

DynamoDBDelegate(java.lang.String endpoint, AWSCredentialsProvider provider, ClientConfiguration clientConfig, com.thinkaurelius.titan.diskstorage.configuration.Configuration titanConfig, java.util.Map<java.lang.String,com.google.common.util.concurrent.RateLimiter> readRateLimit, java.util.Map<java.lang.String,com.google.common.util.concurrent.RateLimiter> writeRateLimit, long maxRetries, long retryMillis, java.lang.String prefix, java.lang.String metricsPrefix, com.google.common.util.concurrent.RateLimiter controlPlaneRateLimiter)

Method Summary

Methods

Modifier and Type	Method and Description
static boolean	areGSIsSameConfiguration(GlobalSecondaryIndexDescription g1, GlobalSecondaryIndexDescription g2)
static boolean	areGSIsSameConfiguration(java.util.List<GlobalSecondaryIndexDescription> g1, java.util.List<GlobalSecondaryIndexDescription> g2)
BatchWriteItemResult	batchWriteItem(BatchWriteItemRequest batchRequest)
static int	calculateItemSizeInBytes(java.util.Map<java.lang.String,AttributeValue> item)
static int	calculateItemUpdateSizeInBytes(java.util.Map<java.lang.String,AttributeValueUpdate> item)
static AttributeValue	clone(AttributeValue val, java.util.IdentityHashMap<AttributeValue,AttributeValue> sourceDestinationMap) Helper method that can clone an Attribute Value
static java.util.Map<java.lang.String,AttributeValue>	cloneItem(java.util.Map<java.lang.String,AttributeValue> item) Helper method that clones an item
static int	computeWcu(int bytes)
static ScanRequest	copyScanRequest(ScanRequest request)
CreateTableResult	createTable(CreateTableRequest request)
void	createTableAndWaitForActive(CreateTableRequest request)
DeleteItemResult	deleteItem(DeleteItemRequest request)
DeleteTableResult	deleteTable(DeleteTableRequest request)
DeleteTableResult	deleteTable(java.lang.String tableName)
DescribeTableResult	describeTable(DescribeTableRequest request)
TableDescription	describeTable(java.lang.String tableName)
boolean	ensureTableDeleted(java.lang.String tableName)
int	estimateCapacityUnits(java.lang.String apiName, java.lang.String tableName)
com.codahale.metrics.Histogram	getBytesHistogram(java.lang.String apiName, java.lang.String tableName)
com.codahale.metrics.Meter	getConsumedCapacityMeter(java.lang.String apiName, java.lang.String tableName)
com.codahale.metrics.Histogram	getHistogram(java.lang.String apiName, java.lang.String tableName, java.lang.String quantity)
GetItemResult	getItem(GetItemRequest request)
java.lang.String	getItemCountMeterName(java.lang.String apiName, java.lang.String tableName)
java.lang.String	getListTablesApiName()
int	getMaxConcurrentUsers()
long	getMaxRetries()
com.codahale.metrics.Meter	getMeter(java.lang.String meterName)
java.lang.String	getMeterName(java.lang.String apiName, java.lang.String tableName)
com.codahale.metrics.Histogram	getPagesHistogram(java.lang.String apiName, java.lang.String tableName)
ParallelScanner	getParallelScanCompletionService(ScanRequest initialRequest)
com.codahale.metrics.Meter	getQuantityMeter(java.lang.String apiName, java.lang.String tableName, java.lang.String quantity)
java.lang.String	getQuantityName(java.lang.String apiName, java.lang.String tableName, java.lang.String quantity)
long	getRetryMillis()
com.codahale.metrics.Timer	getTimer(java.lang.String apiName, java.lang.String tableName)
com.codahale.metrics.Timer.Context	getTimerContext(java.lang.String apiName, java.lang.String tableName)
void	interruptibleSleep(long millis)
boolean	isEmbedded()
ListTablesResult	listAllTables()
ListTablesResult	listTables(ListTablesRequest request)
void	measureItemCount(java.lang.String apiName, java.lang.String tableName, long itemCount)
void	meterConsumedCapacity(java.lang.String apiName, ConsumedCapacity ccu)
java.util.Map<com.thinkaurelius.titan.diskstorage.StaticBuffer,GetItemResult>	parallelGetItem(java.util.List<GetItemWorker> workers)
void	parallelMutate(java.util.List<MutateWorker> workers)
java.util.List<QueryResultWrapper>	parallelQuery(java.util.List<QueryWorker> queryWorkers)
com.thinkaurelius.titan.diskstorage.BackendException	processDynamoDBAPIException(java.lang.Throwable e, java.lang.String apiName, java.lang.String tableName)
PutItemResult	putItem(PutItemRequest request)
QueryResult	query(QueryRequest request, int permitsToConsume)
com.google.common.util.concurrent.RateLimiter	readRateLimit(java.lang.String tableName)
ScanResult	scan(ScanRequest request, int permitsToConsume)
java.util.concurrent.Future<ScanResult>	scanAsync(ScanRequest request, int permitsToConsume)
void	shutdown()
com.thinkaurelius.titan.diskstorage.BackendException	unwrapExecutionException(java.util.concurrent.ExecutionException e, java.lang.String apiName)
UpdateItemResult	updateItem(UpdateItemRequest request)
void	updatePagesHistogram(java.lang.String apiName, java.lang.String tableName, int pagesProcessed)
boolean	waitForTableCreation(java.lang.String tableName, boolean verifyIndexesList, java.util.List<LocalSecondaryIndexDescription> expectedLsiList, java.util.List<GlobalSecondaryIndexDescription> expectedGsiList)
com.google.common.util.concurrent.RateLimiter	writeRateLimit(java.lang.String tableName)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

PAGES

public static final java.lang.String PAGES

See Also:

Constant Field Values

CREATE_TABLE

public static final java.lang.String CREATE_TABLE

See Also:

Constant Field Values

DELETE_TABLE

public static final java.lang.String DELETE_TABLE

See Also:

Constant Field Values

CONNECTION_RESET

public static final java.lang.String CONNECTION_RESET

See Also:

Constant Field Values

MUTATE_ITEM

public static final java.lang.String MUTATE_ITEM

See Also:

Constant Field Values

SUBSCRIBER_THROUGHPUT_LIMIT

public static final java.lang.String SUBSCRIBER_THROUGHPUT_LIMIT

See Also:

Constant Field Values

HASH_RANGE_KEY_SIZE_LIMIT

public static final java.lang.String HASH_RANGE_KEY_SIZE_LIMIT

See Also:

Constant Field Values

UPDATE_ITEM_SIZE_LIMIT

public static final java.lang.String UPDATE_ITEM_SIZE_LIMIT

See Also:

Constant Field Values

VALIDATION_EXCEPTION

public static final java.lang.String VALIDATION_EXCEPTION

See Also:

Constant Field Values

USER_AGENT

public static final java.lang.String USER_AGENT

See Also:

Constant Field Values

TITAN_USER_AGENT

public static final java.lang.String TITAN_USER_AGENT

See Also:

Constant Field Values

PUT_ITEM

public static final java.lang.String PUT_ITEM

See Also:

Constant Field Values

UPDATE_ITEM

public static final java.lang.String UPDATE_ITEM

See Also:

Constant Field Values

DELETE_ITEM

public static final java.lang.String DELETE_ITEM

See Also:

Constant Field Values

QUERY

public static final java.lang.String QUERY

See Also:

Constant Field Values

BATCH_WRITE_ITEM

public static final java.lang.String BATCH_WRITE_ITEM

See Also:

Constant Field Values

GET_ITEM

public static final java.lang.String GET_ITEM

See Also:

Constant Field Values

DESCRIBE_TABLE

public static final java.lang.String DESCRIBE_TABLE

See Also:

Constant Field Values

SCAN

public static final java.lang.String SCAN

See Also:

Constant Field Values

ONE_KILOBYTE

public static final int ONE_KILOBYTE

See Also:

Constant Field Values

ONE_KILOBYTE_MINUS_ONE

public static final int ONE_KILOBYTE_MINUS_ONE

See Also:

Constant Field Values

BASE_LOGICAL_SIZE_OF_NESTED_TYPES

public static final int BASE_LOGICAL_SIZE_OF_NESTED_TYPES

See Also:

Constant Field Values

LOGICAL_SIZE_OF_EMPTY_DOCUMENT

protected static int LOGICAL_SIZE_OF_EMPTY_DOCUMENT

MAX_NUMBER_OF_BYTES_FOR_NUMBER

public static final int MAX_NUMBER_OF_BYTES_FOR_NUMBER

See Also:

Constant Field Values

Constructor Detail

DynamoDBDelegate

public DynamoDBDelegate(java.lang.String endpoint,
                AWSCredentialsProvider provider,
                ClientConfiguration clientConfig,
                com.thinkaurelius.titan.diskstorage.configuration.Configuration titanConfig,
                java.util.Map<java.lang.String,com.google.common.util.concurrent.RateLimiter> readRateLimit,
                java.util.Map<java.lang.String,com.google.common.util.concurrent.RateLimiter> writeRateLimit,
                long maxRetries,
                long retryMillis,
                java.lang.String prefix,
                java.lang.String metricsPrefix,
                com.google.common.util.concurrent.RateLimiter controlPlaneRateLimiter)

Method Detail

processDynamoDBAPIException

public com.thinkaurelius.titan.diskstorage.BackendException processDynamoDBAPIException(java.lang.Throwable e,
                                                                               java.lang.String apiName,
                                                                               java.lang.String tableName)

scan

public ScanResult scan(ScanRequest request,
              int permitsToConsume)
                throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

getParallelScanCompletionService

public ParallelScanner getParallelScanCompletionService(ScanRequest initialRequest)
                                                 throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

scanAsync

public java.util.concurrent.Future<ScanResult> scanAsync(ScanRequest request,
                                                int permitsToConsume)

copyScanRequest

public static ScanRequest copyScanRequest(ScanRequest request)

parallelMutate

public void parallelMutate(java.util.List<MutateWorker> workers)
                    throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

parallelQuery

public java.util.List<QueryResultWrapper> parallelQuery(java.util.List<QueryWorker> queryWorkers)
                                                 throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

parallelGetItem

public java.util.Map<com.thinkaurelius.titan.diskstorage.StaticBuffer,GetItemResult> parallelGetItem(java.util.List<GetItemWorker> workers)
                                                                                              throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

unwrapExecutionException

public com.thinkaurelius.titan.diskstorage.BackendException unwrapExecutionException(java.util.concurrent.ExecutionException e,
                                                                            java.lang.String apiName)

getItem

public GetItemResult getItem(GetItemRequest request)
                      throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

batchWriteItem

public BatchWriteItemResult batchWriteItem(BatchWriteItemRequest batchRequest)
                                    throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

query

public QueryResult query(QueryRequest request,
                int permitsToConsume)
                  throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

putItem

public PutItemResult putItem(PutItemRequest request)
                      throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

updateItem

public UpdateItemResult updateItem(UpdateItemRequest request)
                            throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

deleteItem

public DeleteItemResult deleteItem(DeleteItemRequest request)
                            throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

estimateCapacityUnits

public int estimateCapacityUnits(java.lang.String apiName,
                        java.lang.String tableName)

readRateLimit

public com.google.common.util.concurrent.RateLimiter readRateLimit(java.lang.String tableName)

writeRateLimit

public com.google.common.util.concurrent.RateLimiter writeRateLimit(java.lang.String tableName)

listTables

public ListTablesResult listTables(ListTablesRequest request)
                            throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

listAllTables

public ListTablesResult listAllTables()
                               throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

describeTable

public TableDescription describeTable(java.lang.String tableName)
                               throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

describeTable

public DescribeTableResult describeTable(DescribeTableRequest request)
                                  throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

deleteTable

public DeleteTableResult deleteTable(DeleteTableRequest request)
                              throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

interruptibleSleep

public void interruptibleSleep(long millis)

ensureTableDeleted

public boolean ensureTableDeleted(java.lang.String tableName)
                           throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

deleteTable

public DeleteTableResult deleteTable(java.lang.String tableName)
                              throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

createTable

public CreateTableResult createTable(CreateTableRequest request)
                              throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

waitForTableCreation

public boolean waitForTableCreation(java.lang.String tableName,
                           boolean verifyIndexesList,
                           java.util.List<LocalSecondaryIndexDescription> expectedLsiList,
                           java.util.List<GlobalSecondaryIndexDescription> expectedGsiList)
                             throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

areGSIsSameConfiguration

public static boolean areGSIsSameConfiguration(java.util.List<GlobalSecondaryIndexDescription> g1,
                               java.util.List<GlobalSecondaryIndexDescription> g2)

areGSIsSameConfiguration

public static boolean areGSIsSameConfiguration(GlobalSecondaryIndexDescription g1,
                               GlobalSecondaryIndexDescription g2)

createTableAndWaitForActive

public void createTableAndWaitForActive(CreateTableRequest request)
                                 throws com.thinkaurelius.titan.diskstorage.BackendException

Throws:

com.thinkaurelius.titan.diskstorage.BackendException

shutdown

public void shutdown()

getTimer

public final com.codahale.metrics.Timer getTimer(java.lang.String apiName,
                                  java.lang.String tableName)

getTimerContext

public final com.codahale.metrics.Timer.Context getTimerContext(java.lang.String apiName,
                                                 java.lang.String tableName)

getMeter

public final com.codahale.metrics.Meter getMeter(java.lang.String meterName)

getItemCountMeterName

public final java.lang.String getItemCountMeterName(java.lang.String apiName,
                                     java.lang.String tableName)

measureItemCount

public final void measureItemCount(java.lang.String apiName,
                    java.lang.String tableName,
                    long itemCount)

meterConsumedCapacity

public final void meterConsumedCapacity(java.lang.String apiName,
                         ConsumedCapacity ccu)

getQuantityName

public final java.lang.String getQuantityName(java.lang.String apiName,
                               java.lang.String tableName,
                               java.lang.String quantity)

getQuantityMeter

public final com.codahale.metrics.Meter getQuantityMeter(java.lang.String apiName,
                                          java.lang.String tableName,
                                          java.lang.String quantity)

getConsumedCapacityMeter

public final com.codahale.metrics.Meter getConsumedCapacityMeter(java.lang.String apiName,
                                                  java.lang.String tableName)

getBytesHistogram

public final com.codahale.metrics.Histogram getBytesHistogram(java.lang.String apiName,
                                               java.lang.String tableName)

getHistogram

public final com.codahale.metrics.Histogram getHistogram(java.lang.String apiName,
                                          java.lang.String tableName,
                                          java.lang.String quantity)

getPagesHistogram

public final com.codahale.metrics.Histogram getPagesHistogram(java.lang.String apiName,
                                               java.lang.String tableName)

updatePagesHistogram

public final void updatePagesHistogram(java.lang.String apiName,
                        java.lang.String tableName,
                        int pagesProcessed)

getMeterName

public final java.lang.String getMeterName(java.lang.String apiName,
                            java.lang.String tableName)

getMaxConcurrentUsers

public final int getMaxConcurrentUsers()

cloneItem

public static java.util.Map<java.lang.String,AttributeValue> cloneItem(java.util.Map<java.lang.String,AttributeValue> item)

Helper method that clones an item

Parameters:

item - the item to clone

Returns:

a clone of item.

clone

public static AttributeValue clone(AttributeValue val,
                   java.util.IdentityHashMap<AttributeValue,AttributeValue> sourceDestinationMap)

Helper method that can clone an Attribute Value

Parameters:

val - the AttributeValue to copy

sourceDestinationMap - used to avoid loops by keeping track of references

Returns:

a copy of val

computeWcu

public static final int computeWcu(int bytes)

calculateItemUpdateSizeInBytes

public static int calculateItemUpdateSizeInBytes(java.util.Map<java.lang.String,AttributeValueUpdate> item)

calculateItemSizeInBytes

public static int calculateItemSizeInBytes(java.util.Map<java.lang.String,AttributeValue> item)

getMaxRetries

public long getMaxRetries()

getRetryMillis

public long getRetryMillis()

isEmbedded

public boolean isEmbedded()

getListTablesApiName

public java.lang.String getListTablesApiName()