Purge all items from an AWS dynamodb table with an exponential timing back-off

May 21, 2018
aws dynamodb python

Here is some code I wrote to purge all items from an AWS dynamodb table with an exponential timing back-off.

#!/usr/bin/env python3
""" Purge all items from an AWS dynamodb table with an exponential timing back-off
"""
import logging
from time import sleep
import boto3
from botocore.exceptions import ClientError
AWS_RESOURCE = 'dynamodb'
AWS_REGION = 'us-east-1'
def _purge_table(purge_table, key_attribute, pause_time):
"""Purge all items in purge_table"""
batch = purge_table.batch_writer()
response = purge_table.scan()
sleep(pause_time)
for item in response['Items']:
batch.delete_item(
Key={
key_attribute: item[key_attribute]
}
)
while 'LastEvaluatedKey' in response:
response = purge_table.scan(
ExclusiveStartKey=response['LastEvaluatedKey']
)
sleep(pause_time)
for item in response['Items']:
batch.delete_item(
Key={
key_attribute: item[key_attribute]
}
)
def table_purge(purge_table, key_attribute):
""" Purge all items from purge_table
Catch quota exceptions, backoff, and retry as required
"""
retry_exceptions = ('ProvisionedThroughputExceededException',
'ThrottlingException')
retries = 0
pause_time = 0
while True:
try:
_purge_table(purge_table, key_attribute, pause_time)
break
except ClientError as err:
if err.response['Error']['Code'] not in retry_exceptions:
raise
pause_time = (2 ** retries)
logging.info('Back-off set to %d seconds', pause_time)
retries += 1
if __name__ == "__main__":
logging.getLogger("").setLevel(logging.INFO)
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%d/%m/%Y %I:%M:%S %p')
DB = boto3.resource(AWS_RESOURCE, region_name=AWS_REGION)
table_purge(DB.Table('MyTable'), 'KeyAttribute')
logging.info("Processing Complete")
logging.shutdown()
view raw dynamodb_purge_table.py hosted with ❤ by GitHub

Related

Copy an AWS dynamodb table to an existing table with an exponential timing back-off

May 20, 2018
aws dynamodb python

Return the key with the maximum value in a dictionary using python

May 14, 2018
python

Swap characters at positions 'i' and 'j' in string 's' using python

May 13, 2018
python