Ever need to just completely clear out an S3 bucket or a DynamoDB table using the AWS SDK? Think it will be one SDK call and then you’re done? Not so! The S3 console has an Empty button for each bucket and the AWS CLI has a aws s3 rm --recursive command. But the console doesn’t lend itself to automation and the CLI command doesn’t work for buckets with object versioning turned on. DynamoDB has similar limitations. One of the recommended methods for DynamoDB is to describe the table, delete the table, and then recreate the table with the exact same properties. This might not always be an option though. So in order to use the AWS SDK for either S3 or DynamoDB, you need to write a bit more code to query all the items/objects and then delete them all in chunks. Since I had to do this recently, hopefully these code snippets will save someone a bit of time.

Since AWS has limits on how many items/objects can be deleted in a single batch, I needed a generic function to take in a slice and process it in chunks of a specified size. So for example, if I have a slice of 1000 items, I can use this function to process 25 items at a time:

// ProcessChunks will break a slice into chunks of the given size and process each chunk sequentially.
func ProcessChunks[T any](s []T, chunkSize int, process func([]T) error) error {
	for i := 0; i < len(s); i += chunkSize {
		upper := i + chunkSize
		if upper > len(s) {
			upper = len(s)
		}
		chunk := s[i:upper]

		err := process(chunk)
		if err != nil {
			return err
		}
	}
	return nil
}

In order to delete everything from an S3 bucket, it is not enough to just delete all objects; you also need to delete all object versions and delete markers. This function will completely empty the specified S3 bucket:

// S3EmptyBucket will completely empty an S3 bucket of all objects, object versions, and delete markers.
func S3EmptyBucket(s3Client *s3.Client, bucketName string) error {
	// iterate over all objects and delete them
	listObjectsInput := &s3.ListObjectsV2Input{
		Bucket: aws.String(bucketName),
	}
	for {
		listObjectsOutput, err := s3Client.ListObjectsV2(context.Background(), listObjectsInput)
		if err != nil {
			return fmt.Errorf("unable to list bucket objects: %w", err)
		}

		objects := []types.ObjectIdentifier{}
		for _, item := range listObjectsOutput.Contents {
			objects = append(objects, types.ObjectIdentifier{
				Key: item.Key,
			})
		}
		err = S3DeleteObjects(s3Client, bucketName, objects)
		if err != nil {
			return err
		}

		if listObjectsOutput.IsTruncated {
			listObjectsInput.ContinuationToken = listObjectsOutput.ContinuationToken
		} else {
			break
		}
	}

	// iterate over all object versions and delete them
	listVersionsInput := &s3.ListObjectVersionsInput{
		Bucket: aws.String(bucketName),
	}
	for {
		listVersionsOutput, err := s3Client.ListObjectVersions(context.Background(), listVersionsInput)
		if err != nil {
			return fmt.Errorf("unable to list object versions: %w", err)
		}

		deleteMarkers := []types.ObjectIdentifier{}
		for _, item := range listVersionsOutput.DeleteMarkers {
			deleteMarkers = append(deleteMarkers, types.ObjectIdentifier{
				Key:       item.Key,
				VersionId: item.VersionId,
			})
		}
		err = S3DeleteObjects(s3Client, bucketName, deleteMarkers)
		if err != nil {
			return err
		}

		versions := []types.ObjectIdentifier{}
		for _, item := range listVersionsOutput.Versions {
			versions = append(versions, types.ObjectIdentifier{
				Key:       item.Key,
				VersionId: item.VersionId,
			})
		}
		err = S3DeleteObjects(s3Client, bucketName, versions)
		if err != nil {
			return err
		}

		if listVersionsOutput.IsTruncated {
			listVersionsInput.VersionIdMarker = listVersionsOutput.NextVersionIdMarker
			listVersionsInput.KeyMarker = listVersionsOutput.NextKeyMarker
		} else {
			break
		}
	}

	return nil
}

// S3DeleteObjects will delete all S3 objects provided, breaking them into chunks of 1000 per request.
func S3DeleteObjects(s3Client *s3.Client, bucketName string, objects []types.ObjectIdentifier) error {
	// s3 DeleteObjects supports batch deleting up to 1000 objects at a time
	return ProcessChunks(objects, 1000, func(toDelete []types.ObjectIdentifier) error {
		out, err := s3Client.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{
			Bucket: aws.String(bucketName),
			Delete: &types.Delete{
				Quiet:   true, // in quiet mode the response includes only keys where the delete action encountered an error
				Objects: toDelete,
			},
		})
		if err != nil {
			return fmt.Errorf("failed to batch delete objects: %w", err)
		}
		if len(out.Errors) > 0 {
			return fmt.Errorf("at least one object failed to delete: %v - %v", out.Errors[0].Code, out.Errors[0].Message)
		}
		return nil
	})
}

In order to efficiently delete all items from a DynamoDB table, you need to use a projection expression to query all items and only return the primary key (partition and sort keys). Then you can send batch delete requests with the primary keys until all items are deleted. This function will delete all items from the specified DynamoDB table:

// DDBEmptyTable will clear all rows from a DynamoDB table.
func DDBEmptyTable(dynamodbClient *dynamodb.Client, tableName string, partitionKey string, sortKey string) error {
	// iterate over all items in table and delete them
	projEx := expression.NamesList(expression.Name(partitionKey), expression.Name(sortKey))
	expr, err := expression.NewBuilder().WithProjection(projEx).Build()
	if err != nil {
		return fmt.Errorf("error building projection expression for deleting dynamodb items: %w", err)
	}
	scanInput := &dynamodb.ScanInput{
		TableName:                 aws.String(tableName),
		ExpressionAttributeNames:  expr.Names(),
		ExpressionAttributeValues: expr.Values(),
		ProjectionExpression:      expr.Projection(),
	}
	for {
		scanOutput, err := dynamodbClient.Scan(context.Background(), scanInput)
		if err != nil {
			return fmt.Errorf("unable to scan table items: %w", err)
		}

		items := []types.WriteRequest{}
		for _, item := range scanOutput.Items {
			items = append(items, types.WriteRequest{
				DeleteRequest: &types.DeleteRequest{
					Key: item, // item properties include only the partition key and sort key
				},
			})
		}

		// dynamodb BatchWriteItem supports batch deleting up to 25 items at a time
		err = ProcessChunks(items, 25, func(toDelete []types.WriteRequest) error {
			out, err := dynamodbClient.BatchWriteItem(context.Background(), &dynamodb.BatchWriteItemInput{
				RequestItems: map[string][]types.WriteRequest{
					tableName: toDelete,
				},
			})
			if err != nil {
				return fmt.Errorf("failed to batch delete objects: %w", err)
			}
			if len(out.UnprocessedItems) > 0 {
				return errors.New("at least one item failed to delete")
			}
			return nil
		})
		if err != nil {
			return err
		}

		if len(scanOutput.LastEvaluatedKey) > 0 {
			scanInput.ExclusiveStartKey = scanOutput.LastEvaluatedKey
		} else {
			break
		}
	}
	return nil
}