Cloudformation

JMES Path

# Get all EC2 instances of a cloudformation stack
aws cloudformation describe-stack-resources --stack-name stack-name --query 'StackResources[?ResourceType==`AWS::EC2::Instance`]'

# Get private IP addresses of all EC2 instances of a cloudformation stack
aws ec2 describe-instances --instance-ids $(aws cloudformation describe-stack-resources --stack-name stack-name --query 'StackResources[?ResourceType==`AWS::EC2::Instance`].PhysicalResourceId' --output text) --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text

Create cloudformation stack with parameter file (note, parameter file has to be json)

aws cloudformation create-stack --stack-name dirks-sample-stack --template-body file://dirks-sample-stack.json --parameters file://dirks-sample-stack-dev.json

# or

aws cloudformation create-stack --stack-name dirks-sample-stack --template-body file://dirks-sample-stack.yaml --parameters file://dirks-sample-stack-dev.json

Show all resources created by cloudformation stacks with dirk in their name

for stack_name in $(aws cloudformation describe-stacks --query "Stacks[?contains(StackName, \`dirk\`)].StackName" --output text); do
  echo "Exporting resources for $stack_name"
  aws cloudformation list-stack-resources --stack-name $stack_name > created_resources/$stack_name
done

Trigger cloudformation stack drift detection for multiple stacks that contain a pattern in their name

for stack_name in $(aws cloudformation describe-stacks --query "Stacks[?contains(StackName, \`pattern\`)].StackName" --output text); do
  echo "Detecting drift for $stack_name"
  aws cloudformation detect-stack-drift --stack-name $stack_name
done

Delete all but the default IAM policy version

for old_version in $(aws iam list-policy-versions --policy-arn $ROLE_ARN --query 'Versions[?IsDefaultVersion==`false`].VersionId' --output text); do
  aws iam delete-policy-version --policy-arn $ROLE_ARN --version-id $old_version
done

Delete all untagged images in an ECR repo

REPO_NAME=$1
UNTAGGED_IMAGES=$( aws ecr list-images --repository-name $REPO_NAME --max-items 100 --filter "tagStatus=UNTAGGED" --query 'imageIds[*]' --output json )

echo $UNTAGGED_IMAGES
aws ecr batch-delete-image --repository-name $REPO_NAME --image-ids "$UNTAGGED_IMAGES"

Delete all tagged images in ECR repo older than maximum date

REPO=$1
MAX_DATE=$2

ALL_IMAGE_TAGS=$(aws ecr describe-images --repository-name $REPO --query "imageDetails[?imagePushedAt<\`$MAX_DATE\`].imageTags[0]" --output text | sed -e 's/[[:space:]^]/ imageTag=/g' -e 's/^/imageTag=/')

while [ -n "$ALL_IMAGE_TAGS" ]; do
   IMAGE_TAGS="$(echo $ALL_IMAGE_TAGS|cut -d ' ' -f1-100)"
   aws ecr batch-delete-image --repository-name $REPO --image-ids $IMAGE_TAGS
   ALL_IMAGE_TAGS="$(echo $ALL_IMAGE_TAGS|cut -d ' ' -f101-)"
done

aws cloudformation describe-stacks --query "Stacks[?contains(StackName, \`pattern\`)].StackName" --output text|tr '\t' '\n'|sort

Cloudwatch

# query log groups
aws logs describe-log-groups --query "logGroups[].logGroupName"

# delete log group
aws logs delete-log-group --log-group-name "log-group-name"

SSM parameter store

# query parameters
aws ssm describe-parameters --query "Parameters[].Name"
aws ssm get-parameters-by-path --path "/your/path/"

# delete parameters
aws ssm delete-parameters --names "parameter1" "parameter2"

Misc

List all records of a zone and export them to CSV

aws route53 list-resource-record-sets --hosted-zone-id <zone-id> | jq '.ResourceRecordSets[] | [.Name, .Type, .ResourceRecords[0].Value] | @csv'

API gateway: delete all API keys

for id in $(aws apigateway get-api-keys --name-query $1 --query "items[].id" --output text); do
  aws apigateway delete-api-key --api-key $id
done

Abort all running step functions

SFN_ARN=$1
COUNTER=1
for execution in `aws stepfunctions list-executions --state-machine-arn $SFN_ARN --query 'executions[?status==\`RUNNING\`].executionArn' --output text`; do
    echo "Execution $COUNTER "
    aws stepfunctions stop-execution --execution-arn $execution &
    COUNTER=$[$COUNTER +1]

    if ! (( $COUNTER % 7)) ; then
      wait
    fi
done

List stack resources for stacks whose name match a particular pattern

stack_pattern=$1
target_directory=$2

mkdir -p $target_directory
rm -rf $target_directory/*.json

for stack_name in $(aws cloudformation describe-stacks --query "Stacks[?contains(StackName, \`$stack_pattern\`)].StackName" --output text); do
  echo "Exporting resources for $stack_name"
  aws cloudformation list-stack-resources --stack-name $stack_name > $target_directory/${stack_name}.json
done

Mass rename S3 prefix keys to be Hive-compliant

This script changes the prefix for all objects in an S3 bucket to be Hive compliant, e. g. from 2021/05/12/foobar to year=2021/month=05/day=12/foobar.

#!/bin/zsh

# Parameters

AWS_PROFILE=$1
BUCKET=$2

for year in {2020..2021}; do
    for month in {01..05}; do
        for day in {01..31}; do
              command="AWS_PROFILE=$AWS_PROFILE aws s3 --recursive mv s3://$BUCKET/$year/$month/$day s3://$BUCKET/year=$year/month=$month/day=$day"
              echo $command
              eval $command
        done
    done
done

AWS CLI Settings

export AWS_CLI_AUTO_PROMPT=on
export AWS_CLI_AUTO_PROMPT=on-partial

DynamoDB

Get the pk of 3000 items that don’t yet have a ttl assigned.

aws dynamodb scan --table-name $TABLE_NAME --projection-expression "pk" --filter-expression 'attribute_not_exists(#ttl)' --expression-attribute-names '{"#ttl": "ttl"}' --max-items 3000 --output text | awk '$1=="pk" {print $2}' > pks