#!/bin/bash
# Directory containing temporary backup files
BACKUP_DIR="~/temp_backup"

# Format for backup file names (Ex: bk_2025-03-01.tar)
FILE_NAME="bk_$(date +%Y-%m-%d).tar"
FILE_PATH="$BACKUP_DIR/$FILE_NAME"

# S3 Bucket
S3_BUCKET="s3://your-bucket-name"

# PostgreSQL
PG_HOST=localhost
PG_PORT=5432
PG_USERNAME=postgre
PG_PASSWORD=<PGPASSWORD>
DB_NAME=postgres
DB_SCHEMA_NAME=public

# Execute a database backup leveraging Docker and the `pg_dump` utility.
docker run --rm -v "$BACKUP_DIR":/temp_backup --user root postgres bash -c "PGPASSWORD=$PG_PASSWORD pg_dump --verbose --host=$PG_HOST --port=$PG_PORT --username=$PG_USERNAME --format=t --encoding=UTF-8 --file /temp_backup/$FILE_NAME -n $DB_SCHEMA_NAME $DB_NAME"

# Checking the file's successful creation, then updating it in S3.
if [ -f "$FILE_PATH" ]; then
    echo "Uploading to S3..."
    aws s3 cp "$FILE_PATH" "$S3_BUCKET/$FILE_NAME"

    # If uploading file successfully (exit code = 0) then remove local temporary file (optional)
    if [ $? -eq 0 ]; then
        echo "Uploaded file to S3 successfully. Removing local temporary file"
        rm -f "$FILE_PATH"
    else
        echo "Failed to upload backup file to S3"
    fi
else
    echo "Could not find the backup file: $FILE_PATH"
fi

Create a crontab to run daily or at any time you prefer

crontab -e
0 0 * * * /path/to/backup_script.sh >> /var/log/backup_script.log 2>&1
chmod +x /path/to/backup_script.sh

Check the Crontab log

Ubuntu/Debian

grep CRON /var/log/syslog

CentOS/RedHat

grep CRON /var/log/cron

helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo ls

Cloning repository Redis on bitnami to machine

mkdir redis-sentinel
cd ./redis-sentinel
helm fetch bitnami/redis --untar

Make changes to the configurations

Edit values.yaml file

replica.replicaCount: 2
sentinel.enabled: true
sentinel.quorum: 2
sentinel.masterSet: mymaster
global.redis.password: <YOUR_PASSWORD>

Creating a new namespace and installing redis-sentinel

kubectl create namespace redis-sentinel
helm install redis-sentinel ./ -n redis-sentinel
kubectl get pods -n redis-sentinel

How to find a current primary master host

redis-cli -h redis-sentinel -p 26379 -a 'YOUR_PASSWORD' SENTINEL get-master-addr-by-name mymaster

Example response: redis-sentinel-node-0.redis-sentinel-headless.redis-sentinel.svc.cluster.local

The format of that response is:

<pod_name>.<service_name>.<namespace>.svc.cluster.local

Config Redis Commander for accessing Redis Sentinel

apiVersion: apps/v1
kind: Deployment
metadata:
  name: redis-commander
  annotations:
    container.apparmor.security.beta.kubernetes.io/redis-commander: runtime/default
    container.security.alpha.kubernetes.io/redis-commander: runtime/default
  labels:
    app.kubernetes.io/part-of: redis-sentinel
    app.kubernetes.io/name: redis-commander
spec:
  replicas: 1
  selector:
    matchLabels:
      app: redis-commander
  template:
    metadata:
      labels:
        app: redis-commander
        app.kubernetes.io/part-of: redis-sentinel
        app.kubernetes.io/name: redis-commander
    spec:
      automountServiceAccountToken: false
      containers:
        - name: redis-commander
          image: ghcr.io/joeferner/redis-commander
          imagePullPolicy: Always
          env:
            - name: SENTINEL_GROUP
              value: "mymaster"
            - name: SENTINEL_PASSWORD
              value: "sJxpl1HBQZLpNoI"
            - name: REDIS_PASSWORD
              value: "sJxpl1HBQZLpNoI"
            - name: SENTINELS
              value: "redis-sentinel-node-0.redis-sentinel-headless.redis-sentinel.svc.cluster.local:26379,redis-sentinel-node-1.redis-sentinel-headless.redis-sentinel.svc.cluster.local:26379,redis-sentinel-node-2.redis-sentinel-headless.redis-sentinel.svc.cluster.local:26379"
          ports:
            - name: redis-commander
              containerPort: 8081
          resources:
            limits:
              cpu: "500m"
              memory: "512M"
          securityContext:
            runAsNonRoot: true
            readOnlyRootFilesystem: false
            allowPrivilegeEscalation: false
            capabilities:
              drop:
                - ALL

kubectl -n kube-system get configmap kubeadm-config -o jsonpath='{.data.ClusterConfiguration}' --insecure-skip-tls-verify > kubeadm.yaml

Fine-tune the configuration file

apiServer:
  certSANs:
  - "10.10.10.100"
  - "kubernetes.default"
  - "new-hostname"
  - "X.X.X.X" #newIPaddress
  extraArgs:
    ...

Re-create API Server Certificates

mv /etc/kubernetes/pki/apiserver.{crt,key} ~
kubeadm init phase certs apiserver --config kubeadm.yaml

Benefits of AWS CloudFormation

• Infrastructure as code

• Cost

  • Each resource in the stack is tagged with an identifier so you can easily see how much a stack costs

  • Estimate the costs of resources using the CloudFormation template

  • Saving strategy: In Dev, you could automate the deletion of templates at 5 PM and recreate them at 8 A, safely

• Productivity

  • Automated generations of Diagram for template

• Do not reinvent the wheel

  • Leverage existing templates on the web

  • Leverage the documentation

• Supports (almost) all AWS resources

Service Role

• Use cases:

  • You want to achieve the least privilege principle

  • But you do not want to give the user all the required permissions to create the stack resources

• User must have iam:PassRole permissions

AWS SES

• Fully managed service to send emails securely, globally, and at scale

• Allow inbound/outbound emails

• Reputation dashboard, performance insights, anti-spam feedback

• Use cases: transactional, marketing, and bulk email communications

Amazon Pinpoint

• Supports email, SMS, push, voice, and in-app messaging

• Possibility of receiving replies

• Scales to billions of messages per day

• Use cases: run campaigns by sending marketing, bulk, and transactional SMS messages

• Versus SNS or SES

  • In SNS & SES you managed each message’s audience, content, and delivery schedule.

  • In Pinpoint, you create message templates, delivery schedules, highly-targeted segments, and full campaigns.s

System Manager - SSM Session Manager

• Allows to start a secure shell on EC2 and on-premises servers

• No SSH access, bastion hosts, or SSH Keys needed

• There is no need to open port 22

• Supports Linux, MacOS, Windows

• Send session log data to S3 or CloudWatch Logs

System Manager

Run command

• Execute a script or just run a command

• Run command across multiple instances (using resource groups)

• No need for SSH

• Command output can be shown in the AWS Console, and sent to the S3 bucket or CloudWatch Lo.gs.

• Send notifications to SNS about command status

• Integrated with IAM & CloudTrail

• It can be invoked using EventBridge

Patch Manager

• Automates the process of patching managed instances

• OS updates, application updates, security updates

• Supports EC2 instances and on-premises servers

• Supports Linux, MacOS, Windows

• Patch on-demand or on a schedule using Maintenance Windows

• Scan instances and generate patch compliance reports (missing patches)

Maintenance Windows

• Defines a schedule for when to perform actions on instances

• Example: OS patching, updating drivers, installing software,…

Automation

• Simplifies common maintenance and deployment tasks of EC2 instances and other AWS resources

• Ex: restart instances, create an AMI, EBS Snapshot,…

• Automation Runbook - SSM Documents to define actions pre-formed on EC2 instances or AWS Resources

• Can be triggered using:

  • Manually using AWS Console, AWS CLI, SDK

  • EventBridge

  • On a schedule using Maintenance Windows

  • By AWS Config for rules remediations

AWS Outposts

• Benefits:

  • Low-latency access to on-premises systems

  • Local data processing

  • Data residency

  • Easier migration from on-premises to the cloud

  • Fully managed service

• Some service that work on Outposts: EC2, EBS, S3, EKS, ECS, RDS, EMR

AWS Batch

• Fully managed batch processing at any scale

• Efficiently run 100,000s of computing batch jobs on AWS

• A “batch” job is a job with a start and an end

• Batch will dynamically launch EC2 or Spot Instances

• Batch jobs are defined as Docker images and run on ECS

• Helpful for cost optimizations and focusing less on the infrastructure

Amazon AppFlow

• Fully managed integration service that enables secure transfer of data between SaaS applications and AWS

• Sources: Salesforce, SAP, ServiceNow

• Destinations: S3, Redshift,…

• Frequency: on a schedule, in response to events, on-demand

AWS Amplify

• A set of tools and services that helps develop and deploy scalable full-stack web and mobile applications

Instance Scheduler on AWS

• Automatically start/stop AWS services

• Supports cross-account and cross-region resources

• Schedules are managed in a DynamoDB table

• Supports EC2, EC2 Auto Scaling Groups, and RDS instances

Data Management & Transfer

• AWS Direct Connect: Move Gb/s of data to the cloud, over a private secure network

• Snowball & Snowmobile: Move PB of data to the cloud

• AWS DataSync: Move large amounts of data between on-premise and S3, EFS, and FSx for Windows

Compute and Networking

• EC2 Instances:

  • CPU optimized, GPU optimized

  • Spot Instances, Spot Fleets for cost savings + Auto Scaling

• EC2 Placement Groups: Cluster for good network performance

Networking

• EC2 Enhanced Networking (SR-IOV)

  • Higher bandwidth, higher PPS (packet per second), lower latency

  • Option 1: Elastic Network Adapter (EFA) up to 100 Gbps

  • Option 2: Intel 82599 VF up to 10 Gbps - LEGACY

• Elastic Fabric Adapter (EFA)

  • Improved ENA for HPC, only works for Linux

  • Great for inter-node communications, tightly coupled workloads

  • Leverages Message Passing Interface standard

  • Bypasses the underlying Linux OS to provide low-latency, reliable transport

Storage

• Instance-attached storage:

  • EBS: scale up to 256,000 IOPS with io2 Block Express

  • Instance Store: scale to millions of IOPS, linked to EC2 instance, low latency

• Network storage:

  • S3: large blob, not a file system

  • EFS: scale IOPS based on total size, or use provisioned IOPS

  • FSx for Lustre:

    • HPC-optimized distributed file system, millions of IOPS

    • Backed by S3

Automation and Orchestration

• AWS Batch

  • AWS Batch supports multi-node parallel jobs, which enables the running of single jobs that span multiple EC2 instances

  • Easily schedule jobs and launch EC2 instances accordingly

• AWS ParallelCluster

  • Open-source cluster management tool to deploy HPC on AWS

  • Configure with text files

  • Automate the creation of VPC, Subnet, Cluster type, and instance types

  • Ability to enable EFA on the cluster (improves network performance)

There are different kinds of Disaster Recovery

• On-premise → On-premise: traditional DR, and very expensive

• On-premise → AWS Cloud: hybrid recovery

• AWS Cloud Region A → AWS Cloud Region B

RPO and RTO

RPO

• How much of a data loss

RTO

• The amount of downtime of the application

Pilot Light

• A small version of the app is always running in the cloud

• Useful for the critical core (pilot light)

• Very similar to Backup and Restore

• Faster than Backup and Restore

Warm Standby

• The full system is up and running but at a minimum size

• Upon disaster, we can scale to production load

Multi-Site / Hot Site Approach

• Very low RTO (minutes or seconds) - very expensive

• Full Production Scale is running AWS and On-Premise

Disaster Recovery Tips

• Backup

  • EBS Snapshots, RDS Automated backups / Snapshots,…

  • Regular pushes to S3/S3 IA/Glacier, LifeCycle Policy, Cross Region Replication

  • From On-Premise: Snowball or Storage Gateway

• HA

  • Use Route53 to migrate DNS over from Region to Region

  • RDS Multi-AZ, ElasticCache Multi-AZ, EFS, S3

  • Site to Site VPN as a recovery from Direct Connect

• Replication

  • RDS Replication, AWS Aurora + Global Database

  • Database replication from on-premise to RDS

  • Storage Gateway

• Automation

  • CloudFormation / Elastic Beanstalk to re-create a whole new environment

  • Recover / Reboot EC2 instances with CloudWatch if alarms fail

  • AWS Lambda functions for customized automation

• Chaos

  • Netflix has a “simian-army” randomly terminating EC2

Database Migration Service

• Supports:

  • Homogeneous migrations: Oracle to Oracle

  • Heterogeneous: SQL Server to Aurora

• Continuous Data Replication using the CDC

• Must create an EC2 instance to perform the replication tasks

AWS Schema Conversion Tool

• Convert Database’s Schema from one engine to another

• You do not need to use SCT if you are migrating the same DB engine

RDS & Aurora Migrations

Migrate to MySQL Aurora

• RDS MySQL to Aurora MySQL

  • Option 1: DB Snapshots from RDS MySQL restored as MySQL AuroraDB

  • Options 2: Create an Aurora Read Replica from RDS MySQL, and when the replication lag is 0, promote it as its DB cluster (can take time and cost)

• External MySQL to Aurora MySQL

  • Option 1:

    • Use Percona Xtrabackup to create a file backup in S3

    • Create an Aurora MySQL DB from S3

  • Option 2:

    • Create an Aurora MySQL DB

    • Use the mysqldump utility to migrate MySQL into Aurora (slower than the S3 method)

• Use DMS if both databases are up and running

Migrate to PostgreSQL Aurora

• RDS PostgreSQL to Aurora PostgreSQL

  • Option 1: DB Snapshots from RDS PostgreSQLrestored as PostgreSQL AuroraDB

  • Options 2: Create an Aurora Read Replica from RDS PostgreSQL, and when the replication lag is 0, promote it as its DB cluster (can take time and cost)

• External PostgreSQL to Aurora PostgreSQL

  • Create a backup and put it in S3

  • Import it using the aws_s3 Aurora extension

• Use DMS if both databases are up and running

AWS Backup

• Fully managed services

• Supported services:

  • EC2 / EBS

  • S3

  • RDS / Aurora / DynamoDB

  • DocumentDB / Neptune

  • EFS / FSx (Lustre & Windows File Server)

  • AWS Storage Gateway

• Supports cross-region backups

• Supports cross-account backups

AWS Backup Vault Lock

• Enforce a WORM (Write Once Read Many) state for all the backups that are stored in AWS Backup Vault

• Even the root user cannot delete backups when enabled

AWS Application Discovery Service

• Plan migration projects by gathering information about on-premises data centers

• Server utilization data and dependency mapping are important for migrations

• Agentless Discovery: VM inventory, configuration, and performance history such as CPU, memory, and disk usage

• Agent-based Discovery: System configuration, system performance, running processes, and details of the network connections between systems

• The resulting data can be viewed in the AWS Migration Hub

Transferring large amounts of data to AWS

Example: transfer 200 TB of data in the cloud. We have a 100 Mbps internet connection

• Snowball

  • Will take 2 to 3 snowballs in parallel

  • Takes about 1 week for the end-to-end transfer

  • Can be combined with DMS

• Direct Connect 1 Gbps

  • Long for the one-time setup (over a month)

  • Will take 200(TB) * 1000(GB) * 8(MB)/1 Gbps = 1,600,000s = 18.5d

• The Internet / Site-to-Site VPN

  • Immediate set up

  • Will take 200(TB) \ 1000(GB) \ 1000(MB) * 8(MB)/1 Gbps = 185d

• Private IP

  • 10.0.0.0 - 10.255.255.255 (10.0.0.0/8)

  • 172.16.0.0 - 172.31.255.255 (172.16.0.0/12) ← AWS default VPC in that range

  • 192.168.0.0 - 192.168.255.255 (192.168.0.0/16) ← home networks

• Public IP: All the rest of the IP addresses

Subnet

• AWS reserves 5 IP addressed (first 4 and last 1) in each subnet

  • Example: if CIDR blocks 10.0.0.0/24, then reserved IP addresses are:

    • 10.0.0.0 - Network Address

    • 10.0.0.1 - reserved by AWS for the VPC router

    • 10.0.0.2 - mapping to Amazon-provided DNS

    • 10.0.03 - future use

    • 10.0.0.255 - Network Broadcast Address

NAT Instance

• Allows EC2 Instances in a private subnet to connect to the Internet

• Must be launched in a public subnet

• Must disable EC2 Setting: Source/destination Check

• Must have Elastic IP attached to it

• Route Tables must be configured to route traffic from Private Subnets to the NAT Instance

Comments

• Pre-configured Amazon Linux AMI is available

  • Reached the end of standard support on 31/12/2020

• Not HA / Resilient setup out of the box

  • It would help if you created an ASG in multi-AZ + resilient user-data script

• Internet traffic bandwidth depends on EC2 Instance Type

• You must manage Security Groups & Rules:

  • Inbound

    • Allow HTTP/HTTPS traffic coming from Private Subnets

    • Allow SSH from your home network

  • Outbound

    • Allow HTTP/HTTPS traffic to the Internet

NAT Gateway

• AWS-managed NAT, higher bandwidth, HA, no administration

• Pay per hour for usage and bandwidth

• NATGW is created in a specific AZ, uses an Elastic IP

• Cannot be used by EC2 instance in the same subnet

• Requires an IGW (Private Subnet → NATGW → IGW)

• 5 Gbps of bandwidth with automatic scaling up to 100 Gbps

• No Security Groups to manage/required

NAT Gateway with HA

• NAT Gateway is resilient within a single AZ

• Must create multiple NAT Gateways in multiple AZs for fault-tolerance

VPC Peering

• You can create VPC Peeing connections between VPCs in different AWS accounts/regions

• You can reference a security group in a peered VPC (works cross accounts - same region)

VPC Endpoints (AWS PrivateLink)

• Every AWS service is publicly exposed (public URL)

• VPC Endpoints (powered by AWS PrivateLink) allows you to connect to AWS services using a private network instead of using the public internet

Types of Endpoints

• Interface Endpoints (powered by PrivateLink)

  • Provisions an ENI (private IP address) as an entry point (must attach a Security Group)

  • Supports most AWS Services

  • $ per hour + $ per GB of data processed

• Gateway Endpoints

  • Provisions a gateway and must be used as a target in a route table (does not use security group)

  • Supports both S3 and DynamoDB

  • Free

AWS Site-to-Site VPN

• Virtual Private Gateway (VGW)

  • VPN concentrator on the AWS side of the VPN connection

  • VGW is created and attached to the VPC from which you want to create the Site-to-Site VPN connection

  • Possibility to customize the ASN (Autonomous System Number)

• Customer Gateway (CGW)

  • A software application or physical device on the customer side of the VPN connection

Enable Route Propagation for the VGW in the route table that is accociated with subnets

AWS VPN CloudHub

• Provide secure communication between multiple sites, if you have multiple VPN connections

• Low-cost hub-and-spoke model for primary or secondary network connectivity between different locations (VPN only)

• It’s a VPN connection so it goes over the public internet

• To set it up, connect multiple VPN connections on the same VGW, set dynamic routing, and configure route tables

Direct Connect (DX)

• Provides a dedicated private connection from a remote network to VPC

• Supports both IPv4 and IPv6

• Use Cases:

  • Increase bandwidth throughput

  • More consistent network experience

  • Hybrid Environments (on-prem + cloud)

Connection Types

• Dedicated Connections: 1 Gbps, 10 Gbps and 100 Gbps capacity

• Hosted Connections: 50 Mbps, 500 Mbps, to 10 Gbps

• Lead times are often longer than 1 month to establish a new connection

Encryption

• Data in transit is not encrypted but is private

• AWS Direct Connect + VPN provides an IPsec-encrypted private connection

• Good for an extra level of security

In case Direct Connect fails, you can set up a backup Direct Connect connection (expensive), or a Site-to-Site VPN connectio

Transit Gateway

• For having transitive peering between thousands of VPC and on-prem connection

• Regional resources can work cross-region

• Supports IP Multicast

Traffic Mirroring

• Allows to capture and inspect network traffic in VPC

• Route the traffic to security appliances

• Capture the traffic

  • From (Source)

  • To (Targets)

Egress Only Internet Gateway

• Used for IPv6 only (similar to a NAT Gateway but for IPv6)

• Must update the Route Tables

• Manages encryption keys

• Able to audit KMS usage using CloudTrail

• Having three kinds of Keys:

  1. KMS Keys

  2. Symmetric

  3. Asymmetric

KMS Keys

Types of KMS Keys:

• AWS Owned Keys (free): SSE-S3, SSE-SQS,…

• AWS Managed Keys (free): aws/service-name, ex: aws/rds

• Customer-managed keys created in KMS: $1 per month

• Customer-managed keys imported: $1 per month

  • pay for the API call to KMS ($0.03/10.000 calls)

Automatic Key rotation:

• AWS Managed Keys: automatic every 1 year

• Customer-managed keys: (must be enabled) automatic & on-demand

• Imported KMS key: only manual rotation possible using alias

KMS Multi-Region Keys

• Identical KMS Keys in different regions that can be used interchangeably (you can encrypt in one Region and decrypt in Other Regions)

• Multi-region keys have the same key ID, key material, and automatic rotation….

• KMS multi regions are NOT global (Primary + Replicas)

• Use cases: global client-side encryption, encryption on Global DynamoDB, Global Aurora

S3 Replication Encryption Considerations

• Unencrypted objects and objects encrypted with SSE-S3 are replicated by default.

• Objects encrypted with SSE-C (Customer Key) can be replicated

• For objects encrypted with SSE-KMS, you need to enable the option

  • Specify which KMS key to encrypt the objects

  • Adapt the KMS Key Policy for the target key

  • An IAM Role with kms:Decrypt for the source KMS Key and km:Encrypt for the target KMS Key

  • You might get KMS throttling errors, so you can request a Service Quotas increase.

AMI Sharing Process Encrypted via KMS

1. Must modify the image attribute to add a Launch Permission

2. Must share the KMS Keys

3. The IAM Role/User in the target account must have permission to DescribeKey, ReEncrypted, CreateGrant, and Decrypt.

4. When launching an EC2 Instance from the AMI, the target account can optionally specify a new KMS Key to re-encrypt the volumes.

SSM Parameter Store

• Secure storage for configuration and secrets

• Optional Seamless Encryption using KMS

• Serverless, scalable, durable, easy SDK

• Version tracking

• Security through IAM

• Notifications with EventBridge

• Integration with CloudFormation

Parameters Policies

• Allow to assign a TTL to a parameter to force updating or deleting

• Can assign multiple policies at a time

AWS Secrets Manager

Overview

• Capability to force rotation of secrets every X days

• Integration with RDS (MySQL, PostgreSQL, Aurora)

• Secrets are encrypted using KMS

• Mostly meant for RDS integration

Multi-region Secrets

• Secrets Manager keeps reading replicas in sync with the primary Secret

• Ability to promote a read replica Secret to a standalone Secret

• Use cases: multi-region apps, disaster recovery, multi-region DB,…

AWS Certificate Manager (ACM)

• Easily provision, manage, and deploy TLS Certificates

• Provision in-flight encryption to for websites (HTTPS)

• Supports both public and private TLS Certificates

• Free of charge for public TLS Certificates

• Automatic TLS certificate renewal

• Integration with (load TLS certificates on)

  • Elastic Load Balancers

  • CloudFront Distributions

  • APIs on API Gateway

• Cannot use ACM with EC2

Requesting Public Certificates

1. List domain names to be included in the certificate

   • Fully Qualified Domain Name (FQDN): corp.example.com

   • Wildcard Domain: *.example.com

2. Select Validation Method: DNS Validation or Email Validation

   • DNS Validation is preferred for automation purposes

   • Email validation will send emails to contact addressed in the WHOIS database

   • DNS Validation will leverage a CNAME record to DNS config

3. It will take a few hours to get verified

4. The Public Cerfificated will be enrolled for automatic renewal

   • ACM automatically renews ACM-generated certificates 60 days before expiry

Importing Public Certificates

• No automatic renewal, must import a new certificate before expiry

• ACM sends daily expiration events starting 45 days prior to expiration

  • The # of days can be configured

  • Events are appearing in EventBridge

• AWS Config has a managed rule named acm-certificate-expiration-check to check for expiring certificates

Integration with API Gateway

• Create a Custom Domain Name in API Gateway

• Edge-Optimized (default): For global clients

  • Requests are routed through the CloudFront Edge locations (improves latency)

  • The API Gateway still lives in only one region

  • The TLS Certificate must be in the same region as CloudFront

  • Then setup CNAME or (better) A-Alias record in Route53

• Regional

  • For clients in the same region

  • The TLS Certificates must be imported on API Gateway, in the same region as the API Stage

  • Then setup CNAME or (better) A-Alias record in Route53

Web Application Firewall (WAF)

• Protects web application from common web exploit (Layer 7)

• Layer 7 is HTTP (vs Layer 4 is TCP/UDP - WAF does not support this layer)

• Deploy on

  • Application Load Balancer

  • API Gateway

  • CloudFront

  • Cognito User Pool

  • AppSync GraphSQL API

• Define Web ACL Rules: IP Sets, Rate-based rules,…

• Web ACL are Regional except for CloudFront

• A rule group is a reusable set of rules that you can add to a web ACL

Fixed IP while using WAF with a LB

• WAF does not support the Network Load Balancer (Layer 4)

• We can use Global Accelerator which provides fixed IPv4 for fixed IP and WAF on the ALB

Shield - DDOS Protection

• AWS Shield Standard

  • Free service that is active for every AWS customer

  • Provides protection form attacks such as SYN/UDP Floods, Refection attacks and other layer 3/payer 4 attacks

• AWS Shield Advanced

  • Optional DDoS mitigation service ($3000 per month per organization)

  • Protect against more sophisticated attack on EC2, ELB, CloudFront, AWS Global Accelerator, Route53

  • 24/7 access to AWS DDoS response team

Firewall Manager

• Manage rules in all accounts of an AWS Organization

• Security policy: common set of security rules

  • WAF rules

  • AWS Shield Advanced

  • Security Group for EC2, ALB

  • AWS Network Firewall (VPC Level)

  • Route53 Resolver DNS Firewall

  • Policies are created at the region level

• Rules are applied to new resources as they are created across all and future accounts in Organization

Amazon GuardDuty

• Intelligent Threat discovery to protect AWS Account

• Uses ML algorithms, anomaly detection, 3rd party data

• Input data includes: CloudTrail Events Logs, VPC Flow Logs, DNS Logs,…

• Can be setup EventBridgu rules to be notified in case of findings

• Can protect against CryptoCurrency attacks

Amazon Inspector

• Automated Security Assessments

• Used for EC2, Container Images, Lambda

• Reporting & Integration with AWS Security Hub

• Send findings to EventBridge

Amazon Macie

• Helps identify and alert to sensitive data such as personally identifiable information (PII)

network:
  ethernets:
    enp0s8:
      dhcp4: no
      addresses: [192.168.202.10/24]
  version: 2

Apply the new changes

sudo netplan try

Metrics

• Provides metrics for every service in AWS

• Metrics belong to namespaces

• Dimension is an attribute of a metric (instance id, environment,…)

• Up to 30 dimensions per metric

• Metrics have timestamps

• Can create CloudWatch dashboards of metrics

• Can create Custom Metrics

Metric Streams

• Continually stream CloudWatch metrics to a destination, with near-real-time delivery and low latency

  • Amazon Kinesis Data Firehose

  • 3rd party service provider: Datadog, Sumo Logic,…

Logs

• Log groups: arbitrary name, usually representing an application

• Can define log expiration policies (never expire, 1 day to 10 years,…)

• CloudWatch Logs can send logs to:

  • S3 (export)

  • Kinesis Data Streams/Firehose

  • Lambda

  • OpenSearch

• Logs are encrypted by default

• Can set KMS-based encryption with own keys

Logs - Sources

• SDK, CloudWatch Logs Agent, CloudWatch Unified Agent

• Elastic Beanstalk, ECS, Lambda, VPC Flow Logs, API Gateway, CloudTrail, Route53

Logs Insights

• Search and analyze log data stored in CloudWatch Logs

• Example: find a specific IP inside a log,…

• Provides a purpose-built query language

  • Automatically discovers fields from AWS Services and JSON log events

  • Can save queries and add them to CloudWatch Dashboards

• Can query multiple Log Groups in different AWS Accounts

• It’s a query engine, not a real-time engine

S3 Export

• Log data can take up to 12 hours to become available for export

• The API call is CreateExportTask

• Not near-real-time or real-time,… use Logs Subscription instead

Logs Subscriptions

• Get real-time log events from CloudWatch Logs for processing and analysis

• Send to Kinesis Data Streams/Firehose or Lambda

• Subscription Filter

CloudWatch Unified Agent

• Collected Linux server / EC2 instance directly

• CPU

• Disk metrics

• RAM

• Netstat

• Processes

• Swap Space

• Reminder: out-of-the-box metrics for EC2 - disk, CPU, network (high level)

CloudWatch Alarm

• Used to trigger notifications for any metric

• Various options (sampling, %, max, min, etc,…)

Alarm States:

• OK

• INSUFFICIENT_DATA

• ALARM

Alarm Target

• Stop, Terminate, Reboot or Recover an EC2 Instance

• Trigger Auto Scaling Action

• Send notification to SNS

Composite Alarms

• CloudWatch Alarms are on a single metric

• Composite Alarms monitor the states of multiple other alarms

• AND and OR conditions

Good to know

• Alarms can be created based on CloudWatch Logs Metrics Filters

• To test alarms and notifications, set the alarm state to Alarm using CLI

•   aws cloudwatch set-alarm-state --alarm-name "myalarm" --state-value ALARM --state-reason "testing purpose"
  

EventBridge (formerly CloudWatch Events)

• Schedule: Cron jobs

• Event pattern: event rules to react to a service doing something

• Trigger Lambda function, send SQS/SNQ messages,…

CloudWatch Container Insights

• Collect, aggregate, and summarize metrics and logs from containers

• Available for containers on:

  • ECS

  • EKS

  • K8S on EC2

  • Fargate

CloudWatch Lambda Insights

• Collects, aggregates, and summaries system-level metrics including CPU time, memory, disk, and network

• Lambda insights are provided by the Lambda layer

CloudWatch Contributor Insights

• Analyze log data and create a time series that displays contributor data

  • See metrics about the top-N contributors

  • The total number of unique contributors and their usage

CloudWatch Application Insights

• Provides automated dashboards that show potential problems with monitored applications, to help isolate ongoing issues

• Powered by SageMaker

• Enhance visibility into application health to reduce the time it will take to troubleshoot and repair application

• Findings and alerts are sent to EvenBridge and SSM OpsCenter

CloudTrail

• CloudTrail is enabled by default

• It provides governance, compliance, and audit for AWS account

• Get a history of events / API calls made within the AWS account

  • SDK

  • Console

  • CLI

  • AWS Services

• Can put logs from CloudTrail into S3 or CloudWatch Logs

• A Trail can be applied to All Regions (default) or Single Region

• If a resource is deleted in AWS, investigate CloudTrail first

CloudTrail Events

• Management Events

• Data Events

• CloudTrail Insights Events

  • To detect unusual activity in an AWS account

  • CloudTrail Insights analyses normal management events to create a baseline

  • Then, it continuously analyzes written events to detect unusual activity

CloudTrail Events Retention

• Events are stored for 90 days in CloudTrail

• To keep events beyond this period, log them to S3 and use Athena

AWS Config

• Helps with auditing and recording compliance of AWS resources

• Helps record configurations and changes over time

• Questions that can be solved by AWS Config

  • Is there unrestricted SSH access to my security groups?

  • Do my buckets have any public access?

• You can receive alerts (SNS) for any changes

• AWS Config is a per-region service

• Possibility of storing the configuration data in S3

Config Rules

• Can use AWS managed config rules (over 75)

• Can make custom configuration rules (must be defined in AWS Lambda)

• Rules can be evaluated/triggered:

  • For each configuration change

  • And/or: at regular time intervals

• AWS Config Rules does not prevent actions from happening

Summary

• CloudWatch

  • Performance monitoring & dashboard

  • Events & Alerting

  • Log Aggregation & Analysis

• CloudTrail

  • Record API calls made in the AWS Account by everyone

  • Can define trails for specific resources

  • Global service

• Config

  • Record configuration changes

  • Evaluate resources against compliance rules

  • Get timeline of changes and compliance

sudo du -ch $(docker inspect --format='{{.LogPath}}' $(docker ps -qa)) | sort -h

Set default limited log size when creating new containers

On Linux

• Path: /etc/docker/daemon.json

On Windows

• Path: %USERPROFILE%\.docker\daemon.json

{
  "log-driver": "local",
  "log-opts": {
    "max-size": "10m"
  }
}

• Find objects, text, people, and scenes in images and videos using ML

• Facial analysis and facial search to do user verification, people counting

• Create a database of “familiar faces” or compare them to celebrities

Use cases:

• Labeling

• Content Moderation

• Text Detection

• Face Detection and Analysis

• Face Search and Verification

• Celebrity Regonnition

• Pathing

Content Moderation

• Detect content that is inappropriate, unwanted, or offensive (images and videos)

• Used in social media, broadcast media, advertising, and e-commerce situations to create a safer user experience

• Set a minimum confidence Threshold for items that will be flagged

Amazon Transcribe

• Automatically convert speech to text

• Use cases:

  • transcribe customer service calls

  • automate closed captioning and subtitling

  • generate metadata for media assets

Amazon Polly

• Turn text into lifelike speech using deep learning

• Allowing the creation application to talk

Lexicon & SSML

• Customize the pronunciation of words with Pronunciation Lexicons

  • Stylized words: St3ph4ne => “Stephane”

  • Acronyms: AWS => “Amazon Web Services”

• Upload the lexicons and use them in SynthesizeSpeech

• Generate speech from plain text or documents marked up with Speech Synthesis Markup Language (SSML)

Amazon Lex & Connect

• Amazon Lex: (like Siri)

  • Automatic Speech Recognition to convert speech to text

  • Natural Language Understanding to recognize the intent of text, callers

  • Help build chatbots, call center bots

• Amazon Connect:

  • Receive calls, create contact flows, and cloud-based virtual contact center

  • Can integrate with other CRM systems or AWS

  • No upfront payments, 80% cheaper than traditional contact center solutions

Amazon Comprehend

• For NLP

• Fully managed and serverless service

• Uses ML to find insights and relationships in text

• Use cases:

  • analyze customer interactions

  • create and group articles by topics

Comprehend Medical

• Detects and returns useful information in unstructured clinical text: Physician’s notes, Discharge summaries, and Test results,…

• Use NLP to detect Protected Health Information

SageMaker

• Fully managed service to build LM models

• Typically difficult to do all the processes in one place + provision services

Amazon Forecast

• Fully managed service that uses ML to deliver highly accurate forecasts

• Example: predict the future sales of a raincoat

• Use cases: Product Demand Planning, Financial Planning,…

Amazon Kendra

• Fully managed document search service powered by ML

• Extract answers from a document

Amazon Personalize

• Fully managed ML service to build apps with real-time personalized recommendations

• Use cases: retail stores, media, and entertainment,…

Amazon Textract

• Automatically extracts text, handwriting, and data from any scanned documents using AL and ML

Summary

• Rekognition: face detection, labeling

• Transcribe: audio to text

• Polly: text to audio

• Translate: translations

• Lex: build conversational bots- chatbots

• Connect: cloud contact center

• Comprehend: NLP

• SageMaker: build ML model

• Forecast: build highly accurate forecasts

• Kendra: ML-powered search engine

• Personalize: real-time personalized recommendations

• Textract: detect text and data in documents

• Serverless query service to analyze data stored in S3

• Use SQL language to query the files (built on Presto)

• Supports CSV, JSON, ORC, Avro, and Parquet

• Pricing: 5$ per TB of data scanned

• Commonly used with Amazon Quicksight for reporting/dashboards

Use cases: Business intelligence / analytics / reporting, analyze & query VPC Flow Logs, ELB Logs, CloudTrail trails,…

Exam tips: analyze data in S3 using serverless SQL, use Athena

Federated Query

• Allows you to run SQL queries across data stored in SQL, NoSQL, object,…

• Uses Data Source Connectors that run on AWS Lambda to run Federated Queries

• Store the result back in the S3 bucket

Redshift

• It is based on PostgreSQL, but it’s not used for OLTP

• It’s OLAP (Online Analytical Processing)

• 10x better performance than other data warehouses, scale to PBs of data

• Columnar storage of data & parallel query engine

• Two modes: Serverless Cluster & Provisioned Cluster

• Has a SQL interface for performing the queries

• BI Tools such as Amazon Quicksight or Tableau

• vs Athena: faster queries/joins/aggregations thanks to indexes

Redshift CLuster

• The architecture:

  • Leader node: for query planning, results aggregation

  • Compute node: for performing the queries, send results to the leader node

  • Provisioned mode:

    • Choose instance types in advance

    • Can reserve instances for cost savings

Snapshots & DR

• Snapshots are point-in-time backups of a cluster, stored internally in S3

• Snapshots are incremental

• You can restore a snapshot into a new cluster

• Automated: every 8 hours, every 5 GB, or a schedule

• Manual: snapshot is retained until you delete it

• You can configure Redshift to copy snapshots of a cluster to another region automatically

Redshift Spectrum

• Query data that is already in S3 without loading it

• Must have a Redshift cluster available to start the query

• The query is then submitted to thousands of Redshift Spectrum nodes

OpenSearch

• Two modes:

  • managed cluster

  • serverless cluster

• Does not natively support SQL (can be enabled via a plugin)

• Ingestion from Kinesis Data Firehose, AWS IoT, and CloudWatch Logs,…

• Comes with OpenSearch Dashboards

EMR

• EMR stands for “Elastic MapReduce”

• EMR helps create Hadoop clusters to analyze and process vast amounts of data

• The cluster can be made of hundreds of EC2 instances

• EMR comes bundled with Spark, HBase, Presto, Flink,…

• EMR takes care of all the provisioning and configuration

• Auto-scaling and integrated with Spot instances

Use cases: data processing, machine learning, web indexing, big data,…

Node types & Purchasing

• Master Node

• Core Node

• Task Node (optional)

• Purchasing options:

  • On-demand

  • Reserved (min 1 year): cost savings

  • Spot instances: cheaper

• Can have a long-running cluster, or transient (temporary) cluster

Quicksight

• Serverless machine learning-powered business intelligence service to create interactive dashboards

• Fast, automatically scalable, embeddable, with per-session pricing

• Use cases:

  • Business Analytics

  • Building visualizations

  • Perform ad-hoc analysis

• Integrated with RDS, Aurora, Athena, Redshift, S3,…

• Im-memory computation using the SPICE engine if data is imported into QuickSight

• Enterprise edition: Column-level Security

Glue

• Managed extract, transform, and load (ETL) service

• Useful to prepare and transform data for analytics

• Full serverless services

• Use cases: convert data into Parquet format

Things to know at a high level

• Glue Job Bookmarks: prevent re-processing old data

• Glue Elastic Views:

  • Combine and replicate data across multiple data stores using SQL

  • No custom code

  • Leverages a “virtual table”

• Glue DataBrew: clean and normalize data using pre-built transformation

• Glue Studio: new GUI to create, run, and monitor ETL jobs in Glue

• Glue Streaming ETL (built on Spark): compatible with Kinesis Data Streaming, Kafka, MSK (managed Kafka)

AWS Lake Formation

• Data lake = central place to have all data for analytics purposes

• Fully managed service that makes it easy to set up a data lake in days

• Discover, cleanse, transform, and ingest data into Data Lake

• It automates many complex manual steps (collecting, cleansing, moving, cataloging data,…) and de-duplicate (using ML Transforms)

• Combine structured and unstructured data in the data lake

• Out-of-the-box source blueprints: S3, RDS, Relational & NoSQL DB,…

• Fine-grained Access Control for applications (row and column-level)

• Built on top of AWS Glue

Kinesis Data Analytics

For SQL

• Real-time analytics on Kinesis Data Stream & Firehose using SQL

• Add reference data from S3 to enrich streaming data

• It is fully managed, with no servers to provision

• Automatic scaling

• Pay for actual consumption rate

• Output:

  • Kinesis Data Streams

  • Kinesis Data Firehose

Use cases: Time series analytics, Real-time dashboards, Real-time metrics

For Apache Flink

• Use Flink (Java, Scala, or SQL) to process and analyze streaming data

• Run any Apache Flink application on a managed cluster on AWS

  • provisioning compute resources, parallel computation, automatic scaling

  • application backups

  • Use any Apache Flink programming features

  • Flink does not read from Firehose (use Kinesis Analytics for SQL instead)

MSK (Managed Streaming for Apache Kafka)

• Alternative to Amazon Kinesis

• Fully managed Kafka on AWS

  • Allow to create, update, and delete clusters

  • MSK creates & manages Kafka brokers nodes & Zookeeper nodes

  • Deploy the MSK cluster in VPC, multi-AZ (up to 3 for HA)

  • Automatic recovery from common Kafka failures

  • Data is stored on EBS volumes for as long as you want

• MSK Serverless

  • Run Kafka on AWS on MSK without managing the capacity

  • MSK automatically provisions resources and scales computing & storage

The difference between Kinesis Data Streams vs MSK

sudo yum update -y 

sudo amazon-linux-extras install docker 

sudo yum install docker 

sudo service docker start 

sudo usermod -a -G docker ec2-user

To install Docker Compose

sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose

sudo chmod +x /usr/local/bin/docker-compose

docker-compose version

• Managed PostgreSQL, MySQL, Oracle, SQL Server, DB2, MariaDB, Custom

• Provisioned RDS Instance Size and EBS Volume Type & Size

• Auto-scaling capability for Storage

• Support for Read Replicas and Multi-AZ (for HA and have a standby database)

• Security through IAM, Securities Groups, KMS, SSL in transit

• Automated Backup with Point in time restore feature (up to 35 days)

• Manual DB Snapshot for longer-term recovery

• Managed and Scheduled maintenance (with downtime)

• Support for IAM Authentication, integration with Secrets Manager

• RDS Custom for access to and customize the underlying instance (Oracle & SQL Server)

Use cases: store relational datasets (RDMBS/OLTP), perform SQL queries, transactions

Aurora

• Compatible API for PostgreSQL / MySQL, separation of storage and compute

• Storage: data is stored in 6 replicas, across 3 AZ - HA, self-healing, auto-scaling

• Compute: Cluster of DB Instance across multiple AZ, auto-scaling of Read Replicas

• Cluster: Custom endpoints for writer and reader DB instances

• Same security/monitoring/maintenance features as RDS

• Know the backup & restore options for Aurora

• Aurora Serverless - for unpredictable workloads

• Aurora Global: up to 16 DB Read Instances in reach region, < 1 second storage replication

• Aurora Machine Learning: perform ML using SageMaker & Comprehend on Aurora

• Aurora Database Cloning: new cluster from existing one, faster than restoring a snapshot

Use cases: same as RDS, but with less maintenance, more flexibility, more performance, more features

ElastiCache

• Managed Redis / Memcached

• In-memory data store, sub-millisecond latency

• Support for Clustering (Redis) and Multi-AZ, Read Replicas (sharding)

• Security through IAM, Security Groups, KMS, Redis Auth

• Backup / Snapshot, Point in time restore feature

• Manage and Schedule maintenance

• Requires some application code changes to be leveraged

Use cases: Key/Value store, frequent reads, less writes, cache results for DB queries, store session data for websites, cannot use SQL

DynamoDB

• AWS proprietary technology, managed serverless NoSQL db, millisecond latency

• Capacity modes: provisioned capacity with optional auto-scaling or on-demand capacity

• Can replace ElastiCache as a key/value store

• HA, multi-AZ by default, Read and Writes are decoupled, transaction capability

• DAX cluster for read cache, microsecond read latency

• Security, authentication/author is done through IAM

• Event Processing: DynamoDB Streams to integrate with Lambda or Kinesis Data Streams

• Global Table feature: active-active setup

• Automated backups up to 35 days with PITR (point-in-time recovery), or on-demand backups

• Export to S3 without using RCU in the PITR window, import from S3 without using WCU

• It is great to evolve schemas rapidly

• Use case: serverless applications development (small document 100s KM), distributed serverless cache

S3

• Great for bigger objects, not so great for many small objects

• Serverless, scales infinitely, max object size is 5 TB, version capability

• Tiers: S3 Standard, S3 IA, S3 Intelligent, S3 Glacier + lifecycle policy

• Features: versioning, encryption, replication, MFA-Delete, Access Logs,…

• Security: IAM, bucket policies, ACL, Access Points, Object Lambda, CORD, Object/Vault Lock

• Encryption: SSE-S3, SSE-KMS,…

• Batch operations on objects using S3 Batch, listing files using S3 Inventory

• Performance: Multi-part upload, S3 Transfer Acceleration, S3 Select

• Automation: S3 Event notifications (SNS, SQS, Lambda, EventBridge)

Use cases: static files, key valuie store for big files, website hosting

DocumentDB

• Is the same for MongoDB (like Aurora for MongoDB)

• Similar deployment concepts to Aurora

• Fully managed, HA with replication across 3 AZ

• DocumentDB storage automatically grows in increments of 10 GB

• Automatically scales to workloads with millions of requests per second

Neptune

• Full-managed graph database

• A popular graph database would be a social network

• HA across 3 AZ, up to 15 read replicas

• Build and run applications working with highly connected datasets - optimized for these complex and hard queries

• Can store up to billions of relations and query the graph with milliseconds latency

• Great for knowledge graphs, fraud detection, recommendation engines, social networking

• Support for Streams (like Dynamo Data Streams)

  • Send notifications when certain changes are made

  • Maintain graph data synchronized in another data store (S3, OpenSearch,…)

  • Replicate data across regions in Neptune

Keyspaces (for Apache Cassandra)

• Cassandra is an open-source NoSQL distributed database

• A managed Apache Cassandra-compatible database service

• Serverless, Scalable, HA, fully managed by AWS

• Automatically scale tables up/down based on the application’s traffic

• Tables are replicated 3 times across multiple AZ

• Using the Cassandra Query Language (CQL)

• 100s of requests per second

• Capacity: on-demand mode or provisioned mode with auto-scaling

• Encryption, backup, Point-In-Time recovery up to 35 days

Use cases: store IOT devices info, time-series data,…

Quantum Ledger Database

• A ledger is a book recording financial transactions

• Fully managed, serverless, HA, replication across 3 AZ

• Used to review the history of all the changes made to application data over time

• Immutable system: no entry can be removed or modified, cryptographically verifiable

• 2-3x better performance than common ledger blockchain frameworks, manipulate data using SQL

TimeStream

• Full-managed, fast, scalable, serverless time series database

• Automatically scales up/down to adjust capacity

• Store and analyze trillions of events per day

• 100s time faster & 1/10 the cost of relational databases

• Scheduled queries, multi-measure records, SQL compatibility

Use cases: IoT apps, operatinal applications, real-time analytics,…

Limits

• Execution:

  • Memory allocation: 120 MB - 10 GB

  • Maximum execution time: 900 seconds (15 minutes)

  • Environment variables (4 KB)

  • Concurrency executions: 1000 (can be increased)

  • Disk capacity (in /tmp): 512 MB to 10 GB

• Deployment:

  • Lambda function deployment size (compressed .zip): 50 MB

  • Size of uncompressed deployment: 250 MB

  • You can use the /tmp directory to load other files at startup

  • Size of environment variable: 4 KB

Lambda SnapStart

• Improves Lambda functions performance up to 10x at no extra cost for Java 11 and above

• When enabled, the function is invoked from a pre-initialized state

• When you publish a new version:

  • Lambda initialize function

  • Takes a snapshot of memory and disk state of the initialize function

  • Snapshot is cached for low-latency access

Customization at the Edge

• Edge Function:

  • A code that you write and attach to CloudFront

  • Run close to your users to minimize latency

• CloudFront provides two types: CloudFont Functions & Lambda Edge

• Use case: customize the CDN content

• Pay only for what you use

• Fully serverless

Some use-cases

• Website Security and Privacy

• Dynamic web application at the Edge

• SEO

• Intelligently Route Across Origins and Data Centers

• Bot mitigation at the Edge

• Real-time Image Transformation

• A/B Testing

• Use Authen/Author

• User Tracking and Analytics

CloudFront Functions

• Lightweight functions written in JavaScript

• Sub-ms startup times, millions of requests/second

• Used to change Viewer requests and Viewer responses

• Native feature of CloudFront (manage code directly in CloudFront)

• Use cases: cache key normalization (transform request attributes to create an optimal Cache key), insert/modify/delete HTTP headers, URL rewrites or redirects, generate

• and validate user-generated tokens (e.g. JWT) to allow/deny requests

Lambda Edge

• Lambda functions written in NodeJS or Python

• Scales to 1000s of requests/second

• Used to change CloudFront requests CloudFront and responses to the origin

• Author your functions in one AWS region (us-east-1) then CloudFront replicates to its locations

• Use cases: Longer execution time, code depends on a 3rd library, network access to use external services for processing, file system access or access to the body of HTTP requests

Lambda in VPC

By default, the Lambda function is launched outside VPC, therefore, it cannot access resources in VPC (RDS, ElastiCache, internal ELB,…)

Solutions

• Defint the VPC ID, the Subnets and Security Groups

• Lambda will create an ENI (Elastic Network Interface) in a subnet

Lambda with RDS Proxy

• To reduce the workload when lots of lambda functions directly access the database => using RDS Proxy

• RDS Proxy

  • Improve scalability by pooling and sharing DB connections

  • Improve availability by reducing by 66% the failover time and preserving connections

  • Improve security by enforcing IAM authentication and storing credentials in Secrets Manager

The Lambda Function must be deployed in VPC, because RDS Proxy is never publicly accessible

Amazon DynamoDB

• NoSQL - with transaction support

• Scales to massive workload, distributed database

• Millions of requests per second, trillions of rows, 100s of TB of storage

• Fast and consistent in performance

• Low-cost and auto-scaling capabilities

• There is no maintenance or patching. It is always available

• Standard & Infrequent Access (IA) Table Class

Basics

• Each table has a Primary Key that needs to be defined when creating it

• Each table can have an infinite number of items

• Each item has attributes (can be added over time - can be null)

• The maximum size of an item is 400 KB

• Data types supported are:

  • Scalar Types - String, Number, Binary, Boolean, Null

  • DocumentTypes - List, Map

  • SetTypes - String Set, Number Set, Binary Set

Read/Write Capacity Modes

Provisioned Mode (default)

• You specify the number of reads/writes per second

• You need to plan capacity beforehand

• Pay for provisioned Read Capacity Units (RCU) & Write Capacity Units (WCU)

• Possibility to add auto-scaling mode for RCU & WCU

On-Demand Mode

• Read/writes automatically scale up/down with your workload

• No capacity planning is needed

• Pay for what you use, more expensive

• Great for unpredictable workloads

DynamoDB Accelerator (DAX)

• Help solve read congestion by caching

• Microseconds latency for cached data

• Doesn’t require application logic modification (compatible with existing DynamoDB APIs)

• 5 minutes TTL for cache (default)

Stream Processing

• Ordered stream of item-level modifications (create/update/delete) in a table

• Use cases:

  • React to change in real-time

  • Real-time usage analytics

  • Insert into derivative tables

  • Implement cross-region replication

  • Invoke AWS Lambda on changes to your DynamoDB table

• The characteristics of DynamoDB Streams:

  • 24 hours retention

  • Limited # of consumers

  • Process using AWS Lambda Triggers or Dynamoc DB Stream Kinesis adapter

• The characteristics of Kinesis Data Streams (newer):

  • 1-year retention

  • High # of consumers

  • Process using AWS Lambda, Kinesis Data Analytics, Kinesis Data Firehose, AWS Glue Streaming ETLs,...

Global Tables

• Make a DynamoDB table accessible with low latency in multiple regions

• Active-Active replication

• Applications can READ and WRITE to the table in any region

• Must enable DynamoDB Stream as a pre-requisite

Time To Live

• Automatically delete items after an expiry timestamp

Backups for disaster recovery

• Continuous backups using point-in-time recovery (PITR)

  • Optionally enabled for the last 35 days

  • Point-in-time recovery to any time in the backup window

  • The recovery process creates a new table

• On-demand backups

  • Full backups for long-term retention, until explicitly deleted

  • It does not affect performance or latency

  • Can be configured and managed in AWS Backup (enables cross-region copy)

  • The recovery process creates a new table

Integration with S3

• Export to S3 (must enable PITR)

  • It does not affect the reading capacity of the table

  • Perform data analysis on top of DynamoDB

  • Retain snapshots for auditing

  • ETL on top of S3 data before importing back into DynamoDB

  • Export in DynamoDB Json or ION format

• Import from S3

  • Import CSV, DynamoDB Json, or ION format

  • It does not consume any writing capacity

  • Creates a new table

  • Import errors are logged in CloudWatch

API Gateway

• AWS Lambda + API Gateway: no infra to manage

• Support for the Websocket

• Handle API versioning

• Handle different environments

• Handle security (authen/author)

• Create API keys, handle request throttling

• Swagger/Open API import to quickly define APIs

• Transform and validate requests and responses

• Cache API responses

• Generate SDK and API specifications

• Endpoint Types:

  • Edge-Optimized (default): for global clients

    • Requests are routed through the CloudFront Edge locations

    • The API gateway still lives in only one region

  • Regional

    • For clients in the same region

    • Could manually combine with CloudFront

  • Private

    • Can only be accessed from VPC using VPC Endpoint (ENI)

    • Use a resource policy to define access

AWS Step Functions

• Build serverless visual workflow to orchestrate Lambda functions

• Features: sequence, parallel, conditions, timeouts, error handling,…

• Can integrate with EC2, ECS, On-premises servers, API Gateway, SQS,…

• Possibility of implementing a human approval feature

• Use cases: order fulfillment, data processing, web app, any workflow,…

Amazon Cognito

• Give users an identity to interact with web or mobile app

• Cognito User Pools:

  • Sign-in functionality for app users

  • Integrate with API gateway & ALB

• Cognito Identity Pools (Federated Identity)

  • Provide AWS credentials to users so they can access AWS resources directly

  • Integrate with Cognito User Pools as an identity provider

Cognito User Pools (CUP)

User Features

• Create a serverless database of users for web & mobile app

• Simple login: username/password

• Password reset

• Email & phone number verification

• MFA

• Federated Identities: users from Fb, Google, SAML,…

Integrations

• CUP integrates with API Gateway and ALB

Federated Identities

• Get identities for “users” so they obtain temporary AWS credentials

• Users can then access AWS services directly or through API Gateway

CREATE EXTENSION IF NOT EXISTS unaccent;
CREATE EXTENSION vector SCHEMA "public" VERSION 0.7.2;

Use a custom text search configuration

CREATE TEXT SEARCH CONFIGURATION vietnamese (COPY = simple);
ALTER TEXT SEARCH CONFIGURATION vietnamese
ALTER MAPPING FOR asciiword, word
WITH unaccent, simple;

Example query

SELECT staffCode, userName, staffName, phoneNumber, email
FROM public.users_embedding_table
WHERE to_tsvector('vietnamese', 
                  unaccent(COALESCE(staffCode, '') || ' ' || 
                           COALESCE(userName, '') || ' ' || 
                           COALESCE(staffName, '') || ' ' || 
                           COALESCE(phoneNumber, '') || ' ' || 
                           COALESCE(email, '')
                  )
                 ) @@ plainto_tsquery('vietnamese', unaccent('Hòa'));

1. Amazon Elastic Container Service (ECS) - Amazon’s container

2. Amazon Elastic Kubernetes Service (EKS) - Open-source

3. AWS Fargate - Amazon’s serverless container

4. Amazon Elastic Container Registry (ECR) - Store container images

Amazon ECS

EC2 Launch Type

• Launch Docker containers on AWS = Launch ECS Tasks on ECS Cluster

• You must provision and maintain the infrastructure (the EC2 instances)

• Each EC2 instance must run the ECS Agent to register in the ECS Cluster

• AWS takes care of stopping/starting container instances

Fargate Launch Type

• You do not provision the infrastructure (no EC2 instances)

• It is all Serverless

• AWS runs ECS Tasks for you based on the CPU/RAM you need

• To scale, increase the number of tasks, no more EC2 instances

IAM Roles for ECS

• EC2 Instance Profile (EC2 Launch Type only)

  • Used by the ECS agent

  • Makes API calls to ECS service

  • Send container logs to CloudWatch Logs

  • Pull Docker image from ECR

  • Reference sensitive data in Secrets Manager

• ECS Task Role

  • Allow each task to have a specific role

Load Balancer Integrations

• ALB is supported and works for most use cases

• NLB is recommended only for high throughput / high-performance use cases, or to pair it with AWS Private Link

Data Volumes (EFS)

• Mount EFS file systems onto ECS tasks

• Work for both EC2 and Fargate launch types

• Tasks running in any AZ will share the same data in the EFS file system

• Fargate + EFS = Serverless

• Use cases: persistent multi-AZ shared storage for containers

ECS Service Auto Scaling

• Automatically increase/decrease the desired number of ECS tasks

• ECS Auto Scaling uses AWS Application Auto Scaling

  • ECS Service Average CPU Utilization

  • ECS Service Average RAM

  • ALB request Count per Target - metric from ALB

• Target Tracking - scale based on target value for a specific CloudWatch metric

• Step Scaling - scale based on a specified CloudWatch Alarm

• Schedule Scaling

• ECS Service Auto Scaling (task level) ≠ EC2 Auto Scaling (instance level)

Amazon EKS

• It’s an alternative to ECS, with a similar goal but a different API

• EKS supports EC2 if you want to deploy worker nodes or Fargate to deploy serverless containers

• Use case: if your company is already using K8S on-premises or in another cloud, and wants to migrate to AWS using K8S

Node Types

Managed Node Groups

• Create and manage Nodes (EC2) for you

• Nodes are part of an ASG managed by EKS

• Supports On-Demand or Spot Instances

Self-Managed Nodes

• Nodes are created by you and registered to the EKS cluster and managed by an ASG

• You can use pre-built AMI

• Support On-Demand or Spot Instances

AWS Fargate

• No need to manage nodes

Data Volumes

• Need to specify StorageClass manifest on EKS Cluster

• Leverages a Container Storage Interface compliant driver

• Support for: EBS, EFS (work with Fargate), FSx for Lustre, FSx for NetApp ONTAP

AWS App Runner

• No infra experience is required

• Start with your source code or container image

• Automatically builds and deploys the web app

• Automatic scaling, HA, load balancer, encryption

• VPC access support

• Connect to database, cache, and message queue services

• Use cases: web apps, APIs, microservices, rapid production deployments

AWS App2Container

• CLI Tool for migrating and modernizing Java and DotNET web apps into Docker Containers

• Lift-and-shift apps running in on-premises bare metal, virtual machines, or in any Cloud to AWS

• Generates CloudFormation templates

• Register generated Docker containers to ECR

• Deploy to ECS, EKS, or App Runner

foreach ((int index, Product product) in ProductList.Products.Index())
{
    Console.WriteLine($"Index = {index}, Name = {product.Title}");
}

SearchValues (This feature was introduced in .NET 8)

In .NET 8, the SearchValues type is limited to searching by characters. However, in .NET 9, it is possible to search by multiple strings.

ReadOnlySpan<string> searchWords = ["dummy", "text", "and"];
SearchValues<string> searchValues =
    SearchValues.Create(searchWords, StringComparison.OrdinalIgnoreCase);

var searchString =
    """
    Lorem Ipsum is simply dummy text of
    the printing and typesetting industry.
    """;

var index = searchString
    .AsSpan()
    .IndexOfAny(searchValues);

string[] productTitles = ProductList.Products.Select(x => x.Title).ToArray();

SearchValues<string> svProducts = SearchValues.Create(productTitles, StringComparison.OrdinalIgnoreCase);

IEnumerable<Product> found = ProductList.Products
    .Where(x => svProducts.Contains(x.Title));

The downside is you can not use the type Product, only the string.

Standard Queue

Attributes

• Unlimited throughput, unlimited number of messages in queue

• Default retention of messages: 4 days, maximum of 14 days

• Low latency (< 10 ms on publish and receive)

• Limitation of 256 KB per message sent

Can have duplicate messages (at least once delivery)
Can have out-of-order messages

Producing Messages

• Produced to SQS using the SDK (SendMessage API)

• The message persists in SQS until a consumer deletes it

• Message retention: default 4 days, up to 14 days

Consuming Messages

• Consumers (EC2, Lambda, Servers,…)

• Poll SQS for messages (receive up to 10 messages at a time)

• Delete the messages using the DeleteMessage API

Securities

• Encryption

  • In-flight encryption using HTTPS API

  • At-rest encryption using KMS keys

  • Supporting client-side encryption

• Access Controls: IAM Policies

• SQS Access Policies (similar to S3 bucket policies)

  • Useful for cross-account access to SQS

  • Useful for allowing other services (SNS, S3,…) to write to an SQS

Long Polling

Where a consumer requests messages from the queue, it can optionally “wait” for messages to arrive if there are none in the queue => This is called Long Polling

Long Polling decreases the number of API calls made to SQS while increasing the efficiency and reducing latency for application

The wait time can be between 1 sec to 20 sec

Long Polling is preferable to Short Polling

It can be enabled at the queue level or the API level (WaitTimeSeconds API)

FIFO Queue

• Limit throughput: 300 msg/s without batching, 3000 msg/s with batching

• Exactly-once-send capability (by removing duplicates)

• Messages are processed in order by the consumers

Amazon SNS

• Up to 12,500,000 subscriptions per topic

• 100,000 topics limit

Many services send data directly to SNS for notifications such as:
CloudWatch, AWS Budgets, Lambda, Auto Scaling Group, S3, DynamoDB, CloudFormation, AWS DMS, RDS Events,…

SNS + SQS: Fan out

• Push once in SNS, receive in all SQS queues

• Fully decoupled, no data loss

• SQS allows for data persistence, delayed processing, and retries of work

• Ability to add more SQS subscribers over time

• Make sure the SQS queue access policy allows for SNS to write

• Cross-Region Delivery: works with SQS in other regions

SNS - FIFO Topic

• Similar features as SQS FIFO:

  • Ordering by Message Group ID

  • Deduplication using a Deduplication ID or Content-Based Deduplication

• Can have SQS Standard or SQS FIFO as subscribers

• Limit throughput: 300 msg/s without batching, 3000 msg/s with batching

SNS - Message Filtering

• JSON policy used to filter messages sent to SNS topic’s subscriptions

• If a subscription does not have a filter policy, it receives every message

Kinesis

Overview

• Make it easy to collect, process, and analyze streaming data in real-time

• Ingest real-time data such as Application Logs, Metrics, Website, clickstreams, IoT telemetry data,…

• Kinesis Data Streams: capture, process, and store data streams

• Kinesis Data Firehose: load data streams into AWS data stores

• Kinesis Data Analytics: analyze data streams with SQL or Apache Flink

• Kinesis Video Streams: capture process and store video streams

Kinesis Data Streams

• Retention between 1 day to 365 days

• Once data is inserted in Kinesis, it can not be deleted

• Ability to reprocess data

• Data that share the same partition go to the same shard

Capacity Modes

• Provisioned mode:

  • You choose the number of shards provisioned, Salce manually or using API

  • Each shard gets 1MB/s in (or 1000 records per second)

  • Each shard gets 2MB/s out

  • You pay per shard provisioned per hour

• On-demand mode:

  • No need to provision or manage the capacity

  • Default capacity provisioned (4MB/s or 4000 records per second)

  • Scales automatically based on observed throughput peaks during the last 30 days

  • Pay per stream per hour & data in/out per GB

Security

• Control access/authorization using IAM policies

• Encryption in flight using HTTPS

• Encryption at rest using KMS

• Supporting encrypt/decrypt at the client side

• VPC endpoints for Kinesis to access in VPC

• Monitor API calls using CloudTrail

Kinesis Data Firehose

• Fully managed service, automatic scaling, serverless

  • AWS: Redshift, S3, OpenSearch,…

  • 3rd party: MongoDB, DataDog,…

  • Custom: send to any HTTP endpoint

• Pay for data going through Firehose

• Near real-time

  • Buffer interval: 0 seconds to 900 seconds

  • Buffer size: minimum 1MB

• Supports many data formats, conversions, transformations, compression

• Support custom data transformations using Lambda

• Can send failed or all data to a backup S3 bucket

The comparison table between Data Stream and Firehose