# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: MIT-0 # # Permission is hereby granted, free of charge, to any person obtaining a copy of this # software and associated documentation files (the "Software"), to deal in the Software # without restriction, including without limitation the rights to use, copy, modify, # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # AWSTemplateFormatVersion: '2010-09-09' Description: Data Mask Pipeline Parameters: Prefix: Type: String Default: dcp CreateDMSRole: Type: String AllowedValues: - 'yes' - 'no' Description: Should create or not the dms-vpc-role? Please, check if your account has the role name dms-vpc-role. If yes, select no for not to create this role again, otherwise select yes. Default: 'yes' DBUser: Default: admin Type: String MinLength: '4' MaxLength: '20' AllowedPattern: '[a-zA-Z0-9]*' ConstraintDescription: must contain only alphanumeric characters. DBName: Default: dataset Type: String LatestAmzn2AmiId: Type: AWS::SSM::Parameter::Value Default: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-arm64-gp2 TestUserPassword: Type: String MinLength: 8 NoEcho: true AllowedPattern: '[[[A-Z]{1,}[a-z]{1,}[0-9]{1,}\!\#\@\$\%\^\&*\(\)_+\-=+\[\]\{\}\|]]{8,}' Description: "Minimum password length of 8 characters and a maximum length of 128 characters. Minimum of three of the following mix of character types: uppercase, lowercase, numbers, and ! @ # $ % ^ & * ( ) _ + - = [ ] { } | ' symbols" ConstraintDescription: "Must contain at least 8 characters and a maximum of 128 characters. Must have at least one uppercase letter, one lowercase letter, one number, and a symbol." VPCCIDR: Type: String Default: '10.0.0.0/16' PublicSubnet0CIDR: Type: String Default: "10.0.1.0/24" PublicSubnet1CIDR: Type: String Default: "10.0.2.0/24" PrivateSubnet0CIDR: Type: String Default: "10.0.3.0/24" PrivateSubnet1CIDR: Type: String Default: "10.0.4.0/24" Metadata: AWS::CloudFormation::Interface: ParameterGroups: - Label: default: "Basic configuration" Parameters: - Prefix - LatestAmzn2AmiId - TestUserPassword - Label: default: "Database and DMS configuration" Parameters: - DBName - DBUser - CreateDMSRole - Label: default: "Network Configuration" Parameters: - VPCCIDR - PublicSubnet0CIDR - PublicSubnet1CIDR - PrivateSubnet0CIDR - PrivateSubnet1CIDR ParameterLabels: VPCCIDR: default: "Which VPC should this be deployed to?" CreateDMSRole: default: "Should we create the DMS VPC Role?" TestUserPassword: default: "Password to allow the personas to login to the AWS console" DBName: default: "MySQL database name" DBUser: default: "MySQL database user" LatestAmzn2AmiId: default: "Latest Amazon Linux 2 AMI available (optional to modify)" Prefix: default: "Prefix to be used when creating the resources and assets" PublicSubnet0CIDR: default: "CIDR range for the Public Subnet 0" PublicSubnet1CIDR: default: "CIDR range for the Public Subnet 1" PrivateSubnet0CIDR: default: "CIDR range for the Private Subnet 0" PrivateSubnet1CIDR: default: "CIDR range for the Private Subnet 1" Conditions: CreateDMSVPCRole: !Equals - !Ref CreateDMSRole - 'yes' Resources: # __ ______ ____ # \ \ / / _ \ / ___| # \ \ / /| |_) | | # \ V / | __/| |___ # \_/ |_| \____| # VPC: Type: 'AWS::EC2::VPC' Properties: CidrBlock: !Ref VPCCIDR EnableDnsSupport: true EnableDnsHostnames: true Tags: - Key: Name Value: !Sub '${Prefix}-vpc' Metadata: cfn_nag: rules_to_suppress: - id: W60 reason: "VPC without flow log attached." InternetGateway: Type: 'AWS::EC2::InternetGateway' Properties: Tags: - Key: Name Value: !Sub '${Prefix}-igw' InternetGatewayAttachment: Type: 'AWS::EC2::VPCGatewayAttachment' Properties: InternetGatewayId: !Ref InternetGateway VpcId: !Ref VPC PublicSubnet0: Type: 'AWS::EC2::Subnet' Properties: VpcId: !Ref VPC AvailabilityZone: !Select - 0 - Fn::GetAZs: !Ref 'AWS::Region' CidrBlock: !Ref PublicSubnet0CIDR MapPublicIpOnLaunch: true Tags: - Key: Name Value: !Sub '${Prefix}-public_subnet0' Metadata: cfn_nag: rules_to_suppress: - id: W22 reason: "EC2 needs to communicate with the internet." - id: W33 reason: "EC2 Subnet should have MapPublicIpOnLaunch set to true, because EC2 needs to communicate with the internet" PublicSubnet1: Type: 'AWS::EC2::Subnet' Properties: VpcId: !Ref VPC AvailabilityZone: !Select - 1 - Fn::GetAZs: !Ref 'AWS::Region' CidrBlock: !Ref PublicSubnet1CIDR MapPublicIpOnLaunch: true Tags: - Key: Name Value: !Sub '${Prefix}-public_subnet1' Metadata: cfn_nag: rules_to_suppress: - id: W22 reason: "EC2 needs to communicate with the internet." - id: W33 reason: "EC2 Subnet should have MapPublicIpOnLaunch set to true, because EC2 needs to communicate with the internet" PrivateSubnet0: Type: 'AWS::EC2::Subnet' Properties: VpcId: !Ref VPC AvailabilityZone: !Select - 0 - Fn::GetAZs: !Ref 'AWS::Region' CidrBlock: !Ref PrivateSubnet0CIDR MapPublicIpOnLaunch: false Tags: - Key: Name Value: !Sub '${Prefix}-private_subnet0' PrivateSubnet1: Type: 'AWS::EC2::Subnet' Properties: VpcId: !Ref VPC AvailabilityZone: !Select - 1 - Fn::GetAZs: !Ref 'AWS::Region' CidrBlock: !Ref PrivateSubnet1CIDR MapPublicIpOnLaunch: false Tags: - Key: Name Value: !Sub '${Prefix}-private_subnet1' PublicRouteTable: Type: 'AWS::EC2::RouteTable' Properties: VpcId: !Ref VPC Tags: - Key: Name Value: !Sub '${Prefix}-public_rt' PrivateRouteTable: Type: 'AWS::EC2::RouteTable' Properties: VpcId: !Ref VPC Tags: - Key: Name Value: !Sub '${Prefix}-private_rt' PublicDefaultRoute: Type: 'AWS::EC2::Route' DependsOn: InternetGatewayAttachment Properties: RouteTableId: !Ref PublicRouteTable DestinationCidrBlock: 0.0.0.0/0 GatewayId: !Ref InternetGateway PublicSubnet0RouteTableAssociation: Type: 'AWS::EC2::SubnetRouteTableAssociation' Properties: RouteTableId: !Ref PublicRouteTable SubnetId: !Ref PublicSubnet0 PublicSubnet1RouteTableAssociation: Type: 'AWS::EC2::SubnetRouteTableAssociation' Properties: RouteTableId: !Ref PublicRouteTable SubnetId: !Ref PublicSubnet1 PrivateSubnet0RouteTableAssociation: Type: 'AWS::EC2::SubnetRouteTableAssociation' Properties: RouteTableId: !Ref PrivateRouteTable SubnetId: !Ref PrivateSubnet0 PrivateSubnet1RouteTableAssociation: Type: 'AWS::EC2::SubnetRouteTableAssociation' Properties: RouteTableId: !Ref PrivateRouteTable SubnetId: !Ref PrivateSubnet1 VPCEndpointForDMSS3Target: Type: AWS::EC2::VPCEndpoint Properties: VpcEndpointType: Gateway VpcId: !Ref VPC ServiceName: !Sub com.amazonaws.${AWS::Region}.s3 RouteTableIds: - !Ref PublicRouteTable DMSSG: Type: AWS::EC2::SecurityGroup DependsOn: - PrivateSubnet0 - PrivateSubnet1 Properties: VpcId: !Ref VPC GroupDescription: DMS security group Metadata: cfn_nag: rules_to_suppress: - id: F1000 reason: "All traffic is allowed outbound. It is the desired configuration." RDSSG: Type: 'AWS::EC2::SecurityGroup' DependsOn: - PrivateSubnet0 - PrivateSubnet1 Properties: VpcId: !Ref VPC GroupDescription: Limits security group ingress traffic SecurityGroupIngress: - IpProtocol: tcp FromPort: 3306 ToPort: 3306 CidrIp: !Ref VPCCIDR Description: "Access from the VPC" Metadata: cfn_nag: rules_to_suppress: - id: F1000 reason: "All traffic is allowed outbound. It is desired configuration" - id: W9 reason: "Whole VPC can access this database on a private subnet. Restricting access on the database itself" # # __ __ _ # # | \/ | __ _ ___(_) ___ # # | |\/| |/ _` |/ __| |/ _ \ # # | | | | (_| | (__| | __/ # # |_| |_|\__,_|\___|_|\___| # # MacieSession: Type: AWS::Macie::Session Properties: Status: ENABLED MacieCustomDataIdentifierAccountNumber: Type: AWS::Macie::CustomDataIdentifier DependsOn: "MacieSession" Properties: Description: "Checks for customer account number format" IgnoreWords: - "000000" Keywords: - "account_number" MaximumMatchDistance: 20 Name: "account_number" Regex: (XYZ-)\d{1,2} # _ ____ __ ____ # | |/ / \/ / ___| # | ' /| |\/| \___ \ # | . \| | | |___) | # |_|\_\_| |_|____/ # ConfidentialKey: DependsOn: MacieSession Type: 'AWS::KMS::Key' Properties: Description: Key for confidential bucket EnableKeyRotation: true Tags: - Key: Classification Value: Confidential KeyPolicy: Version: 2012-10-17 Id: key-default-2 Statement: - Sid: Enable IAM User Permissions Effect: Allow Principal: AWS: !Sub arn:aws:iam::${AWS::AccountId}:root Action: 'kms:*' Resource: '*' ConfidentialKeyAlias: Type: 'AWS::KMS::Alias' Properties: AliasName: !Sub 'alias/${AWS::StackName}-confidential-bucket-encryption-key' TargetKeyId: Ref: ConfidentialKey KeyToApplication: Type: 'AWS::KMS::Key' Properties: Description: symmetric encryption KMS key EnableKeyRotation: true KeyPolicy: Version: 2012-10-17 Id: key-teste Statement: - Sid: Enable IAM User Permissions Effect: Allow Principal: AWS: !Sub arn:aws:iam::${AWS::AccountId}:root Action: 'kms:*' Resource: '*' - Sid: Allow administration of the key Effect: Allow Principal: AWS: !GetAtt DataScienceUser.Arn Action: - 'kms:Create*' - 'kms:Describe*' - 'kms:Enable*' - 'kms:List*' - 'kms:Put*' - 'kms:Update*' - 'kms:Revoke*' - 'kms:Disable*' - 'kms:Get*' - 'kms:Delete*' - 'kms:ScheduleKeyDeletion' - 'kms:CancelKeyDeletion' Resource: '*' - Sid: Allow use of the key Effect: Allow Principal: AWS: - !GetAtt GlueJobRole.Arn - !GetAtt DataScienceUser.Arn Action: - 'kms:DescribeKey' - 'kms:Encrypt' - 'kms:Decrypt' - 'kms:ReEncrypt*' - 'kms:GenerateDataKey' - 'kms:GenerateDataKeyWithoutPlaintext' Resource: '*' - Sid: Allow attachment of persistent resources Effect: Allow Principal: AWS: - !GetAtt DataScienceUser.Arn Action: - kms:CreateGrant - kms:ListGrants - kms:RevokeGrant Resource: "*" Condition: Bool: kms:GrantIsForAWSResource: 'true' AliasKey: Type: 'AWS::KMS::Alias' Properties: AliasName: alias/encryptionDataRow TargetKeyId: !Ref KeyToApplication # # ____ _____ # # / ___|___ / # # \___ \ |_ \ # # ___) |__) | # # |____/____/ # # MacieBucket: Type: 'AWS::S3::Bucket' Properties: VersioningConfiguration: Status: Enabled BucketName: !Sub '${Prefix}-macie-${AWS::Region}-${AWS::AccountId}' BucketEncryption: ServerSideEncryptionConfiguration: - ServerSideEncryptionByDefault: SSEAlgorithm: AES256 LoggingConfiguration: DestinationBucketName: !Ref LoggingBucket LogFilePrefix: macie-logs DeletionPolicy: Retain Metadata: cfn_nag: rules_to_suppress: - id: W51 reason: "This is a private bucket." GlueBucket: Type: AWS::S3::Bucket Properties: VersioningConfiguration: Status: Enabled BucketName: !Sub '${Prefix}-glue-${AWS::Region}-${AWS::AccountId}' BucketEncryption: ServerSideEncryptionConfiguration: - ServerSideEncryptionByDefault: SSEAlgorithm: AES256 LoggingConfiguration: DestinationBucketName: !Ref LoggingBucket LogFilePrefix: glue-logs NotificationConfiguration: LambdaConfigurations: - Event: 's3:ObjectCreated:*' Function: !GetAtt StartGlueWorkflowLambdaFunction.Arn DependsOn: - StartGlueWorkflowLambdaFunctionPermission Metadata: cfn_nag: rules_to_suppress: - id: W51 reason: "This is a private bucket." AthenaBucket: Type: AWS::S3::Bucket Properties: VersioningConfiguration: Status: Enabled BucketName: !Sub '${Prefix}-athena-${AWS::Region}-${AWS::AccountId}' BucketEncryption: ServerSideEncryptionConfiguration: - ServerSideEncryptionByDefault: SSEAlgorithm: AES256 LoggingConfiguration: DestinationBucketName: !Ref LoggingBucket LogFilePrefix: athena-logs Metadata: cfn_nag: rules_to_suppress: - id: W51 reason: "This is a private bucket." AssetsBucket: Type: AWS::S3::Bucket Properties: VersioningConfiguration: Status: Enabled BucketName: !Sub '${Prefix}-assets-${AWS::Region}-${AWS::AccountId}' BucketEncryption: ServerSideEncryptionConfiguration: - ServerSideEncryptionByDefault: SSEAlgorithm: AES256 LoggingConfiguration: DestinationBucketName: !Ref LoggingBucket LogFilePrefix: assets-logs DeletionPolicy: Retain Metadata: cfn_nag: rules_to_suppress: - id: W51 reason: "This is a private bucket." LoggingBucket: Type: AWS::S3::Bucket Properties: VersioningConfiguration: Status: Enabled BucketName: !Sub '${Prefix}-logging-${AWS::Region}-${AWS::AccountId}' # AccessControl: LogDeliveryWrite BucketEncryption: ServerSideEncryptionConfiguration: - ServerSideEncryptionByDefault: SSEAlgorithm: AES256 DeletionPolicy: Retain Metadata: cfn_nag: rules_to_suppress: - id: W51 reason: "This is a private bucket." - id: W35 reason: "This is the logging bucket for all other buckets. Does not require log to be enabled for this solution. Avoids recurring loop." MacieFindingsDeliveryStream: DependsOn: - DeliveryPolicy Type: AWS::KinesisFirehose::DeliveryStream Properties: ExtendedS3DestinationConfiguration: BucketARN: !Sub 'arn:aws:s3:::${GlueBucket}' BufferingHints: IntervalInSeconds: '60' SizeInMBs: '1' CompressionFormat: UNCOMPRESSED Prefix: '' RoleARN: !GetAtt DeliveryRole.Arn DeletionPolicy: Retain Metadata: cfn_nag: rules_to_suppress: - id: W88 reason: "Kinesis Firehose DeliveryStream not need to specify SSE because the bucket already using encryption." MacieFindingsEventRule: Type: AWS::Events::Rule Properties: Description: 'All Findings from Amazon Macie' EventPattern: source: - "aws.macie" detail-type: - "Macie Finding" State: "ENABLED" Name: MacieFindinfs2Firehose Targets: - Arn: !GetAtt - MacieFindingsDeliveryStream - Arn Id: MacieFindingsEventRule RoleArn: !GetAtt - EventDeliveryRole - Arn AthenaQueryBucket: Type: AWS::S3::Bucket Properties: VersioningConfiguration: Status: Enabled BucketName: Fn::Join: - "-" - - secure-athena-query - Ref: AWS::AccountId - Ref: AWS::Region BucketEncryption: ServerSideEncryptionConfiguration: - ServerSideEncryptionByDefault: SSEAlgorithm: AES256 LoggingConfiguration: DestinationBucketName: !Ref LoggingBucket LogFilePrefix: athena-queries-logs PublicAccessBlockConfiguration: BlockPublicAcls: true BlockPublicPolicy: true IgnorePublicAcls: true RestrictPublicBuckets: true LifecycleConfiguration: Rules: - Id: ExpirationRule Status: Enabled ExpirationInDays: '7' DeletionPolicy: Retain Metadata: cfn_nag: rules_to_suppress: - id: W51 reason: "This is a private bucket." - id: W35 reason: "Private bucket without logging configured." # ____ _ __ __ # / ___| ___ ___ _ __ ___| |_ ___ | \/ | __ _ _ __ __ _ __ _ ___ _ __ # \___ \ / _ \/ __| '__/ _ \ __/ __| | |\/| |/ _` | '_ \ / _` |/ _` |/ _ \ '__| # ___) | __/ (__| | | __/ |_\__ \ | | | | (_| | | | | (_| | (_| | __/ | # |____/ \___|\___|_| \___|\__|___/ |_| |_|\__,_|_| |_|\__,_|\__, |\___|_| # |___/ MySQLDBSecret: Type: 'AWS::SecretsManager::Secret' Properties: Name: !Sub '${Prefix}-MySQLDB-Secret' Description: "This secret has a dynamically generated secret password." GenerateSecretString: SecretStringTemplate: !Sub '{"username": "${DBUser}"}' GenerateStringKey: "password" PasswordLength: 30 ExcludePunctuation: true Metadata: cfn_nag: rules_to_suppress: - id: W77 reason: "Secrets Manager uses the key aws/secretsmanager." # ____ ____ ____ # | _ \| _ \/ ___| # | |_) | | | \___ \ # | _ <| |_| |___) | # |_| \_\____/|____/ # MySQLDB: Type: 'AWS::RDS::DBInstance' Properties: DBInstanceIdentifier: 'mydbinstance' DBName: !Ref DBName DBInstanceClass: db.t4g.small AllocatedStorage: '20' Engine: MySQL EngineVersion: 8.0.28 MasterUsername: !Sub '{{resolve:secretsmanager:${MySQLDBSecret}::username}}' MasterUserPassword: !Sub '{{resolve:secretsmanager:${MySQLDBSecret}::password}}' DBSubnetGroupName: !Ref DBSubnetGroup PubliclyAccessible : false VPCSecurityGroups: - !Ref RDSSG StorageType: gp2 StorageEncrypted: true DBParameterGroupName: !Ref MySQLDBParameterGroup Metadata: cfn_nag: rules_to_suppress: - id: W28 reason: "Explicit name to use on instructions with the users." - id: F80 reason: "database for test so could delete, it's not necessary have deletion protection enabled" DBSubnetGroup: Type: 'AWS::RDS::DBSubnetGroup' Properties: DBSubnetGroupDescription: DBSubnetGroup for RDS Instance SubnetIds: - !Ref PrivateSubnet0 - !Ref PrivateSubnet1 MySQLDBParameterGroup: Type: 'AWS::RDS::DBParameterGroup' Properties: Description: DCP Parameter Group Family: mysql8.0 Parameters: log_bin_trust_function_creators: 1 # _____ ____ ____ # | ____/ ___|___ \ # | _|| | __) | # | |__| |___ / __/ # |_____\____|_____| # RDSLoaderInstance: Type: AWS::EC2::Instance DependsOn: MySQLDB Properties: ImageId: !Ref LatestAmzn2AmiId InstanceType: t4g.small IamInstanceProfile: !Ref RDSLoaderInstanceProfile SubnetId: !Ref PublicSubnet0 SecurityGroupIds: - !Ref RDSSG UserData: Fn::Base64: !Sub | #!/bin/bash yum install -y mysql jq python3 aws --profile default configure set region ${AWS::Region} export AWS_DEFAULT_REGION=${AWS::Region} DB_USER=$(aws secretsmanager get-secret-value --secret-id dcp-MySQLDB-Secret --query SecretString --output text | jq -r '.username') DB_PASS=$(aws secretsmanager get-secret-value --secret-id dcp-MySQLDB-Secret --query SecretString --output text | jq -r '.password') cd /tmp wget https://s3.amazonaws.com/rds-downloads/rds-ca-2019-root.pem wget https://raw.githubusercontent.com/aws-samples/data-masking-fine-grained-access-using-aws-lake-formation/main/scripts/fake-dataset.py wget https://raw.githubusercontent.com/aws-samples/data-masking-fine-grained-access-using-aws-lake-formation/main/scripts/load-data-into-mysql.sql python3 -m venv venv source ./venv/bin/activate pip install Faker pip install pandas pip install numpy python3 fake-dataset.py deactivate mysql -u $DB_USER -p$DB_PASS -h ${MySQLDB.Endpoint.Address} -P ${MySQLDB.Endpoint.Port} --ssl-ca=rds-ca-2019-root.pem --ssl < /tmp/load-data-into-mysql.sql; sudo shutdown -h now Tags: - Key: Name Value: !Sub '${Prefix}-RDS-Loader' # # ____ __ __ ____ # # | _ \| \/ / ___| # # | | | | |\/| \___ \ # # | |_| | | | |___) | # # |____/|_| |_|____/ # # DMSReplicationInstance: Type: "AWS::DMS::ReplicationInstance" Properties: ReplicationInstanceClass: dms.t3.small PubliclyAccessible: false AvailabilityZone: !GetAtt PublicSubnet0.AvailabilityZone ReplicationSubnetGroupIdentifier: !Ref DMSReplicationSubnetGroup VpcSecurityGroupIds: - !Ref DMSSG Metadata: info: "DMS already uses encryption at rest by default. If you want to use your own generated encryption key please look at this documentation: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-dms-replicationinstance.html#cfn-dms-replicationinstance-kmskeyid" DMSReplicationSubnetGroup: Type: "AWS::DMS::ReplicationSubnetGroup" Properties: ReplicationSubnetGroupDescription: DMS Subnet Group ReplicationSubnetGroupIdentifier: dmssubnetgroup-demo SubnetIds: - !Ref PublicSubnet0 - !Ref PublicSubnet1 RDSCA2019RootDMSCertificate: Type: AWS::DMS::Certificate Properties: CertificateIdentifier: rds-ca-2019-root CertificatePem: |- -----BEGIN CERTIFICATE----- MIIEBjCCAu6gAwIBAgIJAMc0ZzaSUK51MA0GCSqGSIb3DQEBCwUAMIGPMQswCQYD VQQGEwJVUzEQMA4GA1UEBwwHU2VhdHRsZTETMBEGA1UECAwKV2FzaGluZ3RvbjEi MCAGA1UECgwZQW1hem9uIFdlYiBTZXJ2aWNlcywgSW5jLjETMBEGA1UECwwKQW1h em9uIFJEUzEgMB4GA1UEAwwXQW1hem9uIFJEUyBSb290IDIwMTkgQ0EwHhcNMTkw ODIyMTcwODUwWhcNMjQwODIyMTcwODUwWjCBjzELMAkGA1UEBhMCVVMxEDAOBgNV BAcMB1NlYXR0bGUxEzARBgNVBAgMCldhc2hpbmd0b24xIjAgBgNVBAoMGUFtYXpv biBXZWIgU2VydmljZXMsIEluYy4xEzARBgNVBAsMCkFtYXpvbiBSRFMxIDAeBgNV BAMMF0FtYXpvbiBSRFMgUm9vdCAyMDE5IENBMIIBIjANBgkqhkiG9w0BAQEFAAOC AQ8AMIIBCgKCAQEArXnF/E6/Qh+ku3hQTSKPMhQQlCpoWvnIthzX6MK3p5a0eXKZ oWIjYcNNG6UwJjp4fUXl6glp53Jobn+tWNX88dNH2n8DVbppSwScVE2LpuL+94vY 0EYE/XxN7svKea8YvlrqkUBKyxLxTjh+U/KrGOaHxz9v0l6ZNlDbuaZw3qIWdD/I 6aNbGeRUVtpM6P+bWIoxVl/caQylQS6CEYUk+CpVyJSkopwJlzXT07tMoDL5WgX9 O08KVgDNz9qP/IGtAcRduRcNioH3E9v981QO1zt/Gpb2f8NqAjUUCUZzOnij6mx9 McZ+9cWX88CRzR0vQODWuZscgI08NvM69Fn2SQIDAQABo2MwYTAOBgNVHQ8BAf8E BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUc19g2LzLA5j0Kxc0LjZa pmD/vB8wHwYDVR0jBBgwFoAUc19g2LzLA5j0Kxc0LjZapmD/vB8wDQYJKoZIhvcN AQELBQADggEBAHAG7WTmyjzPRIM85rVj+fWHsLIvqpw6DObIjMWokpliCeMINZFV ynfgBKsf1ExwbvJNzYFXW6dihnguDG9VMPpi2up/ctQTN8tm9nDKOy08uNZoofMc NUZxKCEkVKZv+IL4oHoeayt8egtv3ujJM6V14AstMQ6SwvwvA93EP/Ug2e4WAXHu cbI1NAbUgVDqp+DRdfvZkgYKryjTWd/0+1fS8X1bBZVWzl7eirNVnHbSH2ZDpNuY 0SBd8dj5F6ld3t58ydZbrTHze7JJOd8ijySAp4/kiu9UfZWuTPABzDa/DSdz9Dk/ zPW4CXXvhLmE02TA9/HeCw3KEHIwicNuEfw= -----END CERTIFICATE----- DBEndpoint: Type: "AWS::DMS::Endpoint" Properties: DatabaseName: !Ref DBName EndpointType: source EngineName: mysql Password: !Sub '{{resolve:secretsmanager:${MySQLDBSecret}::password}}' Port: !GetAtt MySQLDB.Endpoint.Port ServerName: !GetAtt MySQLDB.Endpoint.Address Username: !Ref DBUser SslMode: verify-ca CertificateArn: !Ref RDSCA2019RootDMSCertificate S3Endpoint: Type: AWS::DMS::Endpoint Properties: EndpointType: target EngineName: s3 ExtraConnectionAttributes: "addColumnName=true;compressionType=GZIP;dataFormat=parquet;" S3Settings: BucketName: !Ref MacieBucket ServiceAccessRoleArn: !Sub arn:aws:iam::${AWS::AccountId}:role/dms-vpc-role RDSToS3Task: Type: 'AWS::DMS::ReplicationTask' Properties: MigrationType: full-load ReplicationInstanceArn: !Ref DMSReplicationInstance SourceEndpointArn: !Ref DBEndpoint TargetEndpointArn: !Ref S3Endpoint ReplicationTaskSettings: | { "TargetMetadata": { "SupportLobs": true }, "FullLoadSettings": { "TargetTablePrepMode": "DROP_AND_CREATE" }, "Logging": { "EnableLogging": true } } TableMappings: !Sub | { "rules": [ { "rule-type": "selection", "rule-id": "1", "rule-name": "1", "object-locator": { "schema-name": "${DBName}", "table-name": "%" }, "rule-action": "include", "filters": [] } ] } # _ _ _ # | | __ _ _ __ ___ | |__ __| | __ _ # | | / _` | '_ ` _ \| '_ \ / _` |/ _` | # | |__| (_| | | | | | | |_) | (_| | (_| | # |_____\__,_|_| |_| |_|_.__/ \__,_|\__,_| # StartGlueWorkflowLambdaFunction: Type: AWS::Lambda::Function Properties: Architectures: - arm64 Runtime: nodejs16.x Handler: index.handler MemorySize: 256 FunctionName: !Sub '${Prefix}-start-glue-workflow' Description: Data Classification Pipeline - Start Glue Workflow Role: !GetAtt StartGlueWorkflowLambdaRole.Arn Environment: Variables: WORKFLOW_NAME: !Ref GlueWorkflow Code: ZipFile: | const AWS = require('aws-sdk'); var glue = new AWS.Glue(); const WORKFLOW_NAME = process.env.WORKFLOW_NAME; exports.handler = async(event) => { let s3 = event.Records[0].s3; console.log(`Starting DCP Glue Workflow for s3://${s3.bucket.name}/${s3.object.key}`); return await glue.startWorkflowRun({ Name: WORKFLOW_NAME }).promise(); }; Metadata: cfn_nag: rules_to_suppress: - id: W89 reason: "This Lambda function is not necessary to be deployed inside a VPC." - id: W92 reason: "It's not necessary define ReservedConcurrentExecutions to reserve simultaneous executions." StartGlueWorkflowLambdaFunctionPermission: Type: AWS::Lambda::Permission Properties: Action: lambda:InvokeFunction FunctionName: !Ref StartGlueWorkflowLambdaFunction Principal: s3.amazonaws.com SourceAccount: !Ref AWS::AccountId SourceArn: !Sub 'arn:aws:s3:::${Prefix}-glue-${AWS::Region}-${AWS::AccountId}' CreateGlueScriptCustomResource: Type: Custom::CustomResource Properties: ServiceToken: !GetAtt CreateGlueScriptLambdaFunction.Arn CreateGlueScriptLambdaFunction: Type: AWS::Lambda::Function Properties: Runtime: nodejs16.x Handler: index.handler MemorySize: 256 FunctionName: !Sub '${Prefix}-create-upload-glue-script' Description: Data Classification Pipeline - Upload Glue script into S3 Role: !GetAtt CreateGlueScriptLambdaRole.Arn Environment: Variables: PREFIX: !Sub ${Prefix} ATHENA_BUCKET: !Sub ${AthenaBucket.Arn} ASSETS_BUCKET: !Ref AssetsBucket ACCOUNT_ID: !Ref AWS::AccountId KEY_NAME: 'alias/encryptionDataRow' SCRIPT_URL: https://raw.githubusercontent.com/aws-samples/data-masking-fine-grained-access-using-aws-lake-formation/main/scripts/glue-job-mask-data-script.py Code: ZipFile: | const https = require('https'); const url = require('url'); const AWS = require('aws-sdk'); const s3 = new AWS.S3(); const FUNCTION_TIMEOUT = 10 * 1000; const ATHENA_BUCKET = process.env.ATHENA_BUCKET; const ASSETS_BUCKET = process.env.ASSETS_BUCKET; const ACCOUNT_ID = process.env.ACCOUNT_ID; const SCRIPT_URL = process.env.SCRIPT_URL; const SCRIPT_KEY = 'scripts/dcp-script.py'; const AWS_REGION = process.env.AWS_REGION.replace(/-/g, '_'); const KEY_NAME= process.env.KEY_NAME; const PREFIX= process.env.PREFIX; exports.handler = async(event, context) => { logRequest(event, context); try { setTimeoutWatchDog(event, context); let script = await downloadGlueScript(); script = script.replace(/PREFIX/g, `${PREFIX}`); script = script.replace(/ATHENA_BUCKET/g, `${ATHENA_BUCKET}`); script = script.replace(/AWS_REGION/g, `${AWS_REGION}`); script = script.replace(/ACCOUNT_ID/g, `${ACCOUNT_ID}`); script = script.replace(/KEY_NAME/g, `${ACCOUNT_ID}`); let params = { Key: SCRIPT_KEY, Bucket: ASSETS_BUCKET, Body: script }; if (event.RequestType == 'Create' || event.RequestType == 'Update') { console.log('Creating Glue script.'); await s3.putObject(params).promise(); } await sendCloudFormationResponse(event, context, "SUCCESS"); } catch (err) { await sendCloudFormationResponse(event, context, "FAILED", err); } }; function logRequest(event, context) { console.log(`"${event.StackId}" "${event.RequestId}" "${context.logStreamName}" "${event.LogicalResourceId}" "${event.ResponseURL}"`); } function setTimeoutWatchDog(event, context) { const timeoutHandler = async() => { await sendCloudFormationResponse(event, context, "FAILED", { 'error': 'Resource timeout' }); }; setTimeout(timeoutHandler, FUNCTION_TIMEOUT); } async function downloadGlueScript() { return new Promise((resolve, reject) => { https.get(SCRIPT_URL, (response) => { let data = ''; response.on('data', (chunk) => data += chunk); response.on('error', (err) => reject(err)); response.on('end', () => resolve(data)); }).on('error', reject); }); } async function sendCloudFormationResponse(event, context, responseStatus, responseData, physicalResourceId, noEcho) { return new Promise((resolve, reject) => { let responseBody = JSON.stringify({ Status: responseStatus, Reason: "See the details in CloudWatch Log Stream: " + context.logStreamName, PhysicalResourceId: physicalResourceId || context.logStreamName, StackId: event.StackId, RequestId: event.RequestId, LogicalResourceId: event.LogicalResourceId, NoEcho: noEcho || false, Data: responseData }); console.log("CFN Payload:\n", responseBody); let parsedUrl = url.parse(event.ResponseURL); let options = { hostname: parsedUrl.hostname, port: 443, path: parsedUrl.path, method: "PUT", headers: { "content-type": "", "content-length": responseBody.length } }; let request = https.request(options, function(response) { console.log(`CFN Response: ${response.statusCode} ${response.statusMessage}`); resolve(context.done()); }); request.on("error", function(error) { console.log("send(..) failed executing https.request(..): " + error); reject(context.done(error)); }); request.write(responseBody); request.end(); }) } Metadata: cfn_nag: rules_to_suppress: - id: W89 reason: "This Lambda function is not necessary to be deployed inside a VPC." - id: W92 reason: "It's not necessary define ReservedConcurrentExecutions to reserve simultaneous executions." # # ____ _ # # / ___| |_ _ ___ # # | | _| | | | |/ _ \ # # | |_| | | |_| | __/ # # \____|_|\__,_|\___| # # GlueDatabase: Type: AWS::Glue::Database Properties: CatalogId: !Ref AWS::AccountId DatabaseInput: Name: !Sub '${Prefix}' Description: 'Data Classification Pipeline' GlueDatabaseDataset: Type: AWS::Glue::Database Properties: CatalogId: !Ref AWS::AccountId DatabaseInput: Name: dataset Description: 'Data Classification Pipeline' GlueDatabaseDatasetMasked: Type: AWS::Glue::Database Properties: CatalogId: !Ref AWS::AccountId DatabaseInput: Name: dataset_masked Description: 'Database with dataset masked' GlueDatabaseDatasetEncrypted: Type: AWS::Glue::Database Properties: CatalogId: !Ref AWS::AccountId DatabaseInput: Name: dataset_encrypted Description: 'Database with dataset encrypted' GlueInputTableCrawlerDataset: Type: AWS::Glue::Crawler Properties: Role: !GetAtt GlueJobRole.Arn Description: Data Classification Pipeline Input Table Crawler DatabaseName: !Ref GlueDatabaseDataset Targets: S3Targets: - Path: !Sub 's3://${MacieBucket}/dataset' Name: dcp-macie-crawler SchemaChangePolicy: UpdateBehavior: "UPDATE_IN_DATABASE" DeleteBehavior: "DEPRECATE_IN_DATABASE" Configuration: "{\"Version\":1.0,\"CrawlerOutput\":{\"Partitions\":{\"AddOrUpdateBehavior\":\"InheritFromTable\"},\"Tables\":{\"AddOrUpdateBehavior\":\"MergeNewColumns\"}}}" GlueInputTableCrawler: Type: AWS::Glue::Crawler Properties: Role: !GetAtt GlueJobRole.Arn Description: Data Classification Pipeline Input Table Crawler DatabaseName: !Ref GlueDatabase Targets: S3Targets: - Path: !Ref GlueBucket Name: dcp-glue-crawler SchemaChangePolicy: UpdateBehavior: "UPDATE_IN_DATABASE" DeleteBehavior: "DEPRECATE_IN_DATABASE" Configuration: "{\"Version\":1.0,\"Grouping\":{\"TableGroupingPolicy\":\"CombineCompatibleSchemas\"},\"CrawlerOutput\":{\"Partitions\":{\"AddOrUpdateBehavior\":\"InheritFromTable\"},\"Tables\":{\"AddOrUpdateBehavior\":\"MergeNewColumns\"}}}" GlueOutputTableCrawlerMasked: Type: AWS::Glue::Crawler Properties: Role: !GetAtt GlueJobRole.Arn Description: Data Classification Pipeline Output Table Crawler DatabaseName: dataset_masked Targets: S3Targets: - Path: !Sub 's3://${AthenaBucket}/masked' TablePrefix: '' Name: dcp-athena-dataset-masked-crawler SchemaChangePolicy: UpdateBehavior: "UPDATE_IN_DATABASE" DeleteBehavior: "LOG" Configuration: "{\"Version\":1.0,\"CrawlerOutput\":{\"Partitions\":{\"AddOrUpdateBehavior\":\"InheritFromTable\"},\"Tables\":{\"AddOrUpdateBehavior\":\"MergeNewColumns\"}}}" GlueOutputTableCrawlerEncrypted: Type: AWS::Glue::Crawler Properties: Role: !GetAtt GlueJobRole.Arn Description: Data Classification Pipeline Output Table Crawler DatabaseName: dataset_encrypted Targets: S3Targets: - Path: !Sub 's3://${AthenaBucket}/encrypted' TablePrefix: '' Name: dcp-athena-dataset-encrypted-crawler SchemaChangePolicy: UpdateBehavior: "UPDATE_IN_DATABASE" DeleteBehavior: "LOG" Configuration: "{\"Version\":1.0,\"CrawlerOutput\":{\"Partitions\":{\"AddOrUpdateBehavior\":\"InheritFromTable\"},\"Tables\":{\"AddOrUpdateBehavior\":\"MergeNewColumns\"}}}" GlueJob: Type: AWS::Glue::Job Properties: Name: 'dcp-etl-job' Description: 'Data Classification Pipeline ETL Job' Role: !GetAtt GlueJobRole.Arn Command: Name: glueetl PythonVersion: 3 ScriptLocation: !Sub 's3://${AssetsBucket}/scripts/dcp-script.py' DefaultArguments: "--TempDir": !Sub "s3://${AssetsBucket}/temporary/" "--class": "GlueApp" "--enable-continuous-cloudwatch-log": "true" "--enable-metrics": "true" "--enable-spark-ui": "true" "--job-bookmark-option": "job-bookmark-enable" "--job-language": "python" "--spark-event-logs-path": !Sub "s3://${AssetsBucket}/sparkHistoryLogs/" WorkerType: 'G.1X' NumberOfWorkers: 2 MaxRetries: 1 GlueVersion: '2.0' DependsOn: CreateGlueScriptCustomResource GlueWorkflow: Type: AWS::Glue::Workflow Properties: Description: 'Data Classification Pipeline Workflow' Name: 'dcp-workflow' GlueWFStartTrigger: Type: AWS::Glue::Trigger Properties: Description: 'Start Trigger' Name: start-workflow Type: ON_DEMAND Actions: - CrawlerName: !Ref GlueInputTableCrawlerDataset WorkflowName: !Ref GlueWorkflow GlueWFPostInputCrawlerDatasetTrigger: Type: AWS::Glue::Trigger Properties: Name: 'Post Input Crawler Success Condition' Type: "CONDITIONAL" StartOnCreation: true Actions: - CrawlerName: !Ref GlueInputTableCrawler Predicate: Conditions: - LogicalOperator: EQUALS CrawlerName: !Ref GlueInputTableCrawlerDataset CrawlState: SUCCEEDED Logical: ANY WorkflowName: !Ref GlueWorkflow DependsOn: GlueJob GlueWFPostInputCrawlerTrigger: Type: AWS::Glue::Trigger Properties: Name: 'Post Input Crawler Dataset Success Condition' Type: "CONDITIONAL" StartOnCreation: true Actions: - JobName: !Ref GlueJob Predicate: Conditions: - LogicalOperator: EQUALS CrawlerName: !Ref GlueInputTableCrawler CrawlState: SUCCEEDED Logical: ANY WorkflowName: !Ref GlueWorkflow GlueWFPostETLJobTrigger: Type: AWS::Glue::Trigger Properties: Name: 'Post ETL Job Success Condition' Type: "CONDITIONAL" StartOnCreation: true Actions: - CrawlerName: !Ref GlueOutputTableCrawlerMasked Predicate: Conditions: - LogicalOperator: EQUALS JobName: !Ref GlueJob State: 'SUCCEEDED' Logical: 'ANY' WorkflowName: !Ref GlueWorkflow GlueWFPostInputCrawlerMaskedTrigger: Type: AWS::Glue::Trigger Properties: Name: 'Post Input Crawler Masked Success Condition' Type: "CONDITIONAL" StartOnCreation: true Actions: - CrawlerName: !Ref GlueOutputTableCrawlerEncrypted Predicate: Conditions: - LogicalOperator: EQUALS CrawlerName: !Ref GlueOutputTableCrawlerMasked CrawlState: SUCCEEDED Logical: ANY WorkflowName: !Ref GlueWorkflow # |_ _| / \ | \/ | # | | / _ \ | |\/| | # | | / ___ \| | | | # |___/_/ \_\_| |_| # RDSLoaderInstanceProfile: Type: "AWS::IAM::InstanceProfile" Properties: Path: "/" Roles: - !Ref RDSLoaderRole RDSLoaderRole: Type: 'AWS::IAM::Role' Properties: RoleName: !Sub - ${Prefix}-RDSLoaderRole-${Hash} - { Hash: !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref 'AWS::StackId']]]] } AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Principal: Service: - ec2.amazonaws.com Action: - 'sts:AssumeRole' Path: / ManagedPolicyArns: - 'arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore' Policies: - PolicyName: !Sub - ${Prefix}-SecretsManager-${Hash} - { Hash: !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref 'AWS::StackId']]]] } PolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Action: - secretsmanager:GetResourcePolicy - secretsmanager:GetSecretValue - secretsmanager:DescribeSecret Resource: - !Ref MySQLDBSecret Metadata: cfn_nag: rules_to_suppress: - id: W28 reason: "Explicit name to use on instructions with the users." RDSServiceRole: Type: "AWS::IAM::ServiceLinkedRole" # Condition: CreateRDSServiceRole Properties: AWSServiceName: "rds.amazonaws.com" Description: Allows Amazon RDS to manage AWS resources on your behalf DMSAccessRole: Type: "AWS::IAM::Role" Condition: CreateDMSVPCRole Properties: RoleName: 'dms-vpc-role' AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Action: sts:AssumeRole Effect: Allow Principal: Service: - dms.amazonaws.com ManagedPolicyArns: - 'arn:aws:iam::aws:policy/service-role/AmazonDMSVPCManagementRole' Metadata: cfn_nag: rules_to_suppress: - id: W28 reason: "Explicit name to use on instructions with the users." DMStoS3Policy: Type: AWS::IAM::Policy Properties: PolicyName: !Sub - ${Prefix}-DMStoS3Policy-${Hash} - { Hash: !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref 'AWS::StackId']]]] } PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Action: - s3:PutObject - s3:DeleteObject - s3:PutObjectTagging Resource: - !GetAtt MacieBucket.Arn - !Sub "${MacieBucket.Arn}/*" - Effect: Allow Action: - s3:ListBucket Resource: - !GetAtt MacieBucket.Arn - !Sub "${MacieBucket.Arn}" Roles: - dms-vpc-role GlueJobRole: Type: AWS::IAM::Role Properties: RoleName: !Sub - ${Prefix}-GlueJobRole-${Hash} - { Hash: !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref 'AWS::StackId']]]] } AssumeRolePolicyDocument: Version: '2012-10-17' Statement: - Action: 'sts:AssumeRole' Effect: 'Allow' Principal: Service: - glue.amazonaws.com - lakeformation.amazonaws.com ManagedPolicyArns: - 'arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole' Policies: - PolicyName: !Sub - ${Prefix}-S3ReadWriteAccess-${Hash} - { Hash: !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref 'AWS::StackId']]]] } PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Action: - 's3:ListBucket' - 's3:GetObject' - 's3:PutObject' - 's3:DeleteObject' Resource: - !Sub ${GlueBucket.Arn} - !Sub ${AthenaBucket.Arn} - !Sub ${MacieBucket.Arn} - !Sub ${AssetsBucket.Arn} - !Sub ${GlueBucket.Arn}/* - !Sub ${AthenaBucket.Arn}/* - !Sub ${AssetsBucket.Arn}/* - !Sub ${MacieBucket.Arn}/* Metadata: cfn_nag: rules_to_suppress: - id: W28 reason: "Explicit name to use on instructions with the users." StartGlueWorkflowLambdaRole: Type: AWS::IAM::Role Properties: RoleName: !Sub - ${Prefix}-StartGlueWorkflowLambdaRole-${Hash} - { Hash: !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref 'AWS::StackId']]]] } AssumeRolePolicyDocument: Version: '2012-10-17' Statement: - Action: 'sts:AssumeRole' Effect: 'Allow' Principal: Service: 'lambda.amazonaws.com' ManagedPolicyArns: - 'arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole' Policies: - PolicyName: !Sub - ${Prefix}-StartGlueWorkflow-${Hash} - { Hash: !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref 'AWS::StackId']]]] } PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Action: - 'glue:StartWorkflowRun' Resource: - "*" Metadata: cfn_nag: rules_to_suppress: - id: W28 reason: "Explicit name to use on instructions with the users." - id: W11 reason: "Policy to lambda." IngestDataIntoRDSLambdaRole: Type: AWS::IAM::Role Properties: RoleName: !Sub - ${Prefix}-IngestDataIntoRDSLambdaRole-${Hash} - { Hash: !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref 'AWS::StackId']]]] } AssumeRolePolicyDocument: Version: '2012-10-17' Statement: - Action: 'sts:AssumeRole' Effect: 'Allow' Principal: Service: 'lambda.amazonaws.com' ManagedPolicyArns: - 'arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole' Policies: - PolicyName: !Sub - ${Prefix}-SecretsManagerMySQLDBSecret-${Hash} - { Hash: !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref 'AWS::StackId']]]] } PolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Action: - secretsmanager:GetSecretValue - secretsmanager:DescribeSecret Resource: - !Ref MySQLDBSecret Metadata: cfn_nag: rules_to_suppress: - id: W28 reason: "Explicit name to use on instructions with the users." CreateGlueScriptLambdaRole: Type: AWS::IAM::Role Properties: RoleName: !Sub - ${Prefix}-GlueScriptLambdaRole-${Hash} - { Hash: !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref 'AWS::StackId']]]] } AssumeRolePolicyDocument: Version: '2012-10-17' Statement: - Action: 'sts:AssumeRole' Effect: 'Allow' Principal: Service: 'lambda.amazonaws.com' ManagedPolicyArns: - 'arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole' Policies: - PolicyName: !Sub - ${Prefix}-S3PutObject-${Hash} - { Hash: !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref 'AWS::StackId']]]] } PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Action: - 's3:PutObject' Resource: - !Sub '${AssetsBucket.Arn}/*' Metadata: cfn_nag: rules_to_suppress: - id: W28 reason: "Explicit name to use on instructions with the users." DeliveryRole: Type: AWS::IAM::Role Properties: RoleName: !Sub - ${Prefix}-KinesisRole-${Hash} - { Hash: !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref 'AWS::StackId']]]] } AssumeRolePolicyDocument: Version: 2012-10-17 Statement: - Sid: '' Effect: Allow Principal: Service: firehose.amazonaws.com Action: 'sts:AssumeRole' Condition: StringEquals: 'sts:ExternalId': !Ref 'AWS::AccountId' Metadata: cfn_nag: rules_to_suppress: - id: W28 reason: "Explicit name to use on instructions with the users." DeliveryPolicy: Type: AWS::IAM::Policy Properties: PolicyName: !Sub - ${Prefix}-FirehoseDeliveryPolicy-${Hash} - { Hash: !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref 'AWS::StackId']]]] } PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Action: - 's3:AbortMultipartUpload' - 's3:GetBucketLocation' - 's3:GetObject' - 's3:ListBucket' - 's3:ListBucketMultipartUploads' - 's3:PutObject' Resource: - !Sub 'arn:aws:s3:::${GlueBucket}' - !Sub 'arn:aws:s3:::${GlueBucket}/*' Roles: - !Ref DeliveryRole EventDeliveryRole: Type: AWS::IAM::Role Properties: RoleName: !Sub - ${Prefix}-CloudWatchRole-${Hash} - { Hash: !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref 'AWS::StackId']]]] } AssumeRolePolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Principal: Service: events.amazonaws.com Action: 'sts:AssumeRole' Metadata: cfn_nag: rules_to_suppress: - id: W28 reason: "Explicit name to use on instructions with the users." EventDeliveryPolicy: Type: AWS::IAM::Policy Properties: PolicyName: !Sub - ${Prefix}-EventFirehoseDeliveryPolicy-${Hash} - { Hash: !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref 'AWS::StackId']]]] } PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Action: - 'firehose:PutRecord' - 'firehose:PutRecordBatch' Resource: - !Sub 'arn:aws:firehose:${AWS::Region}:${AWS::AccountId}:deliverystream/${MacieFindingsDeliveryStream}' Roles: - !Ref EventDeliveryRole lfAdminGroup: Type: 'AWS::IAM::Group' Properties: GroupName: lfAdminGroup Path: '/' ManagedPolicyArns: - arn:aws:iam::aws:policy/AWSLakeFormationDataAdmin - arn:aws:iam::aws:policy/AWSGlueConsoleFullAccess - arn:aws:iam::aws:policy/CloudWatchLogsReadOnlyAccess - arn:aws:iam::aws:policy/AWSLakeFormationCrossAccountManager - arn:aws:iam::aws:policy/AmazonAthenaFullAccess - arn:aws:iam::aws:policy/AmazonS3FullAccess Policies: - PolicyName: !Sub - ${Prefix}-DataAccessPolicy-lfAdminGroup-${Hash} - { Hash: !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref 'AWS::StackId']]]] } PolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Action: - lakeformation:GetDataAccess - lakeformation:GrantPermissions - lakeformation:RevokePermissions - lakeformation:BatchGrantPermissions - lakeformation:BatchRevokePermissions - lakeformation:ListPermissions - lakeformation:GetResourceLFTags - lakeformation:ListLFTags - lakeformation:GetLFTag - lakeformation:SearchTablesByLFTags - lakeformation:SearchDatabasesByLFTags - iam:CreateRole - iam:CreatePolicy - iam:AttachRolePolicy Resource: "*" Metadata: cfn_nag: rules_to_suppress: - id: W28 reason: "Explicit name to use on instructions with the users." lfPersonaGroup: Type: 'AWS::IAM::Group' Properties: GroupName: lfPersonaGroup Path: '/' ManagedPolicyArns: - arn:aws:iam::aws:policy/AmazonAthenaFullAccess Policies: - PolicyName: !Sub - ${Prefix}-DataAccessPolicy-lfPersonaGroup-${Hash} - { Hash: !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref 'AWS::StackId']]]] } PolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Action: - lakeformation:GetDataAccess - lakeformation:GrantPermissions - lakeformation:RevokePermissions - lakeformation:BatchGrantPermissions - lakeformation:BatchRevokePermissions - lakeformation:ListPermissions - lakeformation:GetResourceLFTags - lakeformation:ListLFTags - lakeformation:GetLFTag - lakeformation:SearchTablesByLFTags - lakeformation:SearchDatabasesByLFTags - iam:CreateRole - iam:CreatePolicy - iam:AttachRolePolicy Resource: "*" - PolicyName: !Sub - ${Prefix}-AthenaBucketPolicy-${Hash} - { Hash: !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref 'AWS::StackId']]]] } PolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Action: - s3:Put* - s3:Get* - s3:List* Resource: - Fn::Join: - '' - - 'arn:aws:s3:::' - Ref: AthenaBucket - "/*" - PolicyName: !Sub - ${Prefix}-SecureAthenaQueryResult-${Hash} - { Hash: !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref 'AWS::StackId']]]] } PolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Action: - s3:Put* - s3:Get* - s3:List* Resource: - Fn::Join: - '' - - 'arn:aws:s3:::' - Ref: AthenaQueryBucket - "/*" Metadata: cfn_nag: rules_to_suppress: - id: W28 reason: "Explicit name to use on instructions with the users." LfAdminUser: Type: AWS::IAM::User DependsOn: lfAdminGroup Properties: Path: "/" LoginProfile: Password: !Ref TestUserPassword PasswordResetRequired: false UserName: secure-lf-admin Metadata: cfn_nag: rules_to_suppress: - id: W50 reason: "The user is not required to set a new password." - id: F51 reason: "It is a Ref to a NoEcho parameter without a default" AnalystUser: Type: AWS::IAM::User DependsOn: lfPersonaGroup Properties: Path: "/" LoginProfile: Password: !Ref TestUserPassword PasswordResetRequired: false UserName: secure-lf-business-analyst Metadata: cfn_nag: rules_to_suppress: - id: W50 reason: "The user is not required to set a new password." - id: F51 reason: "It is a Ref to a NoEcho parameter without a default" DataScienceUser: Type: AWS::IAM::User DependsOn: lfPersonaGroup Properties: Path: "/" LoginProfile: Password: !Ref TestUserPassword PasswordResetRequired: false UserName: secure-lf-data-scientist Metadata: cfn_nag: rules_to_suppress: - id: W50 reason: "The user is not required to set a new password." - id: F51 reason: "It is a Ref to a NoEcho parameter without a default" personaGroupAddition2: Type: 'AWS::IAM::UserToGroupAddition' DependsOn: lfPersonaGroup Properties: GroupName: lfPersonaGroup Users: - !Ref AnalystUser - !Ref DataScienceUser adminGroupAddition2: Type: 'AWS::IAM::UserToGroupAddition' DependsOn: lfAdminGroup Properties: GroupName: lfAdminGroup Users: - !Ref LfAdminUser # ___ _ _ # / _ \ _ _| |_ _ __ _ _| |_ ___ # | | | | | | | __| '_ \| | | | __/ __| # | |_| | |_| | |_| |_) | |_| | |_\__ \ # \___/ \__,_|\__| .__/ \__,_|\__|___/ # |_| Outputs: EndpointAddress: Description: Address of the RDS endpoint. Value: !GetAtt MySQLDB.Endpoint.Address EndpointPort: Description: Port of the RDS endpoint. Value: !GetAtt MySQLDB.Endpoint.Port