{ "Parameters": { "StorageLocationBase": { "Type": "String", "Default": "my-bucket/my-dl", "Description": "The logging base path to create an S3 bucket with default encryption for CDP. By default CDP will create the optional subdirectory in the bucket. It is possible to either use the same bucket or different buckets for StorageLocationBase and LogsLocationBase." }, "LogsLocationBase": { "Type": "String", "Default": "my-bucket/my-dl", "Description": "The storage base path to create an S3 bucket with default encryption for CDP. By default CDP will create the optional subdirectory in the bucket. It is possible to either use the same bucket or different buckets for StorageLocationBase and LogsLocationBase." }, "prefix": { "Type": "String", "Default": "replace-me", "Description": "prefix for IAM objects, separated by a dash. Dynamo table will be prefix-dynamodb-table" }, "s3KmsEncyption": { "Type": "String", "Default": false, "Description": "If set to True S3 will be configured with AWS managed KMS server side encryption" } }, "Conditions": { "kms": { "Fn::Equals": [ { "Ref": "s3KmsEncyption" }, true ] }, "needBucket2": { "Fn::Not": [ { "Fn::Equals": [ { "Fn::Select" : [ "0", { "Fn::Split" : [ "/", { "Ref": "LogsLocationBase" } ] } ] }, { "Fn::Select" : [ "0", { "Fn::Split" : [ "/", { "Ref": "StorageLocationBase" } ] } ] } ] } ] } }, "Resources": { "CDPS3Bucket1": { "Type": "AWS::S3::Bucket", "Properties": { "BucketEncryption": { "ServerSideEncryptionConfiguration": [ { "ServerSideEncryptionByDefault": { "Fn::If": [ "kms", { "SSEAlgorithm": "aws:kms", "KMSMasterKeyID": { "Ref": "CdpKey" } }, { "SSEAlgorithm": "AES256" } ] } } ] }, "BucketName": { "Fn::Select" : [ "0", { "Fn::Split" : [ "/", { "Ref": "LogsLocationBase" } ] } ] }, "PublicAccessBlockConfiguration": { "BlockPublicAcls": true, "BlockPublicPolicy": true, "IgnorePublicAcls": true, "RestrictPublicBuckets": true } } }, "CDPS3Bucket2": { "Type": "AWS::S3::Bucket", "Condition": "needBucket2", "Properties": { "BucketEncryption": { "ServerSideEncryptionConfiguration": [ { "ServerSideEncryptionByDefault": { "Fn::If": [ "kms", { "SSEAlgorithm": "aws:kms", "KMSMasterKeyID": { "Ref": "CdpKey" } }, { "SSEAlgorithm": "AES256" } ] } } ] }, "BucketName": { "Fn::Select" : [ "0", { "Fn::Split" : [ "/", { "Ref": "StorageLocationBase" } ] } ] }, "PublicAccessBlockConfiguration": { "BlockPublicAcls": true, "BlockPublicPolicy": true, "IgnorePublicAcls": true, "RestrictPublicBuckets": true } } }, "IDbrokerAssumeRolePolicy": { "Type": "AWS::IAM::ManagedPolicy", "Properties": { "PolicyDocument": { "Version": "2012-10-17", "Statement": [ { "Sid": "VisualEditor0", "Effect": "Allow", "Action": [ "sts:AssumeRole" ], "Resource": [ "*" ] } ] }, "ManagedPolicyName": { "Fn::Join": [ "_", [ { "Ref": "prefix" }, "idbroker-assume-role-policy" ] ] } } }, "BucketAccessPolicy": { "Type": "AWS::IAM::ManagedPolicy", "Properties": { "PolicyDocument": { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "s3:CreateJob", "s3:GetAccountPublicAccessBlock", "s3:HeadBucket", "s3:ListJobs" ], "Resource": "*" }, { "Sid": "AllowListingOfDataLakeFolder", "Effect": "Allow", "Action": [ "s3:GetAccelerateConfiguration", "s3:GetAnalyticsConfiguration", "s3:GetBucketAcl", "s3:GetBucketCORS", "s3:GetBucketLocation", "s3:GetBucketLogging", "s3:GetBucketNotification", "s3:GetBucketPolicy", "s3:GetBucketPolicyStatus", "s3:GetBucketPublicAccessBlock", "s3:GetBucketRequestPayment", "s3:GetBucketTagging", "s3:GetBucketVersioning", "s3:GetBucketWebsite", "s3:GetEncryptionConfiguration", "s3:GetInventoryConfiguration", "s3:GetLifecycleConfiguration", "s3:GetMetricsConfiguration", "s3:GetObject", "s3:GetObjectAcl", "s3:GetObjectTagging", "s3:GetObjectVersion", "s3:GetObjectVersionAcl", "s3:GetObjectVersionTagging", "s3:GetReplicationConfiguration", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:ListMultipartUploadParts" ], "Resource": [ { "Fn::Join": [ "", [ "arn:aws:s3:::", { "Fn::Select" : [ "0", { "Fn::Split" : [ "/", { "Ref": "StorageLocationBase" } ] } ] } ] ] }, { "Fn::Join": [ "", [ "arn:aws:s3:::", { "Fn::Select" : [ "0", { "Fn::Split" : [ "/", { "Ref": "StorageLocationBase" } ] } ] }, "/*" ] ] } ] } ] }, "ManagedPolicyName": { "Fn::Join": [ "-", [ { "Ref": "prefix" }, "bucket-access-policy" ] ] } } }, "DynamoDBPolicy": { "Type": "AWS::IAM::ManagedPolicy", "Properties": { "PolicyDocument": { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "dynamodb:List*", "dynamodb:DescribeReservedCapacity*", "dynamodb:DescribeLimits", "dynamodb:DescribeTimeToLive" ], "Resource": "*" }, { "Effect": "Allow", "Action": [ "dynamodb:BatchGetItem", "dynamodb:BatchWriteItem", "dynamodb:CreateTable", "dynamodb:DeleteItem", "dynamodb:DescribeTable", "dynamodb:GetItem", "dynamodb:PutItem", "dynamodb:Query", "dynamodb:UpdateItem", "dynamodb:Scan", "dynamodb:TagResource", "dynamodb:UntagResource" ], "Resource": { "Fn::Join": [ "", [ "arn:aws:dynamodb:*:*:table/", { "Ref": "prefix" }, "-dynamodb-table" ] ] } } ] }, "ManagedPolicyName": { "Fn::Join": [ "-", [ { "Ref": "prefix" }, "dynamodb-policy" ] ] } } }, "LogPolicy": { "Type": "AWS::IAM::ManagedPolicy", "Properties": { "PolicyDocument": { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "s3:ListBucket" ], "Resource": { "Fn::Join": [ "", [ "arn:aws:s3:::", { "Fn::Select" : [ "0", { "Fn::Split" : [ "/", { "Ref": "LogsLocationBase" } ] } ] } ] ] } }, { "Effect": "Allow", "Action": [ "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts", "s3:PutObject" ], "Resource": { "Fn::Join": [ "", [ "arn:aws:s3:::", { "Ref": "LogsLocationBase" }, "/*" ] ] } } ] }, "ManagedPolicyName": { "Fn::Join": [ "-", [ { "Ref": "prefix" }, "log-policy" ] ] } } }, "RangerAuditS3Policy": { "Type": "AWS::IAM::ManagedPolicy", "Properties": { "PolicyDocument": { "Version": "2012-10-17", "Statement": [ { "Sid": "FullObjectAccessUnderAuditDir", "Effect": "Allow", "Action": [ "s3:GetObject", "s3:PutObject" ], "Resource": { "Fn::Join": [ "", [ "arn:aws:s3:::", { "Ref": "StorageLocationBase" }, "/ranger/audit/*" ] ] } }, { "Sid": "LimitedAccessToDataLakeBucket", "Effect": "Allow", "Action": [ "s3:AbortMultipartUpload", "s3:ListBucket", "s3:ListBucketMultipartUploads" ], "Resource": { "Fn::Join": [ "", [ "arn:aws:s3:::", { "Fn::Select" : [ "0", { "Fn::Split" : [ "/", { "Ref": "StorageLocationBase" } ] } ] } ] ] } } ] }, "ManagedPolicyName": { "Fn::Join": [ "-", [ { "Ref": "prefix" }, "ranger-audit-s3-policy" ] ] } } }, "DatalakeAdminS3Policy": { "Type": "AWS::IAM::ManagedPolicy", "Properties": { "PolicyDocument": { "Version": "2012-10-17", "Statement": [ { "Sid": "VisualEditor3", "Effect": "Allow", "Action": [ "s3:AbortMultipartUpload", "s3:DeleteObject", "s3:DeleteObjectVersion", "s3:GetAccelerateConfiguration", "s3:GetAnalyticsConfiguration", "s3:GetBucketAcl", "s3:GetBucketCORS", "s3:GetBucketLocation", "s3:GetBucketLogging", "s3:GetBucketNotification", "s3:GetBucketObjectLockConfiguration", "s3:GetBucketPolicy", "s3:GetBucketPolicyStatus", "s3:GetBucketPublicAccessBlock", "s3:GetBucketRequestPayment", "s3:GetBucketTagging", "s3:GetBucketVersioning", "s3:GetBucketWebsite", "s3:GetEncryptionConfiguration", "s3:GetInventoryConfiguration", "s3:GetLifecycleConfiguration", "s3:GetMetricsConfiguration", "s3:GetObject", "s3:GetObjectAcl", "s3:GetObjectLegalHold", "s3:GetObjectRetention", "s3:GetObjectTagging", "s3:GetObjectVersion", "s3:GetObjectVersionAcl", "s3:GetObjectVersionTagging", "s3:GetReplicationConfiguration", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:ListBucketVersions", "s3:ListMultipartUploadParts", "s3:PutObject" ], "Resource": [ { "Fn::Join": [ "", [ "arn:aws:s3:::", { "Ref": "StorageLocationBase" } ] ] }, { "Fn::Join": [ "", [ "arn:aws:s3:::", { "Ref": "StorageLocationBase" }, "/*" ] ] } ] } ] }, "ManagedPolicyName": { "Fn::Join": [ "-", [ { "Ref": "prefix" }, "datalake-admin-s3-policy" ] ] } } }, "IDBrokerRole": { "Type": "AWS::IAM::Role", "Properties": { "AssumeRolePolicyDocument": { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Service": "ec2.amazonaws.com" }, "Action": "sts:AssumeRole" } ] }, "Description": "allows cdp to assume roles", "ManagedPolicyArns": [ { "Ref": "IDbrokerAssumeRolePolicy" } ], "RoleName": { "Fn::Join": [ "-", [ { "Ref": "prefix" }, "idbroker-role" ] ] } }, "DependsOn": "IDbrokerAssumeRolePolicy" }, "LogRole": { "Type": "AWS::IAM::Role", "Properties": { "AssumeRolePolicyDocument": { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Service": "ec2.amazonaws.com" }, "Action": "sts:AssumeRole" } ] }, "Description": "used by CDP to write logs", "ManagedPolicyArns": [ { "Ref": "LogPolicy" } ], "RoleName": { "Fn::Join": [ "-", [ { "Ref": "prefix" }, "log-role" ] ] } }, "DependsOn": "LogPolicy" }, "RangerAuditRole": { "Type": "AWS::IAM::Role", "Properties": { "AssumeRolePolicyDocument": { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "AWS": { "Fn::Join": [ "", [ "arn:aws:iam::", { "Ref": "AWS::AccountId" }, ":role/", { "Ref": "IDBrokerRole" } ] ] } }, "Action": "sts:AssumeRole" } ] }, "Description": "Used by CDP to write ranger audits", "ManagedPolicyArns": [ { "Ref": "RangerAuditS3Policy" }, { "Ref": "BucketAccessPolicy" }, { "Ref": "DynamoDBPolicy" } ], "RoleName": { "Fn::Join": [ "-", [ { "Ref": "prefix" }, "ranger-audit-role" ] ] } }, "DependsOn": [ "IDBrokerRole", "RangerAuditS3Policy", "BucketAccessPolicy", "DynamoDBPolicy" ] }, "DatalakeAdminRole": { "Type": "AWS::IAM::Role", "Properties": { "AssumeRolePolicyDocument": { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "AWS": { "Fn::Join": [ "", [ "arn:aws:iam::", { "Ref": "AWS::AccountId" }, ":role/", { "Ref": "IDBrokerRole" } ] ] } }, "Action": "sts:AssumeRole" } ] }, "Description": "Map to Datalake Admin Group", "ManagedPolicyArns": [ { "Ref": "DatalakeAdminS3Policy" }, { "Ref": "BucketAccessPolicy" }, { "Ref": "DynamoDBPolicy" } ], "RoleName": { "Fn::Join": [ "-", [ { "Ref": "prefix" }, "datalake-admin-role" ] ] } }, "DependsOn": "IDBrokerRole" }, "DataAccessInstanceProfile": { "Type": "AWS::IAM::InstanceProfile", "Properties": { "InstanceProfileName": { "Fn::Join": [ "-", [ { "Ref": "prefix" }, "data-access-instance-profile" ] ] }, "Roles": [ { "Ref": "IDBrokerRole" } ] }, "DependsOn": "IDBrokerRole" }, "LogAccessInstanceProfile": { "Type": "AWS::IAM::InstanceProfile", "Properties": { "InstanceProfileName": { "Fn::Join": [ "-", [ { "Ref": "prefix" }, "log-access-instance-profile" ] ] }, "Roles": [ { "Ref": "LogRole" } ] }, "DependsOn": "LogRole" }, "CdpKey": { "Condition": "kms", "Type": "AWS::KMS::Key", "Properties": { "Description": "CDP KMS Key for use with S3", "KeyPolicy": { "Version": "2012-10-17", "Id": "key-cdp-1", "Statement": [ { "Sid": "Enable IAM User Permissions", "Effect": "Allow", "Principal": { "AWS": { "Fn::Sub": "arn:aws:iam::${AWS::AccountId}:root" } }, "Action": "kms:*", "Resource": "*" }, { "Sid": "Allow administration of the key", "Effect": "Allow", "Principal": { "AWS": [ { "Fn::Sub": "arn:aws:iam::${AWS::AccountId}:root" } ] }, "Action": [ "kms:Create*", "kms:Describe*", "kms:Enable*", "kms:List*", "kms:Put*", "kms:Update*", "kms:Revoke*", "kms:Disable*", "kms:Get*", "kms:Delete*", "kms:ScheduleKeyDeletion", "kms:CancelKeyDeletion" ], "Resource": "*" }, { "Sid": "Allow use of the key", "Effect": "Allow", "Principal": { "AWS": [ { "Fn::Sub": "arn:aws:iam::${AWS::AccountId}:root" }, { "Fn::GetAtt": [ "LogRole", "Arn" ] }, { "Fn::GetAtt": [ "RangerAuditRole", "Arn" ] }, { "Fn::GetAtt": [ "DatalakeAdminRole", "Arn" ] } ] }, "Action": [ "kms:DescribeKey", "kms:Encrypt", "kms:Decrypt", "kms:ReEncrypt*", "kms:GenerateDataKey", "kms:GenerateDataKeyWithoutPlaintext" ], "Resource": "*" } ] } } }, "AwsCdpSseKmsReadWritePolicy": { "Condition": "kms", "Type": "AWS::IAM::ManagedPolicy", "Properties": { "PolicyDocument": { "Version": "2012-10-17", "Statement": [ { "Sid": "VisualEditor0", "Effect": "Allow", "Action": [ "kms:Decrypt", "kms:GenerateDataKey" ], "Resource": [ { "Fn::GetAtt": [ "CdpKey", "Arn" ]} ] } ] }, "ManagedPolicyName": { "Fn::Join": [ "_", [ { "Ref": "prefix" }, "aws-cdp-sse-kms-read-write-policy" ] ] }, "Roles": [ { "Ref": "LogRole" }, { "Ref": "RangerAuditRole" }, { "Ref": "DatalakeAdminRole" } ] } } }, "Outputs": { "LogInstanceProfile": { "Description": "Log Instance Profile Name", "Value": { "Ref" : "LogAccessInstanceProfile" } }, "LogsLocationBase": { "Description": "S3 path for log directory", "Value": { "Ref" : "LogsLocationBase" } }, "RangerAuditRole": { "Description": "Ranger Audit Role Name", "Value": { "Ref" : "RangerAuditRole" } }, "DataInstanceProfile": { "Description": "Data Access Instance Profile Name", "Value": { "Ref" : "DataAccessInstanceProfile" } }, "StorageLocationBase": { "Description": "S3 path for data directory", "Value": { "Ref" : "StorageLocationBase" } }, "DataAccessRole": { "Description": "Data Access Role Name", "Value": { "Ref" : "DatalakeAdminRole" } }, "DyanmoDbTableName": { "Description": "Data Access Role Name", "Value": { "Fn::Join" : [ "", [ { "Ref" : "prefix" }, "-dynamodb-table" ] ] } } } }