Skip to content

Instantly share code, notes, and snippets.

@mikeapted

mikeapted/serverless_data_lake_lab_prep_template.json

Created April 13, 2019 14:37
Show Gist options
  • Save mikeapted/0e6cc3fdc417c262b956f3f30dc17609 to your computer and use it in GitHub Desktop.
Save mikeapted/0e6cc3fdc417c262b956f3f30dc17609 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
serverless_data_lake_lab_prep_template.json
{
"AWSTemplateFormatVersion": "2010-09-09",
"Description": "PA16 2018-12-13 - @akirmak - RevHist: PA16: sagemaker notebook role type fixed. PA15:-(parameters added for AcctId and S3 bucket's name initials)",
"Parameters": {
"yourInitials": {
"Description": "Your Initials to be used in the s3-bucket created. All in small letters pls. e.g. It shall be 'fs' for Frank Sinatra",
"Type": "String",
"MinLength": "2",
"MaxLength": "5"
}
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"38df71f9-6e6c-4cb3-bc01-40d0988453b1": {
"size": {
"width": 60,
"height": 60
},
"position": {
"x": 540,
"y": 180
},
"z": 1,
"embeds": []
},
"13e1ef2d-884d-4875-8b55-27a4843db116": {
"size": {
"width": 60,
"height": 60
},
"position": {
"x": 540,
"y": 60
},
"z": 1,
"embeds": []
},
"c599e0d5-d036-4fa1-9503-59cebc8349d1": {
"size": {
"width": 60,
"height": 60
},
"position": {
"x": 540,
"y": 270
},
"z": 1,
"embeds": []
},
"e058c21b-ec9d-4936-95eb-2c492465d87b": {
"size": {
"width": 60,
"height": 60
},
"position": {
"x": 360,
"y": 270
},
"z": 1,
"embeds": [],
"isassociatedwith": [
"c599e0d5-d036-4fa1-9503-59cebc8349d1"
]
},
"ed799633-3378-4ef8-8cbe-37b6c7ad5181": {
"size": {
"width": 60,
"height": 60
},
"position": {
"x": 330,
"y": 180
},
"z": 1,
"embeds": [],
"isassociatedwith": [
"38df71f9-6e6c-4cb3-bc01-40d0988453b1"
]
},
"e517d929-5247-426f-9c9e-2c8b7c9a37c6": {
"size": {
"width": 60,
"height": 60
},
"position": {
"x": 320,
"y": 80
},
"z": 0,
"embeds": [],
"isassociatedwith": [
"13e1ef2d-884d-4875-8b55-27a4843db116"
]
}
}
},
"Resources": {
"tameGlueRoleSlessDataLakeImmersion": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": [
"glue.amazonaws.com"
]
},
"Action": [
"sts:AssumeRole"
]
}
]
},
"ManagedPolicyArns": [
"arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole"
]
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "38df71f9-6e6c-4cb3-bc01-40d0988453b1"
}
}
},
"tameGluePolicySlessDataLakeImmersion": {
"Type": "AWS::IAM::Policy",
"Properties": {
"PolicyName": "AWSGlueServicePolicyServerlessDataLakeImmersion",
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Sid": "VisualEditor0",
"Effect": "Allow",
"Action": [
"s3:PutObject",
"s3:GetObject",
"s3:DeleteObject"
],
"Resource": {
"Fn::Join": [
"",
[
"arn:aws:s3:::",
{
"Ref": "yourInitials"
},
"-tame-bda-immersion/*"
]
]
}
},
{
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject"
],
"Resource": {
"Fn::Join": [
"",
[
"arn:aws:s3:::",
{
"Ref": "yourInitials"
},
"-tame-bda-immersion/compressed-parquet*"
]
]
}
}
]
},
"Roles": [
{
"Ref": "tameGlueRoleSlessDataLakeImmersion"
}
]
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "ed799633-3378-4ef8-8cbe-37b6c7ad5181"
}
}
},
"tameFHoseRoleSlessDataLakeImmersion": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "firehose.amazonaws.com"
},
"Action": "sts:AssumeRole",
"Condition": {
"StringEquals": {
"sts:ExternalId": {
"Ref": "AWS::AccountId"
}
}
}
}
]
},
"ManagedPolicyArns": [
"arn:aws:iam::aws:policy/CloudWatchLogsFullAccess"
]
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "13e1ef2d-884d-4875-8b55-27a4843db116"
}
}
},
"tameFHosePolicySlessDataLakeImmersion": {
"Type": "AWS::IAM::Policy",
"Properties": {
"PolicyName": "FirehosePolicyServerlessDataLakeImmersion",
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Action": [
"glue:GetTableVersions"
],
"Resource": "*"
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"s3:AbortMultipartUpload",
"s3:GetBucketLocation",
"s3:GetObject",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:PutObject"
],
"Resource": [
{
"Fn::Join": [
"",
[
"arn:aws:s3:::",
{
"Ref": "yourInitials"
},
"-tame-bda-immersion"
]
]
},
{
"Fn::Join": [
"",
[
"arn:aws:s3:::",
{
"Ref": "yourInitials"
},
"-tame-bda-immersion/*"
]
]
},
"arn:aws:s3:::%FIREHOSE_BUCKET_NAME%",
"arn:aws:s3:::%FIREHOSE_BUCKET_NAME%/*"
]
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"lambda:InvokeFunction",
"lambda:GetFunctionConfiguration"
],
"Resource": {
"Fn::Join": [
"",
[
"arn:aws:lambda:",
{
"Ref": "AWS::Region"
},
":",
{
"Ref": "AWS::AccountId"
},
":function:%FIREHOSE_DEFAULT_FUNCTION%:%FIREHOSE_DEFAULT_VERSION%"
]
]
}
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"logs:PutLogEvents"
],
"Resource": [
{
"Fn::Join": [
"",
[
"arn:aws:logs:",
{
"Ref": "AWS::Region"
},
":",
{
"Ref": "AWS::AccountId"
},
":log-group:/aws/kinesisfirehose/tamebda-rta-kinesisfh-prodcat:log-stream:*"
]
]
}
]
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"kinesis:DescribeStream",
"kinesis:GetShardIterator",
"kinesis:GetRecords"
],
"Resource": {
"Fn::Join": [
"",
[
"arn:aws:kinesis:",
{
"Ref": "AWS::Region"
},
":",
{
"Ref": "AWS::AccountId"
},
":stream/%FIREHOSE_STREAM_NAME%"
]
]
}
},
{
"Effect": "Allow",
"Action": [
"kms:Decrypt"
],
"Resource": [
"arn:aws:kms:region:accountid:key/%SSE_KEY_ARN%"
],
"Condition": {
"StringEquals": {
"kms:ViaService": "kinesis.%REGION_NAME%.amazonaws.com"
},
"StringLike": {
"kms:EncryptionContext:aws:kinesis:arn": {
"Fn::Join": [
"",
[
"arn:aws:kinesis:%REGION_NAME%:",
{
"Ref": "AWS::AccountId"
},
":stream/%FIREHOSE_STREAM_NAME%"
]
]
}
}
}
}
]
},
"Roles": [
{
"Ref": "tameFHoseRoleSlessDataLakeImmersion"
}
]
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "e517d929-5247-426f-9c9e-2c8b7c9a37c6"
}
}
},
"tameSageMakerNBookRoleSlessDataLake": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": "sagemaker.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
},
"ManagedPolicyArns": [
"arn:aws:iam::aws:policy/AmazonS3FullAccess",
"arn:aws:iam::aws:policy/AmazonAthenaFullAccess"
]
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "c599e0d5-d036-4fa1-9503-59cebc8349d1"
}
}
},
"tameSageMakerNBookPolicySlessDataLake": {
"Type": "AWS::IAM::Policy",
"Properties": {
"PolicyName": "SageMakerNotebookPolicyServerlessDataLake",
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Sid": "VisualEditor0",
"Effect": "Allow",
"Action": [
"logs:CreateLogStream",
"logs:DescribeLogStreams",
"s3:ListBucket",
"logs:PutLogEvents"
],
"Resource": [
{
"Fn::Join": [
"",
[
"arn:aws:s3:::aws-glue-jes-prod-",
{
"Ref": "AWS::Region"
},
"-assets"
]
]
},
{
"Fn::Join": [
"",
[
"arn:aws:logs:",
{
"Ref": "AWS::Region"
},
":",
{
"Ref": "AWS::AccountId"
},
":log-group:/aws/sagemaker/*"
]
]
},
{
"Fn::Join": [
"",
[
"arn:aws:logs:",
{
"Ref": "AWS::Region"
},
":",
{
"Ref": "AWS::AccountId"
},
":log-group:/aws/sagemaker/*:log-stream:aws-glue-*"
]
]
}
]
},
{
"Sid": "VisualEditor1",
"Effect": "Allow",
"Action": "s3:GetObject",
"Resource": {
"Fn::Join": [
"",
[
"arn:aws:s3:::aws-glue-jes-prod-",
{
"Ref": "AWS::Region"
},
"-assets*"
]
]
}
},
{
"Sid": "VisualEditor2",
"Effect": "Allow",
"Action": [
"s3:PutAnalyticsConfiguration",
"s3:GetObjectVersionTagging",
"s3:CreateBucket",
"s3:ReplicateObject",
"s3:GetObjectAcl",
"s3:DeleteBucketWebsite",
"s3:PutLifecycleConfiguration",
"s3:GetObjectVersionAcl",
"s3:PutObjectTagging",
"s3:DeleteObject",
"s3:DeleteObjectTagging",
"s3:GetBucketPolicyStatus",
"s3:GetBucketWebsite",
"s3:PutReplicationConfiguration",
"s3:DeleteObjectVersionTagging",
"s3:GetBucketNotification",
"s3:PutBucketCORS",
"s3:GetReplicationConfiguration",
"s3:ListMultipartUploadParts",
"s3:GetObject",
"s3:PutBucketNotification",
"s3:PutObject",
"s3:PutBucketLogging",
"s3:GetAnalyticsConfiguration",
"s3:GetObjectVersionForReplication",
"s3:GetLifecycleConfiguration",
"s3:ListBucketByTags",
"s3:GetBucketTagging",
"s3:GetInventoryConfiguration",
"s3:PutAccelerateConfiguration",
"s3:DeleteObjectVersion",
"s3:GetBucketLogging",
"s3:ListBucketVersions",
"s3:ReplicateTags",
"s3:RestoreObject",
"s3:GetAccelerateConfiguration",
"s3:ListBucket",
"s3:GetBucketPolicy",
"s3:PutEncryptionConfiguration",
"s3:GetEncryptionConfiguration",
"s3:GetObjectVersionTorrent",
"s3:AbortMultipartUpload",
"s3:GetBucketRequestPayment",
"s3:PutBucketTagging",
"s3:GetObjectTagging",
"s3:GetMetricsConfiguration",
"s3:DeleteBucket",
"s3:PutBucketVersioning",
"s3:GetBucketPublicAccessBlock",
"s3:ListBucketMultipartUploads",
"s3:PutMetricsConfiguration",
"s3:PutObjectVersionTagging",
"s3:GetBucketVersioning",
"s3:GetBucketAcl",
"s3:PutInventoryConfiguration",
"s3:GetObjectTorrent",
"s3:PutBucketRequestPayment",
"s3:PutBucketWebsite",
"s3:GetBucketCORS",
"s3:GetBucketLocation",
"s3:GetObjectVersion",
"s3:ReplicateDelete"
],
"Resource": {
"Fn::Join": [
"",
[
"arn:aws:s3:::aws-athena-query-results-",
{
"Ref": "AWS::AccountId"
},
{
"Ref": "AWS::Region"
},
"*"
]
]
}
},
{
"Sid": "VisualEditor3",
"Effect": "Allow",
"Action": [
"s3:GetAccountPublicAccessBlock",
"s3:ListAllMyBuckets",
"s3:HeadBucket"
],
"Resource": "*"
},
{
"Sid": "VisualEditor4",
"Effect": "Allow",
"Action": "logs:CreateLogGroup",
"Resource": [
{
"Fn::Join": [
"",
[
"arn:aws:logs:",
{
"Ref": "AWS::Region"
},
":",
{
"Ref": "AWS::AccountId"
},
":log-group:/aws/sagemaker/*"
]
]
},
{
"Fn::Join": [
"",
[
"arn:aws:logs:",
{
"Ref": "AWS::Region"
},
":",
{
"Ref": "AWS::AccountId"
},
":log-group:/aws/sagemaker/*:log-stream:aws-glue-*"
]
]
}
]
},
{
"Sid": "VisualEditor5",
"Effect": "Allow",
"Action": [
"glue:GetDevEndpoints",
"glue:UpdateDevEndpoint",
"glue:GetDevEndpoint"
],
"Resource": {
"Fn::Join": [
"",
[
"arn:aws:glue:",
{
"Ref": "AWS::Region"
},
":",
{
"Ref": "AWS::AccountId"
},
":devEndpoint/gj-tame-bda-kdg-raw2parquet-devEndpoint*"
]
]
}
}
]
},
"Roles": [
{
"Ref": "tameSageMakerNBookRoleSlessDataLake"
}
]
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "e058c21b-ec9d-4936-95eb-2c492465d87b"
}
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment