Skip to content

Instantly share code, notes, and snippets.

@karthikeayan

karthikeayan/firehose.tf

Created May 12, 2020 13:50
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save karthikeayan/a67e93b4937a7958716dfecaa6ff7767 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save karthikeayan/a67e93b4937a7958716dfecaa6ff7767 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. karthikeayan renamed this gist May 12, 2020. 1 changed file with 0 additions and 0 deletions.
    0 gistfile1.txt → firehose.tf
    View file
    File renamed without changes.
  2. karthikeayan created this gist May 12, 2020.
    154 changes: 154 additions & 0 deletions gistfile1.txt
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,154 @@
    resource "aws_iam_role" "firehose_role" {
    name = "firehose_es_delivery_role"

    assume_role_policy = <<EOF
    {
    "Version": "2012-10-17",
    "Statement": [
    {
    "Action": "sts:AssumeRole",
    "Principal": {
    "Service": "firehose.amazonaws.com"
    },
    "Effect": "Allow",
    "Sid": ""
    }
    ]
    }
    EOF
    }

    resource "aws_iam_role_policy" "firehose_es_policy" {
    name = "firehose_es_policy"
    role = aws_iam_role.firehose_role.id
    policy = <<EOF
    {
    "Version": "2012-10-17",
    "Statement": [
    {
    "Sid": "",
    "Effect": "Allow",
    "Action": [
    "ec2:DescribeVpcs",
    "ec2:DescribeVpcAttribute",
    "ec2:DescribeSubnets",
    "ec2:DescribeSecurityGroups",
    "ec2:DescribeNetworkInterfaces",
    "ec2:CreateNetworkInterface",
    "ec2:CreateNetworkInterfacePermission",
    "ec2:DeleteNetworkInterface"
    ],
    "Resource": "*"
    },
    {
    "Sid": "",
    "Effect": "Allow",
    "Action": [
    "s3:AbortMultipartUpload",
    "s3:GetBucketLocation",
    "s3:GetObject",
    "s3:ListBucket",
    "s3:ListBucketMultipartUploads",
    "s3:PutObject"
    ],
    "Resource": [
    "arn:aws:s3:::${aws_s3_bucket.firehose_bucket.arn}",
    "arn:aws:s3:::${aws_s3_bucket.firehose_bucket.arn}/*"
    ]
    },
    {
    "Effect": "Allow",
    "Action": [
    "es:ESHttpPost",
    "es:ESHttpPut",
    "es:DescribeElasticsearchDomain",
    "es:DescribeElasticsearchDomains",
    "es:DescribeElasticsearchDomainConfig"
    ],
    "Resource": [
    "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}",
    "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}*"
    ]
    },
    {
    "Effect": "Allow",
    "Action": "es:ESHttpGet",
    "Resource": [
    "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/_all/_settings",
    "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/_cluster/stats",
    "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/index-name*/_mapping/type-name",
    "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/roletest*/_mapping/roletest",
    "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/_nodes",
    "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/_nodes/stats",
    "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/_nodes/*/stats",
    "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/_stats",
    "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/index-name*/_stats",
    "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/roletest*/_stats"
    ]
    },
    {
    "Sid": "",
    "Effect": "Allow",
    "Action": [
    "logs:PutLogEvents"
    ],
    "Resource": [
    "arn:aws:logs:us-east-1:someaccount:log-group:/aws/kinesisfirehose/accel-${local.workspace}-ms:log-stream:*"
    ]
    },
    {
    "Sid": "",
    "Effect": "Allow",
    "Action": [
    "kinesis:DescribeStream",
    "kinesis:GetShardIterator",
    "kinesis:GetRecords",
    "kinesis:ListShards"
    ],
    "Resource": "arn:aws:kinesis:us-east-1:someaccount:stream/%FIREHOSE_STREAM_NAME%"
    },
    {
    "Effect": "Allow",
    "Action": [
    "kms:Decrypt"
    ],
    "Resource": [
    "arn:aws:kms:us-east-1:someaccount:key/%SSE_KEY_ID%"
    ],
    "Condition": {
    "StringEquals": {
    "kms:ViaService": "kinesis.%REGION_NAME%.amazonaws.com"
    },
    "StringLike": {
    "kms:EncryptionContext:aws:kinesis:arn": "arn:aws:kinesis:%REGION_NAME%:someaccount:stream/%FIREHOSE_STREAM_NAME%"
    }
    }
    }
    ]
    }
    EOF
    }

    resource "aws_s3_bucket" "firehose_bucket" {
    bucket = "accel-${local.workspace}-firehose-logs"
    acl = "private"
    }

    resource "aws_kinesis_firehose_delivery_stream" "accel_es_stream" {
    name = "accel-${local.workspace}-ms"
    destination = "elasticsearch"

    s3_configuration {
    role_arn = aws_iam_role.firehose_role.arn
    bucket_arn = aws_s3_bucket.firehose_bucket.arn
    buffer_size = 1
    buffer_interval = 60
    compression_format = "GZIP"
    }

    elasticsearch_configuration {
    domain_arn = local.workspace == "mgmt" ? "dummy" : data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn
    role_arn = aws_iam_role.firehose_role.arn
    index_name = "accel-firehose-${local.workspace}"
    }
    }