Created
May 12, 2021 16:39
-
-
Save vindimy/aa27a148277299c61c18d76c6ccd8d17 to your computer and use it in GitHub Desktop.
Revisions
-
vindimy created this gist
May 12, 2021 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,476 @@ { "AWSTemplateFormatVersion": "2010-09-09", "Description": "Elastic SIEM - Filebeat ingestion of Cloudtrail logs from S3", "Parameters": { "KeyPair": { "Type": "AWS::EC2::KeyPair::KeyName", "Default": "", "Description": "Name of an existing EC2 KeyPair to enable SSH access" }, "InstanceClass" : { "Type" : "String", "Description" : "EC2 instance class - should be large enough to accommodate task's CPU/memory reservation, if specified", "Default" : "xlarge", "AllowedValues": ["large","xlarge","2xlarge","4xlarge"] }, "TaskName": { "Type": "String", "Description": "ECS cluster and task name", "Default": "siem-filebeat-cloudtrail", "MinLength": 3, "MaxLength": 25 }, "TaskCount": { "Type": "Number", "Description": "Number of ECS tasks to run", "Default": 2, "MinValue": 1, "MaxValue": 10 }, "TaskCPU": { "Type": "Number", "Description": "CPU units to allocate to ECS task. 1024 = one CPU. Leave at 0 to not restrict CPU.", "Default": 0, "AllowedValues": [0,512,1024,2048,4096,8192] }, "TaskMemory": { "Type": "Number", "Description": "Memory units to allocate to ECS task, in MB. Leave at 0 to not restrict memory.", "Default": 0, "AllowedValues": [0,512,1024,2048,3072,4096,5120,6144,7168,8192,16384] }, "ECSOnDemandPercentage" : { "Type" : "Number", "Description" : "Specify percentage of EC2 hosts that should run on-demand (0: run all EC2 as spot instances, 100: run all EC2 as on-demand instances).", "Default" : 0, "AllowedValues": [0,33,50,66,100] }, "FilebeatTaskImage": { "Type": "String", "Description": "Filebeat ECS task Docker image path", "Default": "docker.elastic.co/beats/filebeat:7.12.1" }, "ElasticsearchHostPort": { "Type": "String", "Description": "protocol://host:port value for Elasticsearch output", "Default": "http://es-host:9200" }, "S3BucketName": { "Type": "String", "Description": "Name for S3 bucket to be used for CloudTrail storage. Leave blank to create one using this template (recommended).", "Default": "my-cloudtrail-bucket" }, "LogRetentionDays": { "Type": "Number", "Description": "ECS log group retention days", "Default": 3, "AllowedValues": [1,3,5,7,14,30,60,90] } }, "Mappings" : { "AWSRegionArch2AMI": { "us-east-1": {"64": "ami-007cd1678c6286a05", "RepoVersion": "2"}, "us-west-2": {"64": "ami-01e1d12a048e67ed2", "RepoVersion": "2"} } }, "Conditions" : { "TaskCPUDefined" : {"Fn::Not": [{"Fn::Equals": [{"Ref": "TaskCPU"}, 0]}]}, "TaskMemoryDefined" : {"Fn::Not": [{"Fn::Equals": [{"Ref": "TaskMemory"}, 0]}]}, "CreateS3Bucket" : {"Fn::Equals": [{"Ref": "S3BucketName"}, ""]} }, "Resources": { "ECSSecurityGroup": { "Type": "AWS::EC2::SecurityGroup", "Properties": { "VpcId": {"Fn::ImportValue": "Main-vpcid"}, "SecurityGroupIngress": [ {"CidrIp": "10.0.0.0/8", "IpProtocol": "-1", "FromPort": "-1", "ToPort": "-1"} ], "GroupDescription": {"Fn::Sub": "${TaskName} ECS security group"} } }, "ECSLogGroup": { "Type" : "AWS::Logs::LogGroup", "Properties" : { "LogGroupName" : {"Fn::Sub": "/ecs/${ECSCluster}/${TaskName}"}, "RetentionInDays" : {"Ref": "LogRetentionDays"} } }, "SIEMS3Bucket": { "Type": "AWS::S3::Bucket", "Properties" : { "NotificationConfiguration": { "QueueConfigurations" : [ { "Event" : "s3:ObjectCreated:*", "Queue" : {"Fn::Sub": "${SIEMS3Queue.Arn}"} } ] } }, "Condition": "CreateS3Bucket" }, "SIEMS3BucketPolicy": { "Type": "AWS::S3::BucketPolicy", "Properties": { "Bucket": {"Ref": "SIEMS3Bucket"}, "PolicyDocument": { "Version": "2012-10-17", "Statement": [ { "Sid": "AWSCloudTrailAclCheck", "Effect": "Allow", "Principal": { "Service": "cloudtrail.amazonaws.com" }, "Action": "s3:GetBucketAcl", "Resource": {"Fn::Sub": "${SIEMS3Bucket.Arn}"} }, { "Sid": "AWSCloudTrailWrite", "Effect": "Allow", "Principal": { "Service": "cloudtrail.amazonaws.com" }, "Action": "s3:PutObject", "Resource": {"Fn::Sub": "${SIEMS3Bucket.Arn}/AWSLogs/*"}, "Condition": { "StringEquals": {"s3:x-amz-acl": "bucket-owner-full-control"} } }, { "Sid": "AllowReadFromECSRole", "Effect": "Allow", "Principal": { "AWS": {"Fn::Sub": "${ECSExecutionRole.Arn}"} }, "Action": [ "s3:Get*", "s3:List*" ], "Resource": [ {"Fn::Sub": "${SIEMS3Bucket.Arn}"}, {"Fn::Sub": "${SIEMS3Bucket.Arn}/*"} ] } ] } }, "Condition": "CreateS3Bucket" }, "SIEMS3Queue": { "Type": "AWS::SQS::Queue" }, "SIEMS3QueuePolicy": { "Type" : "AWS::SQS::QueuePolicy", "Properties" : { "PolicyDocument" : { "Statement": [ { "Sid": "S3SNSPolicy", "Effect": "Allow", "Principal": { "Service": "s3.amazonaws.com" }, "Resource": "*", "Action": "sqs:SendMessage" } ] }, "Queues" : [{"Ref": "SIEMS3Queue"}] } }, "ECSRole": { "Type": "AWS::IAM::Role", "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "ec2.amazonaws.com" } } ], "Version": "2012-10-17" }, "ManagedPolicyArns": [ "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role", "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" ], "Policies": [{ "PolicyDocument": { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "ec2:Describe*", "ec2:List*", "elasticfilesystem:Describe*" ], "Resource": "*" }, { "Effect": "Allow", "Action": [ "s3:List*", "s3:Get*" ], "Resource": [ "arn:aws:s3:::edmunds-shared-services-bootstrap", "arn:aws:s3:::edmunds-shared-services-bootstrap/*", ] }, { "Effect": "Allow", "Action": [ "s3:ListAllMyBuckets", "s3:HeadBucket" ], "Resource": "*" } ] }, "PolicyName": {"Fn::Join": ["-", [{"Ref": "AWS::StackName"}, "EC2-Policy"]]} }] } }, "ECSInstanceProfile": { "Type": "AWS::IAM::InstanceProfile", "Properties": { "Roles": [{"Ref": "ECSRole"}] } }, "EcsInstanceLT" : { "Type" : "AWS::EC2::LaunchTemplate", "Properties" : { "LaunchTemplateData": { "ImageId": {"Fn::FindInMap": ["AWSRegionArch2AMI", {"Ref" : "AWS::Region"}, "64"]}, "BlockDeviceMappings" : [ ], "IamInstanceProfile": {"Name": { "Ref": "ECSInstanceProfile" }}, "KeyName" : { "Ref" : "KeyPair" }, "SecurityGroupIds" : [ {"Fn::ImportValue": "Main-basesecgroupid"}, { "Ref" : "ECSSecurityGroup" } ], "UserData" : { "Fn::Base64": { "Fn::Join" : ["", [ "#cloud-config\n", "repo_releasever: ", {"Fn::FindInMap": ["AWSRegionArch2AMI", {"Ref" : "AWS::Region"}, "RepoVersion"]}, "\n", "runcmd:\n", " - set -x", "\n", " - export AWS_DEFAULT_REGION=", {"Ref": "AWS::Region"}, "\n", " - export INSTANCEID=$(curl http://169.254.169.254/latest/meta-data/instance-id)\n", " - export ECS_CLUSTER_NAME=", {"Ref": "ECSCluster"}, "\n", " - yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm", "\n", " - echo -e \"ECS_CLUSTER=", {"Ref": "ECSCluster"}, "\" >/etc/ecs/ecs.config", "\n", " - export ECS_ENABLE_SPOT_INSTANCE_DRAINING=true", "\n" ] ] } } } } }, "ECSAutoScaleGroup" : { "Type" : "AWS::AutoScaling::AutoScalingGroup", "Properties" : { "AvailabilityZones" : {"Fn::Split": [",", {"Fn::ImportValue": "Main-privatesubnetAZs"}]}, "VPCZoneIdentifier" : {"Fn::Split": [",", {"Fn::ImportValue": "Main-privatesubnets"}]}, "MixedInstancesPolicy" : { "InstancesDistribution" : { "OnDemandAllocationStrategy" : "prioritized", "OnDemandBaseCapacity" : 0, "OnDemandPercentageAboveBaseCapacity" : {"Ref": "ECSOnDemandPercentage"}, "SpotAllocationStrategy" : "capacity-optimized" }, "LaunchTemplate" : { "LaunchTemplateSpecification" : { "LaunchTemplateId": {"Ref": "EcsInstanceLT"}, "Version": {"Fn::GetAtt": ["EcsInstanceLT", "LatestVersionNumber"]} }, "Overrides" : [ {"InstanceType" : {"Fn::Sub": "c4.${InstanceClass}"}}, {"InstanceType" : {"Fn::Sub": "c5.${InstanceClass}"}}, {"InstanceType" : {"Fn::Sub": "m4.${InstanceClass}"}}, {"InstanceType" : {"Fn::Sub": "m5.${InstanceClass}"}} ] } }, "MinSize" : {"Ref": "TaskCount"}, "MaxSize" : {"Ref": "TaskCount"}, "DesiredCapacity" : {"Ref": "TaskCount"}, "Tags" : [ { "Key" : "Name", "Value" : {"Fn::Join" : ["", [{ "Ref" : "TaskName" }, " - ECS Cluster"]]}, "PropagateAtLaunch" : "true" }, { "Key" : "Cluster", "Value" : { "Ref" : "TaskName" }, "PropagateAtLaunch" : "true" }, { "Key" : "Environment", "Value" : {"Fn::ImportValue": "Main-account-name"}, "PropagateAtLaunch" : "true" }, { "Key" : "Application ID", "Value" : {"Ref": "TaskName"}, "PropagateAtLaunch" : "true"}, { "Key" : "Application Role", "Value" : "ECS Task", "PropagateAtLaunch" : "true"} ] } }, "ECSExecutionRole": { "Type": "AWS::IAM::Role", "Properties": { "AssumeRolePolicyDocument": { "Version" : "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Service": [ "ecs-tasks.amazonaws.com" ], "AWS": [ {"Fn::Sub": "${ECSRole.Arn}"} ] }, "Action": [ "sts:AssumeRole" ] } ] }, "ManagedPolicyArns": [ "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" ], "Policies": [{ "PolicyDocument": { "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Action": ["sqs:ReceiveMessage", "sqs:DeleteMessage*", "sqs:Get*", "sqs:List*"], "Resource": {"Fn::Sub": "${SIEMS3Queue.Arn}"} }, { "Effect": "Allow", "Action": ["s3:Get*", "s3:List*"], "Resource": {"Fn::If": [ "CreateS3Bucket", [{"Fn::Sub": "${SIEMS3Bucket.Arn}"}, {"Fn::Sub": "${SIEMS3Bucket.Arn}/*"}], [{"Fn::Sub": "arn:aws:s3:::${S3BucketName}"}, {"Fn::Sub": "arn:aws:s3:::${S3BucketName}/*"}] ]} } ] }, "PolicyName": {"Fn::Join": ["-", [{"Ref": "AWS::StackName"}, "ECS-Policy"]]} }], "Path": "/" } }, "ECSCluster": { "Type" : "AWS::ECS::Cluster" }, "ECSTaskDefinition": { "Type" : "AWS::ECS::TaskDefinition", "Properties" : { "Cpu" : {"Fn::If": ["TaskCPUDefined", {"Ref": "TaskCPU"}, {"Ref": "AWS::NoValue"}]}, "Family" : {"Ref": "TaskName"}, "Memory" : {"Fn::If": ["TaskMemoryDefined", {"Ref": "TaskMemory"}, {"Ref": "AWS::NoValue"}]}, "NetworkMode": "host", "Volumes": [ ], "RequiresCompatibilities": ["EC2"], "ContainerDefinitions" : [{ "Name": {"Ref": "TaskName"}, "Cpu": {"Fn::If": ["TaskCPUDefined", {"Ref": "TaskCPU"}, {"Ref": "AWS::NoValue"}]}, "Memory": {"Fn::If": ["TaskMemoryDefined", {"Ref": "TaskMemory"}, {"Ref": "AWS::NoValue"}]}, "MemoryReservation": {"Fn::If": ["TaskMemoryDefined", {"Ref": "TaskMemory"}, 512]}, "Image": {"Ref": "FilebeatTaskImage"}, "EntryPoint": ["/usr/share/filebeat/filebeat"], "Command": [ "run", "-environment", "container", "-v", "--E", "output.elasticsearch.enabled=true", "--E", {"Fn::Sub": "output.elasticsearch.hosts=['${ElasticsearchHostPort}']"}, "--E", "output.elasticsearch.username=filebeat", "--E", "output.elasticsearch.password=filebeat", "--E", "output.elasticsearch.bulk_max_size=500", "--E", "output.elasticsearch.worker=8", "--E", "output.elasticsearch.index=filebeat-%{[agent.version]}-cloudtrail-%{+yyyy.MM.dd}", "--E", "setup.template.name=filebeat", "--E", "setup.template.pattern=filebeat-*", "--E", "setup.ilm.enabled=false", "--E", "setup.template.settings.index.max_docvalue_fields_search=250", "--modules=aws", "--M", "aws.cloudtrail.enabled=true", "--M", {"Fn::Sub": "aws.cloudtrail.var.role_arn=${ECSExecutionRole.Arn}"}, "--M", {"Fn::Sub": "aws.cloudtrail.var.queue_url=${SIEMS3Queue}"}, "--M", "aws.cloudwatch.enabled=false", "--M", "aws.ec2.enabled=false", "--M", "aws.elb.enabled=false", "--M", "aws.s3access.enabled=false", "--M", "aws.vpcflow.enabled=false" ], "Environment": [ ], "PortMappings": [ ], "MountPoints": [ ], "Ulimits": [{ "SoftLimit": 65536, "HardLimit": 65536, "Name": "nofile" }], "LogConfiguration": { "LogDriver": "awslogs", "Options": { "awslogs-region": {"Ref": "AWS::Region"}, "awslogs-group": {"Ref": "ECSLogGroup"}, "awslogs-stream-prefix": {"Ref": "TaskName"} } } }], "ExecutionRoleArn": {"Fn::GetAtt": ["ECSExecutionRole", "Arn"]}, "Tags" : [ { "Key" : "Name", "Value" : { "Ref" : "TaskName" } }, { "Key" : "Environment", "Value" : {"Fn::ImportValue": "Main-account-name"} }, { "Key" : "Application ID", "Value" : {"Ref": "TaskName"} }, { "Key" : "Application Role", "Value" : "ECS Task" } ] } }, "ECSService": { "Type" : "AWS::ECS::Service", "Properties" : { "Cluster" : {"Fn::GetAtt": ["ECSCluster", "Arn"]}, "DeploymentConfiguration" : { "MaximumPercent" : 200, "MinimumHealthyPercent" : 0 }, "DesiredCount" : {"Ref": "TaskCount"}, "LaunchType" : "EC2", "TaskDefinition" : {"Ref": "ECSTaskDefinition"}, "PlacementConstraints": [{ "Type" : "distinctInstance" }], "Tags" : [ { "Key" : "Name", "Value" : { "Ref" : "TaskName" } }, { "Key" : "Environment", "Value" : {"Fn::ImportValue": "Main-account-name"} }, { "Key" : "Application ID", "Value" : {"Ref": "TaskName"} }, { "Key" : "Application Role", "Value" : "ECS Task" } ] } } }, "Outputs" : { "SIEMS3QueueURL" : { "Description" : "S3 bucket SQS notification queue URL", "Value" : { "Fn::Sub" : "${SIEMS3Queue}" } }, "ECSClusterURL" : { "Description" : "Filebeat ECS cluster URL", "Value" : { "Fn::Sub" : "https://${AWS::Region}.console.aws.amazon.com/ecs/home?region=${AWS::Region}#/clusters/${ECSCluster}/services" } }, } }