#!/usr/bin/env bash # REF: https://cloud.google.com/docs/enterprise/best-practices-for-enterprise-organizations export PROJECT_ID=$(gcloud config get-value project) export PROJECT_USER=$(gcloud config get-value core/account) # set current user export PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="value(projectNumber)") export IDNS=${PROJECT_ID}.svc.id.goog # workflow identity domain export GCP_REGION="us-west4" # CHANGEME (OPT) export GCP_ZONE="us-west4-a" # CHANGEME (OPT) export DOMAIN="msparr.com" # CHANGEME (OPT) export TEST_NS="argo" # CHANGEME (OPT) - also using for subdomain for TLS export NETWORK_NAME="default" # configure gcloud sdk gcloud config set compute/region $GCP_REGION gcloud config set compute/zone $GCP_ZONE ##################################################################### # FOLDERS ##################################################################### export SANDBOX_FOLDER=$FOLDER export DEMO_PARENT_FOLDER="argo-demo" export SHARED_FOLDER="shared-services" export DEPT_FOLDER="engineering" export PRODUCT_FOLDER="saas-app1" export NONPROD_FOLDER="non-production" export PROD_FOLDER="production" gcloud resource-manager folders create \ --display-name=$DEMO_PARENT_FOLDER \ --folder=$SANDBOX_FOLDER export DEMO_FOLDER_ID=324182485460 # make note of folder ID after create (folders/) gcloud resource-manager folders create \ --display-name=$SHARED_FOLDER \ --folder=$DEMO_FOLDER_ID export SHARED_FOLDER_ID=12773735309 # make note of folder ID after create (folders/) gcloud resource-manager folders create \ --display-name=$DEPT_FOLDER \ --folder=$DEMO_FOLDER_ID export DEPT_FOLDER_ID=1030948867935 # make note of folder ID after create (folders/) gcloud resource-manager folders create \ --display-name=$PRODUCT_FOLDER \ --folder=$DEPT_FOLDER_ID export PRODUCT_FOLDER_ID=858045355436 # make note of folder ID after create (folders/) gcloud resource-manager folders create \ --display-name=$NONPROD_FOLDER \ --folder=$PRODUCT_FOLDER_ID export NONPROD_FOLDER_ID=958190730640 # make note of folder ID after create (folders/) gcloud resource-manager folders create \ --display-name=$PROD_FOLDER \ --folder=$PRODUCT_FOLDER_ID export PROD_FOLDER_ID=40282131100 # make note of folder ID after create (folders/) ##################################################################### # ORG POLICIES (SET ON DEMO PARENT FOLDER FOR DEMO [SHOULD SET ON ORG]) # REF: https://cloud.google.com/resource-manager/docs/organization-policy/org-policy-constraints#how-to_guides # REF: https://cloud.google.com/storage/docs/org-policy-constraints ##################################################################### # disable external IPs for VMs export IP_POLICY_FILE=policy-extip.json cat > $IP_POLICY_FILE << EOF { "constraint": "constraints/compute.vmExternalIpAccess", "listPolicy": { "allValues": "DENY" } } EOF gcloud resource-manager org-policies set-policy $IP_POLICY_FILE --folder=$DEMO_FOLDER_ID # restrict prod networks to production folder (optional) export SHARED_VPC_POLICY_FILE=policy-allowedsubnetsprod.yaml cat > $SHARED_VPC_POLICY_FILE << EOF constraint: constraints/compute.restrictSharedVpcSubnetworks listPolicy: allowed_values: - projects/$HOST_PROJECT_PROD_ID/regions/$GCP_REGION/subnetworks/k8s-nodes-prod - projects/$HOST_PROJECT_PROD_ID/regions/$GCP_REGION/subnetworks/bastion-prod - projects/$HOST_PROJECT_PROD_ID/regions/$GCP_REGION/subnetworks/vms-prod - projects/$HOST_PROJECT_PROD_ID/regions/$GCP_REGION/subnetworks/dbs-prod - projects/$HOST_PROJECT_PROD_ID/regions/$GCP_REGION/subnetworks/vpcconn-prod - projects/$HOST_PROJECT_PROD_ID/regions/$GCP_REGION/subnetworks/memorystore-prod EOF gcloud beta resource-manager org-policies set-policy $SHARED_VPC_POLICY_FILE --folder=$PROD_FOLDER_ID # disable default networks ( constraints/compute.skipDefaultNetworkCreation ) gcloud resource-manager org-policies enable-enforce \ --folder $DEMO_FOLDER_ID \ compute.skipDefaultNetworkCreation # require OS Login for all VMs ( compute.requireOsLogin ) gcloud resource-manager org-policies enable-enforce \ --folder $DEMO_FOLDER_ID \ compute.requireOsLogin # disable edit role on default service accounts ( iam.automaticIamGrantsForDefaultServiceAccounts ) gcloud resource-manager org-policies enable-enforce \ --folder $DEMO_FOLDER_ID \ iam.automaticIamGrantsForDefaultServiceAccounts # disable SA key creation ( iam.disableServiceAccountKeyCreation ) gcloud resource-manager org-policies enable-enforce \ --folder $DEMO_FOLDER_ID \ iam.disableServiceAccountKeyCreation # disable SA key upload ( iam.disableServiceAccountKeyUpload ) gcloud resource-manager org-policies enable-enforce \ --folder $DEMO_FOLDER_ID \ iam.disableServiceAccountKeyUpload # disable lien removal since we allow cross-project SA ( iam.restrictCrossProjectServiceAccountLienRemoval ) gcloud resource-manager org-policies enable-enforce \ --folder $DEMO_FOLDER_ID \ iam.restrictCrossProjectServiceAccountLienRemoval # require uniform bucket level access ( storage.uniformBucketLevelAccess ) gcloud resource-manager org-policies enable-enforce \ --folder $DEMO_FOLDER_ID \ storage.uniformBucketLevelAccess ##################################################################### # PROJECTS ##################################################################### export BILLING_ACCOUNT_ID=$BILLING # created previously export DEVOPS_PROJECT_ID="argo-demo-devops-1" export DEVOPS_PROJECT_NUM=$(gcloud projects describe $DEVOPS_PROJECT_ID --format="value(projectNumber)") export HOST_PROJECT_NONPROD_ID="argo-demo-nw-nonprod-1" export HOST_PROJECT_PROD_ID="argo-demo-nw-prod-1" export DEV_PROJECT_ID="argo-demo-development-1" export DEV_PROJECT_NUM=$(gcloud projects describe $DEV_PROJECT_ID --format="value(projectNumber)") export PROD_PROJECT_ID="argo-demo-production-1" export PROD_PROJECT_NUM=$(gcloud projects describe $PROD_PROJECT_ID --format="value(projectNumber)") # SVC: devops gcloud projects create $DEVOPS_PROJECT_ID \ --folder $SHARED_FOLDER_ID gcloud beta billing projects link $DEVOPS_PROJECT_ID \ --billing-account=$BILLING_ACCOUNT_ID gcloud services enable compute.googleapis.com \ container.googleapis.com \ storage.googleapis.com \ cloudbuild.googleapis.com \ artifactregistry.googleapis.com \ --project $DEVOPS_PROJECT_ID # disable deletion (key project) gcloud alpha resource-manager liens create \ --restrictions=resourcemanager.projects.delete \ --reason="Contains critical service accounts" \ --project $DEVOPS_PROJECT_ID # allow SA key creation ONLY on devops project gcloud resource-manager org-policies disable-enforce \ --project $DEVOPS_PROJECT_ID \ iam.disableServiceAccountKeyCreation # disable public buckets ( storage.publicAccessPrevention ) gcloud resource-manager org-policies enable-enforce \ --project $DEVOPS_PROJECT_ID \ storage.publicAccessPrevention # SVC: development gcloud projects create $DEV_PROJECT_ID \ --folder $NONPROD_FOLDER_ID gcloud beta billing projects link $DEV_PROJECT_ID \ --billing-account=$BILLING_ACCOUNT_ID gcloud services enable compute.googleapis.com \ container.googleapis.com \ --project $DEV_PROJECT_ID # SVC: production gcloud projects create $PROD_PROJECT_ID \ --folder $PROD_FOLDER_ID gcloud beta billing projects link $PROD_PROJECT_ID \ --billing-account=$BILLING_ACCOUNT_ID gcloud services enable compute.googleapis.com \ container.googleapis.com \ --project $PROD_PROJECT_ID # HOST: non-prod gcloud projects create $HOST_PROJECT_NONPROD_ID \ --folder $SHARED_FOLDER_ID gcloud beta billing projects link $HOST_PROJECT_NONPROD_ID \ --billing-account=$BILLING_ACCOUNT_ID gcloud services enable compute.googleapis.com \ container.googleapis.com \ --project $HOST_PROJECT_NONPROD_ID # associated service projects gcloud compute shared-vpc enable $HOST_PROJECT_NONPROD_ID gcloud compute shared-vpc associated-projects add $DEVOPS_PROJECT_ID \ --host-project=$HOST_PROJECT_NONPROD_ID gcloud compute shared-vpc associated-projects add $DEV_PROJECT_ID \ --host-project=$HOST_PROJECT_NONPROD_ID # verify gcloud compute shared-vpc associated-projects list $HOST_PROJECT_NONPROD_ID # HOST: prod gcloud projects create $HOST_PROJECT_PROD_ID \ --folder $SHARED_FOLDER_ID gcloud beta billing projects link $HOST_PROJECT_PROD_ID \ --billing-account=$BILLING_ACCOUNT_ID gcloud services enable compute.googleapis.com \ container.googleapis.com \ --project $HOST_PROJECT_PROD_ID # associate service projects gcloud compute shared-vpc enable $HOST_PROJECT_PROD_ID gcloud compute shared-vpc associated-projects add $PROD_PROJECT_ID \ --host-project=$HOST_PROJECT_PROD_ID # disable deleting production gcloud alpha resource-manager liens create \ --restrictions=resourcemanager.projects.delete \ --reason="Production environment" \ --project $PROD_PROJECT_ID # verify gcloud compute shared-vpc associated-projects list $HOST_PROJECT_PROD_ID ##################################################################### # SERVICE ACCOUNTS # REF: https://cloud.google.com/iam/docs/best-practices-for-securing-service-accounts#data-access-logs # REF: https://cloud.google.com/kubernetes-engine/docs/how-to/hardening-your-cluster#use_least_privilege_sa # REF: https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-shared-vpc ##################################################################### export SA_TF_ADMIN="terraform-admin" export SA_CI_SERVER="ci-server" export SA_BASTION_NONPROD="bastion-nonprod" export SA_BASTION_PROD="bastion-prod" export SA_K8S_DEVOPS="k8s-devops" export SA_K8S_DEV="k8s-dev" export SA_K8S_PROD="k8s-prod" export SA_NAT_DEV="nat-dev" export SA_NAT_PROD="nat-prod" gcloud iam service-accounts create $SA_TF_ADMIN --project $DEVOPS_PROJECT_ID gcloud iam service-accounts create $SA_CI_SERVER --project $DEVOPS_PROJECT_ID #gcloud iam service-accounts create $SA_BASTION_NONPROD --project $DEVOPS_PROJECT_ID #gcloud iam service-accounts create $SA_BASTION_PROD --project $DEVOPS_PROJECT_ID gcloud iam service-accounts create $SA_K8S_DEVOPS --project $DEVOPS_PROJECT_ID gcloud iam service-accounts create $SA_K8S_DEV --project $DEV_PROJECT_ID gcloud iam service-accounts create $SA_K8S_PROD --project $PROD_PROJECT_ID #gcloud iam service-accounts create $SA_NAT_DEVOPS --project $HOST_PROJECT_NONPROD_ID #gcloud iam service-accounts create $SA_NAT_DEV --project $HOST_PROJECT_NONPROD_ID #gcloud iam service-accounts create $SA_NAT_PROD --project $HOST_PROJECT_PROD_ID # gke cluster (devops) - optional object viewer on artifact registry bucket gcloud projects add-iam-policy-binding $DEVOPS_PROJECT_ID \ --member "serviceAccount:$SA_K8S_DEVOPS@$DEVOPS_PROJECT_ID.iam.gserviceaccount.com" \ --role roles/logging.logWriter gcloud projects add-iam-policy-binding $DEVOPS_PROJECT_ID \ --member "serviceAccount:$SA_K8S_DEVOPS@$DEVOPS_PROJECT_ID.iam.gserviceaccount.com" \ --role roles/monitoring.metricWriter gcloud projects add-iam-policy-binding $DEVOPS_PROJECT_ID \ --member "serviceAccount:$SA_K8S_DEVOPS@$DEVOPS_PROJECT_ID.iam.gserviceaccount.com" \ --role roles/monitoring.viewer gcloud projects add-iam-policy-binding $DEVOPS_PROJECT_ID \ --member "serviceAccount:$SA_K8S_DEVOPS@$DEVOPS_PROJECT_ID.iam.gserviceaccount.com" \ --role roles/stackdriver.resourceMetadata.writer gcloud projects add-iam-policy-binding $HOST_PROJECT_NONPROD_ID \ --member "serviceAccount:service-$DEVOPS_PROJECT_NUM@container-engine-robot.iam.gserviceaccount.com" \ --role=roles/compute.securityAdmin gcloud projects add-iam-policy-binding $HOST_PROJECT_NONPROD_ID \ --member "serviceAccount:service-$DEVOPS_PROJECT_NUM@container-engine-robot.iam.gserviceaccount.com" \ --role roles/container.hostServiceAgentUser gcloud projects add-iam-policy-binding $HOST_PROJECT_NONPROD_ID \ --member "serviceAccount:service-$DEVOPS_PROJECT_NUM@container-engine-robot.iam.gserviceaccount.com" \ --role roles/compute.networkUser gcloud projects add-iam-policy-binding $HOST_PROJECT_NONPROD_ID \ --member "serviceAccount:$DEVOPS_PROJECT_NUM@cloudservices.gserviceaccount.com" \ --role roles/compute.networkUser gcloud iam service-accounts add-iam-policy-binding \ $SA_K8S_DEVOPS@$DEVOPS_PROJECT_ID.iam.gserviceaccount.com \ --member="user:$PROJECT_USER" \ --role="roles/iam.serviceAccountUser" # gke cluster (dev) - optional object viewer on artifact registry bucket gcloud projects add-iam-policy-binding $DEV_PROJECT_ID \ --member "serviceAccount:$SA_K8S_DEV@$DEV_PROJECT_ID.iam.gserviceaccount.com" \ --role roles/logging.logWriter gcloud projects add-iam-policy-binding $DEV_PROJECT_ID \ --member "serviceAccount:$SA_K8S_DEV@$DEV_PROJECT_ID.iam.gserviceaccount.com" \ --role roles/monitoring.metricWriter gcloud projects add-iam-policy-binding $DEV_PROJECT_ID \ --member "serviceAccount:$SA_K8S_DEV@$DEV_PROJECT_ID.iam.gserviceaccount.com" \ --role roles/monitoring.viewer gcloud projects add-iam-policy-binding $DEV_PROJECT_ID \ --member "serviceAccount:$SA_K8S_DEV@$DEV_PROJECT_ID.iam.gserviceaccount.com" \ --role roles/stackdriver.resourceMetadata.writer gcloud projects add-iam-policy-binding $HOST_PROJECT_NONPROD_ID \ --member "serviceAccount:service-$DEV_PROJECT_NUM@container-engine-robot.iam.gserviceaccount.com" \ --role=roles/compute.securityAdmin gcloud projects add-iam-policy-binding $HOST_PROJECT_NONPROD_ID \ --member "serviceAccount:service-$DEV_PROJECT_NUM@container-engine-robot.iam.gserviceaccount.com" \ --role roles/container.hostServiceAgentUser gcloud projects add-iam-policy-binding $HOST_PROJECT_NONPROD_ID \ --member "serviceAccount:service-$DEV_PROJECT_NUM@container-engine-robot.iam.gserviceaccount.com" \ --role roles/compute.networkUser gcloud projects add-iam-policy-binding $HOST_PROJECT_NONPROD_ID \ --member "serviceAccount:$DEV_PROJECT_NUM@cloudservices.gserviceaccount.com" \ --role roles/compute.networkUser gcloud iam service-accounts add-iam-policy-binding \ $SA_K8S_DEV@$DEV_PROJECT_ID.iam.gserviceaccount.com \ --member="user:$PROJECT_USER" \ --role="roles/iam.serviceAccountUser" \ --project $DEV_PROJECT_ID # gke cluster (prod) - optional object viewer on artifact registry bucket gcloud projects add-iam-policy-binding $PROD_PROJECT_ID \ --member "serviceAccount:$SA_K8S_PROD@$PROD_PROJECT_ID.iam.gserviceaccount.com" \ --role roles/logging.logWriter gcloud projects add-iam-policy-binding $PROD_PROJECT_ID \ --member "serviceAccount:$SA_K8S_PROD@$PROD_PROJECT_ID.iam.gserviceaccount.com" \ --role roles/monitoring.metricWriter gcloud projects add-iam-policy-binding $PROD_PROJECT_ID \ --member "serviceAccount:$SA_K8S_PROD@$PROD_PROJECT_ID.iam.gserviceaccount.com" \ --role roles/monitoring.viewer gcloud projects add-iam-policy-binding $PROD_PROJECT_ID \ --member "serviceAccount:$SA_K8S_PROD@$PROD_PROJECT_ID.iam.gserviceaccount.com" \ --role roles/stackdriver.resourceMetadata.writer gcloud projects add-iam-policy-binding $HOST_PROJECT_PROD_ID \ --member "serviceAccount:service-$PROD_PROJECT_NUM@container-engine-robot.iam.gserviceaccount.com" \ --role=roles/compute.securityAdmin gcloud projects add-iam-policy-binding $HOST_PROJECT_PROD_ID \ --member "serviceAccount:service-$PROD_PROJECT_NUM@container-engine-robot.iam.gserviceaccount.com" \ --role roles/container.hostServiceAgentUser gcloud projects add-iam-policy-binding $HOST_PROJECT_PROD_ID \ --member "serviceAccount:service-$PROD_PROJECT_NUM@container-engine-robot.iam.gserviceaccount.com" \ --role roles/compute.networkUser gcloud projects add-iam-policy-binding $HOST_PROJECT_PROD_ID \ --member "serviceAccount:$PROD_PROJECT_NUM@cloudservices.gserviceaccount.com" \ --role roles/compute.networkUser gcloud iam service-accounts add-iam-policy-binding \ $SA_K8S_PROD@$PROD_PROJECT_ID.iam.gserviceaccount.com \ --member="user:$PROJECT_USER" \ --role="roles/iam.serviceAccountUser" ##################################################################### # NETWORKS # Instances on this network will not be reachable until firewall rules # are created. As an example, you can allow all internal traffic between # instances as well as SSH, RDP, and ICMP by running: # $ gcloud compute firewall-rules create \ # --network devops-10-19-0-0 --allow tcp,udp,icmp --source-ranges # $ gcloud compute firewall-rules create \ # --network devops-10-19-0-0 --allow tcp:22,tcp:3389,icmp ##################################################################### # devops export VPC_DEVOPS="devops-10-19-0-0" export SUBNET_K8S_NODES_DEVOPS="10.19.0.0/22" export SUBNET_K8S_PODS_DEVOPS="10.89.0.0/18" export SUBNET_K8S_SVCS_DEVOPS="10.89.64.0/22" export SUBNET_BASTION_DEVOPS="10.19.64.0/29" gcloud compute networks create $VPC_DEVOPS \ --bgp-routing-mode=global \ --subnet-mode=custom \ --project $HOST_PROJECT_NONPROD_ID gcloud compute networks subnets create k8s-nodes-devops \ --network=$VPC_DEVOPS \ --range=$SUBNET_K8S_NODES_DEVOPS \ --region=$GCP_REGION \ --secondary-range k8s-pods-devops=$SUBNET_K8S_PODS_DEVOPS \ --secondary-range k8s-svcs-devops=$SUBNET_K8S_SVCS_DEVOPS \ --enable-private-ip-google-access \ --project $HOST_PROJECT_NONPROD_ID gcloud compute networks subnets create bastion-devops \ --network=$VPC_DEVOPS \ --range=$SUBNET_BASTION_DEVOPS \ --region=$GCP_REGION \ --project $HOST_PROJECT_NONPROD_ID # dev export VPC_DEV="dev-10-11-0-0" export SUBNET_K8S_NODES_DEV="10.11.0.0/22" export SUBNET_K8S_PODS_DEV="10.81.0.0/18" export SUBNET_K8S_SVCS_DEV="10.81.64.0/22" export SUBNET_BASTION_DEV="10.11.64.0/29" export SUBNET_VMS_DEV="10.11.65.0/24" export SUBNET_DBS_DEV="10.11.70.0/24" export SUBNET_VPCCONN_DEV="10.11.90.0/28" export SUBNET_MEMORYSTORE_DEV="10.11.91.0/29" gcloud compute networks create $VPC_DEV \ --bgp-routing-mode=global \ --subnet-mode=custom \ --project $HOST_PROJECT_NONPROD_ID gcloud compute networks subnets create k8s-nodes-dev \ --network=$VPC_DEV \ --range=$SUBNET_K8S_NODES_DEV \ --region=$GCP_REGION \ --secondary-range k8s-pods-dev=$SUBNET_K8S_PODS_DEV \ --secondary-range k8s-svcs-dev=$SUBNET_K8S_SVCS_DEV \ --enable-private-ip-google-access \ --project $HOST_PROJECT_NONPROD_ID gcloud compute networks subnets create bastion-dev \ --network=$VPC_DEV \ --range=$SUBNET_BASTION_DEV \ --region=$GCP_REGION \ --project $HOST_PROJECT_NONPROD_ID gcloud compute networks subnets create vms-dev \ --network=$VPC_DEV \ --range=$SUBNET_VMS_DEV \ --region=$GCP_REGION \ --project $HOST_PROJECT_NONPROD_ID gcloud compute networks subnets create dbs-dev \ --network=$VPC_DEV \ --range=$SUBNET_DBS_DEV \ --region=$GCP_REGION \ --project $HOST_PROJECT_NONPROD_ID gcloud compute networks subnets create vpcconn-dev \ --network=$VPC_DEV \ --range=$SUBNET_VPCCONN_DEV \ --region=$GCP_REGION \ --project $HOST_PROJECT_NONPROD_ID gcloud compute networks subnets create memorystore-dev \ --network=$VPC_DEV \ --range=$SUBNET_MEMORYSTORE_DEV \ --region=$GCP_REGION \ --project $HOST_PROJECT_NONPROD_ID # prod export VPC_PROD="prod-10-10-0-0" export SUBNET_K8S_NODES_PROD="10.10.0.0/22" export SUBNET_K8S_PODS_PROD="10.80.0.0/18" export SUBNET_K8S_SVCS_PROD="10.80.64.0/22" export SUBNET_BASTION_PROD="10.10.64.0/29" export SUBNET_VMS_PROD="10.10.65.0/24" export SUBNET_DBS_PROD="10.10.70.0/24" export SUBNET_VPCCONN_PROD="10.10.90.0/28" export SUBNET_MEMORYSTORE_PROD="10.10.91.0/29" gcloud compute networks create $VPC_PROD \ --bgp-routing-mode=global \ --subnet-mode=custom \ --project $HOST_PROJECT_PROD_ID gcloud compute networks subnets create k8s-nodes-prod \ --network=$VPC_PROD \ --range=$SUBNET_K8S_NODES_PROD \ --region=$GCP_REGION \ --secondary-range k8s-pods-prod=$SUBNET_K8S_PODS_PROD \ --secondary-range k8s-svcs-prod=$SUBNET_K8S_SVCS_PROD \ --enable-private-ip-google-access \ --project $HOST_PROJECT_PROD_ID gcloud compute networks subnets create bastion-prod \ --network=$VPC_PROD \ --range=$SUBNET_BASTION_PROD \ --region=$GCP_REGION \ --project $HOST_PROJECT_PROD_ID gcloud compute networks subnets create vms-prod \ --network=$VPC_PROD \ --range=$SUBNET_VMS_PROD \ --region=$GCP_REGION \ --project $HOST_PROJECT_PROD_ID gcloud compute networks subnets create dbs-prod \ --network=$VPC_PROD \ --range=$SUBNET_DBS_PROD \ --region=$GCP_REGION \ --project $HOST_PROJECT_PROD_ID gcloud compute networks subnets create vpcconn-prod \ --network=$VPC_PROD \ --range=$SUBNET_VPCCONN_PROD \ --region=$GCP_REGION \ --project $HOST_PROJECT_PROD_ID gcloud compute networks subnets create memorystore-prod \ --network=$VPC_PROD \ --range=$SUBNET_MEMORYSTORE_PROD \ --region=$GCP_REGION \ --project $HOST_PROJECT_PROD_ID ##################################################################### # NAT (allow private nodes to reach Internet) ##################################################################### # devops export ROUTER_DEVOPS=router-devops gcloud compute routers create $ROUTER_DEVOPS \ --region $GCP_REGION \ --network $VPC_DEVOPS \ --project=$HOST_PROJECT_NONPROD_ID gcloud beta compute routers nats create nat-devops \ --router=$ROUTER_DEVOPS \ --region=$GCP_REGION \ --auto-allocate-nat-external-ips \ --nat-all-subnet-ip-ranges \ --project=$HOST_PROJECT_NONPROD_ID # dev export ROUTER_DEV=router-dev gcloud compute routers create $ROUTER_DEV \ --region $GCP_REGION \ --network $VPC_DEV \ --project=$HOST_PROJECT_NONPROD_ID gcloud beta compute routers nats create nat-dev \ --router=$ROUTER_DEV \ --region=$GCP_REGION \ --auto-allocate-nat-external-ips \ --nat-all-subnet-ip-ranges \ --project=$HOST_PROJECT_NONPROD_ID # prod export ROUTER_PROD=router-prod gcloud compute routers create $ROUTER_PROD \ --region $GCP_REGION \ --network $VPC_PROD \ --project=$HOST_PROJECT_PROD_ID gcloud beta compute routers nats create nat-prod \ --router=$ROUTER_PROD \ --region=$GCP_REGION \ --auto-allocate-nat-external-ips \ --nat-all-subnet-ip-ranges \ --project=$HOST_PROJECT_PROD_ID ##################################################################### # IAM POLICIES # Use groups (preferred) for IAM role grants, and monitoring changes # REF: https://cloud.google.com/iam/docs/groups-in-cloud-console # REF: https://cloud.google.com/iam/docs/groups-in-cloud-console#view-logs # REF: https://cloud.google.com/iam/docs/job-functions/networking#separate_network_security_teams # REF: https://cloud.google.com/resource-manager/docs/access-control-org#restricting_visibility ##################################################################### export GROUP_ADMIN="orgadmins@$DOMAIN" export GROUP_BILLING="billingadmins@$DOMAIN" export GROUP_NETWORK="networkadmins@$DOMAIN" export GROUP_SECURITY="securityadmins@$DOMAIN" export GROUP_DEVOPS="devops@$DOMAIN" export GROUP_DEV_SAASAPP1="dev-saasapp1@$DOMAIN" # ORGANIZATION LEVEL (this demo will use folder given shared sandbox) # org admin gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ --member=group:$GROUP_ADMIN --role=roles/resourcemanager.organizationAdmin # billing admin gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ --member=group:$GROUP_BILLING --role=roles/billing.admin # network admin gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ --member=group:$GROUP_NETWORK --role=roles/compute.xpnAdmin gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ --member=group:$GROUP_NETWORK --role=roles/compute.networkAdmin gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ --member=group:$GROUP_NETWORK --role=roles/servicenetworking.networksAdmin # security admin gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ --member=group:$GROUP_SECURITY --role=roles/resourcemanager.organizationAdmin gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ --member=group:$GROUP_SECURITY --role=roles/browser gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ --member=group:$GROUP_SECURITY --role=roles/logging.viewer gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ --member=group:$GROUP_SECURITY --role=roles/logging.privateLogViewer gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ --member=group:$GROUP_SECURITY --role=roles/iam.securityAdmin gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ --member=group:$GROUP_SECURITY --role=roles/iam.serviceAccountAdmin gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ --member=group:$GROUP_SECURITY --role=roles/compute.securityAdmin # gcloud organizations add-iam-policy-binding $ORG_ID \ # --member=group:$GROUP_SECURITY --role=roles/resourcemanager.organizationViewer # gcloud organizations add-iam-policy-binding $ORG_ID \ # --member=group:$GROUP_SECURITY --role=roles/orgpolicy.policyAdmin # devops (SRE) gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ --member=group:$GROUP_DEVOPS --role=roles/browser gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ --member=group:$GROUP_DEVOPS --role=roles/logging.viewer ##################################################################### # DNS (internal) # REF: https://cloud.google.com/kubernetes-engine/docs/how-to/cloud-dns ##################################################################### # TODO (optional) ##################################################################### # K8S CLUSTERS (GOOGLE KUBERNETES ENGINE [GKE]) # REF: https://cloud.google.com/architecture/prep-kubernetes-engine-for-prod # REF: https://cloud.google.com/kubernetes-engine/docs/how-to/hardening-your-cluster # REF: https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-shared-vpc ##################################################################### export DEVOPS_CLUSTER=devops export DEV_CLUSTER=west-dev export PROD_CLUSTER=west-prod export AUTH_NETWORK="174.45.73.139/32" # TODO: assign internal DNS to clusters (optional) # devops (AutoPilot) gcloud container --project $DEVOPS_PROJECT_ID clusters create-auto $DEVOPS_CLUSTER \ --region $GCP_REGION \ --release-channel "regular" \ --enable-private-nodes \ --enable-master-authorized-networks --master-authorized-networks $AUTH_NETWORK \ --network "projects/$HOST_PROJECT_NONPROD_ID/global/networks/$VPC_DEVOPS" \ --subnetwork "projects/$HOST_PROJECT_NONPROD_ID/regions/$GCP_REGION/subnetworks/k8s-nodes-devops" \ --cluster-secondary-range-name "k8s-pods-devops" \ --services-secondary-range-name "k8s-svcs-devops" \ --service-account $SA_K8S_DEVOPS@$DEVOPS_PROJECT_ID.iam.gserviceaccount.com # dev (AutoPilot) gcloud container --project $DEV_PROJECT_ID clusters create-auto $DEV_CLUSTER \ --region $GCP_REGION \ --release-channel "regular" \ --enable-private-nodes \ --enable-master-authorized-networks --master-authorized-networks $AUTH_NETWORK \ --network "projects/$HOST_PROJECT_NONPROD_ID/global/networks/$VPC_DEV" \ --subnetwork "projects/$HOST_PROJECT_NONPROD_ID/regions/$GCP_REGION/subnetworks/k8s-nodes-dev" \ --cluster-secondary-range-name "k8s-pods-dev" \ --services-secondary-range-name "k8s-svcs-dev" \ --service-account $SA_K8S_DEV@$DEV_PROJECT_ID.iam.gserviceaccount.com # prod (AutoPilot) gcloud container --project $PROD_PROJECT_ID clusters create-auto $PROD_CLUSTER \ --region $GCP_REGION \ --release-channel "regular" \ --enable-private-nodes \ --enable-master-authorized-networks --master-authorized-networks $AUTH_NETWORK \ --network "projects/$HOST_PROJECT_PROD_ID/global/networks/$VPC_PROD" \ --subnetwork "projects/$HOST_PROJECT_PROD_ID/regions/$GCP_REGION/subnetworks/k8s-nodes-prod" \ --cluster-secondary-range-name "k8s-pods-prod" \ --services-secondary-range-name "k8s-svcs-prod" \ --service-account $SA_K8S_PROD@$PROD_PROJECT_ID.iam.gserviceaccount.com ##################################################################### # ARGO CD + ARGO ROLLOUTS # REF: https://argo-cd.readthedocs.io/en/stable/getting_started/ # REF: https://argo-cd.readthedocs.io/en/stable/operator-manual/ingress/ # REF: https://argoproj.github.io/argo-rollouts/ # REF: https://www.youtube.com/watch?v=hIL0E2gLkf8 # REF: https://argoproj.github.io/argo-rollouts/features/kubectl-plugin/ ##################################################################### # auth to devops project cluster gcloud config set project $DEVOPS_PROJECT_ID gcloud container clusters get-credentials $DEVOPS_CLUSTER --region $GCP_REGION # grant cluster admin to current user kubectl create clusterrolebinding argo-cluster-admin-binding \ --clusterrole=cluster-admin \ --user=$PROJECT_USER # install argo cd kubectl create namespace argocd kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml # patch to install without TLS to avoid loop (expose with ingress instead) kubectl -n argocd edit deployments.apps argocd-server export ARGOCD_PASS=$(kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d; echo) export ARGOCD_OPTS='--port-forward-namespace argocd' # expose with port-forward (optional) kubectl port-forward svc/argocd-server -n argocd 8080:443 & # enable argo cd CLI (Mac using Homebrew) brew install argocd # TODO: authenticate CLI, add cluster(s) and install agent # TODO: add devops pod CIDR as authorized networks for other clusters # install argo rollouts kubectl create namespace argo-rollouts kubectl apply -n argo-rollouts -f https://github.com/argoproj/argo-rollouts/releases/latest/download/install.yaml # enable argo-rollouts kubectl extension (Mac using Homebrew) brew install argoproj/tap/kubectl-argo-rollouts