Skip to content

Instantly share code, notes, and snippets.

@dogenpunk

dogenpunk/README.md

Forked from olivergeorge/README.md
Created May 8, 2019 03:41
Show Gist options
  • Save dogenpunk/0f291f45baee7ee34c6dbe9d21dedc5a to your computer and use it in GitHub Desktop.
Save dogenpunk/0f291f45baee7ee34c6dbe9d21dedc5a to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. @olivergeorge olivergeorge revised this gist Jul 24, 2018. 1 changed file with 2 additions and 0 deletions.
    2 changes: 2 additions & 0 deletions README.md
    View file
    Original file line number Diff line number Diff line change
    @@ -1 +1,3 @@
    Experiment to setup a single step deploy process for Datomic Ions including setting up the AWS API Gateway.

    Next experiment will be generating a CloudFormation template for the app.
  2. @olivergeorge olivergeorge revised this gist Jul 23, 2018. 1 changed file with 3 additions and 3 deletions.
    6 changes: 3 additions & 3 deletions api_release.clj
    View file
    Original file line number Diff line number Diff line change
    @@ -11,9 +11,9 @@
    {:keys [Policy]} (aws "lambda" "get-policy" "--function-name" function-name)
    policy-data (when Policy (json/read-str Policy :key-fn keyword))]

    (if-not (find-by (submap? {:Condition {:ArnLike {:AWS:SourceArn source-arn}}
    :Resource FunctionArn})
    (:Statement policy-data))
    (when-not (find-by (submap? {:Condition {:ArnLike {:AWS:SourceArn source-arn}}
    :Resource FunctionArn})
    (:Statement policy-data))

    (aws "lambda" "add-permission"
    "--function-name" function-name
  3. @olivergeorge olivergeorge revised this gist Jul 23, 2018. 1 changed file with 3 additions and 1 deletion.
    4 changes: 3 additions & 1 deletion user.clj
    View file
    Original file line number Diff line number Diff line change
    @@ -10,4 +10,6 @@
    (ion-release/release ion-config {})
    (api-release/sync-rest-api api-config)
    (api-release/deploy-api (:app-name api-config) "dev")
    (api-release/deploy-uri (:app-name api-config) "dev"))
    (println "Deployed to" (api-release/deploy-uri (:app-name api-config) "dev")))

    (comment (deploy))
  4. @olivergeorge olivergeorge revised this gist Jul 23, 2018. 1 changed file with 1 addition and 10 deletions.
    11 changes: 1 addition & 10 deletions README.md
    View file
    Original file line number Diff line number Diff line change
    @@ -1,10 +1 @@
    Experiment to populate an AWS API Gateway for Datomic web service ions.

    Changes

    * Decompleted from ion-config.edn
    * Moved code into libs

    To do

    * Pass data as maps to allow extensions
    Experiment to setup a single step deploy process for Datomic Ions including setting up the AWS API Gateway.
  5. @olivergeorge olivergeorge revised this gist Jul 23, 2018. 2 changed files with 20 additions and 29 deletions.
    18 changes: 6 additions & 12 deletions api_release.clj
    View file
    Original file line number Diff line number Diff line change
    @@ -1,31 +1,25 @@
    (ns deploy.api-release
    (:require [clojure.data.json :as json]
    [clojure.string :as string]
    [deploy.aws-utils :refer [aws find-by submap? get-stack-context]]))
    [deploy.aws-utils :refer [aws find-by submap? get-stack-context]])
    (:import (java.util UUID)))

    (defn sync-permission
    "Setup permission based via accreate-only changes"
    [region client-id rest-api-id path-part function-name FunctionArn]
    (let [source-arn (str "arn:aws:execute-api:" region ":" client-id ":" rest-api-id "/*/*/" path-part)
    statement-id (str "ion" (hash (str function-name source-arn)))
    {:keys [Policy]} (aws "lambda" "get-policy" "--function-name" function-name)
    policy-data (when Policy (json/read-str Policy :key-fn keyword))]

    (if-let [statement (find-by (submap? {:Sid statement-id}) (:Statement policy-data))]

    (let [kvs {:Condition {:ArnLike {:AWS:SourceArn source-arn}}
    :Resource FunctionArn}]
    (when-not (submap? kvs statement)
    (clojure.pprint/pprint
    {:msg "permission statement incompatible with config"
    :data {:kvs kvs :statement statement}})
    (throw (ex-info "permission statement incompatible with config" {:kvs kvs :statement statement}))))
    (if-not (find-by (submap? {:Condition {:ArnLike {:AWS:SourceArn source-arn}}
    :Resource FunctionArn})
    (:Statement policy-data))

    (aws "lambda" "add-permission"
    "--function-name" function-name
    "--action" "lambda:InvokeFunction"
    "--principal" "apigateway.amazonaws.com"
    "--statement-id" statement-id
    "--statement-id" (.toString (UUID/randomUUID))
    "--source-arn" source-arn))))

    (defn sync-integration
    31 changes: 14 additions & 17 deletions ion_release.clj
    View file
    Original file line number Diff line number Diff line change
    @@ -5,20 +5,17 @@
    (defn release
    "Do push and deploy of app. Supports stable and unstable releases. Returns when deploy finishes running."
    [ion-config args]
    (try
    (let [group (:group (aws-utils/get-stack-context (:app-name ion-config)))
    push-data (ion-dev/push args)
    deploy-args (merge (select-keys args [:creds-profile :region :uname :group])
    (select-keys push-data [:rev])
    {:group group})]
    (let [deploy-data (ion-dev/deploy deploy-args)
    deploy-status-args (merge (select-keys args [:creds-profile :region])
    (select-keys deploy-data [:execution-arn]))]
    (loop []
    (let [status-data (ion-dev/deploy-status deploy-status-args)]
    (if (= "RUNNING" (:code-deploy-status status-data))
    (do (Thread/sleep 5000) (recur))
    status-data)))))
    (catch Exception e
    {:deploy-status "ERROR"
    :message (.getMessage e)})))
    (let [group (:group (aws-utils/get-stack-context (:app-name ion-config)))
    push-data (ion-dev/push args)
    deploy-args (merge (select-keys args [:creds-profile :region :uname :group])
    (select-keys push-data [:rev])
    {:group group})]
    (let [deploy-data (ion-dev/deploy deploy-args)
    deploy-status-args (merge (select-keys args [:creds-profile :region])
    (select-keys deploy-data [:execution-arn]))]
    (loop []
    (let [status-data (ion-dev/deploy-status deploy-status-args)]
    (case (:code-deploy-status status-data)
    "RUNNING" (do (Thread/sleep 5000) (recur))
    "SUCCEEDED" status-data
    (throw (ex-info "ion-release did not succeed" status-data))))))))
  6. @olivergeorge olivergeorge revised this gist Jul 23, 2018. 1 changed file with 7 additions and 2 deletions.
    9 changes: 7 additions & 2 deletions README.md
    View file
    Original file line number Diff line number Diff line change
    @@ -1,5 +1,10 @@
    Experiment to populate an AWS API Gateway for Datomic web service ions.

    Thoughts
    Changes

    * Should be possible to separate from ion-config.edn (say rest-api.edn)
    * Decompleted from ion-config.edn
    * Moved code into libs

    To do

    * Pass data as maps to allow extensions
  7. @olivergeorge olivergeorge revised this gist Jul 23, 2018. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions user.clj
    View file
    Original file line number Diff line number Diff line change
    @@ -9,5 +9,5 @@
    (defn deploy []
    (ion-release/release ion-config {})
    (api-release/sync-rest-api api-config)
    (api-release/deploy-api api-config "dev")
    (api-release/deploy-uri api-config "dev"))
    (api-release/deploy-api (:app-name api-config) "dev")
    (api-release/deploy-uri (:app-name api-config) "dev"))
  8. @olivergeorge olivergeorge revised this gist Jul 23, 2018. 6 changed files with 194 additions and 194 deletions.
    11 changes: 11 additions & 0 deletions api_config.edn
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,11 @@
    {:app-name "arthur"
    :resources [{:path-part "hello-world"
    :methods [{:http-method "ANY"
    :authorization-type "NONE"
    :integration {:lambda-name :hello-world-web
    :timeout-in-millis 29000}}]}
    {:path-part "echo"
    :methods [{:http-method "ANY"
    :authorization-type "NONE"
    :integration {:lambda-name :echo-web
    :timeout-in-millis 29000}}]}]}
    124 changes: 124 additions & 0 deletions api_release.clj
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,124 @@
    (ns deploy.api-release
    (:require [clojure.data.json :as json]
    [clojure.string :as string]
    [deploy.aws-utils :refer [aws find-by submap? get-stack-context]]))

    (defn sync-permission
    "Setup permission based via accreate-only changes"
    [region client-id rest-api-id path-part function-name FunctionArn]
    (let [source-arn (str "arn:aws:execute-api:" region ":" client-id ":" rest-api-id "/*/*/" path-part)
    statement-id (str "ion" (hash (str function-name source-arn)))
    {:keys [Policy]} (aws "lambda" "get-policy" "--function-name" function-name)
    policy-data (when Policy (json/read-str Policy :key-fn keyword))]

    (if-let [statement (find-by (submap? {:Sid statement-id}) (:Statement policy-data))]

    (let [kvs {:Condition {:ArnLike {:AWS:SourceArn source-arn}}
    :Resource FunctionArn}]
    (when-not (submap? kvs statement)
    (clojure.pprint/pprint
    {:msg "permission statement incompatible with config"
    :data {:kvs kvs :statement statement}})
    (throw (ex-info "permission statement incompatible with config" {:kvs kvs :statement statement}))))

    (aws "lambda" "add-permission"
    "--function-name" function-name
    "--action" "lambda:InvokeFunction"
    "--principal" "apigateway.amazonaws.com"
    "--statement-id" statement-id
    "--source-arn" source-arn))))

    (defn sync-integration
    "Setup integration based via accreate-only changes"
    [region rest-api-id resource-id http-method function-arn timeout-in-millis]
    (when-not (aws "apigateway" "get-integration"
    "--rest-api-id" rest-api-id
    "--resource-id" resource-id
    "--http-method" http-method)

    (let [service_api (str "2015-03-31/functions/" function-arn "/invocations") ; TODO: Okay to hardcode this date?
    uri (str "arn:aws:apigateway:" region ":lambda:path/" service_api)]

    (aws "apigateway" "put-integration"
    "--rest-api-id" rest-api-id
    "--resource-id" resource-id
    "--http-method" http-method
    "--type" "AWS_PROXY"
    "--integration-http-method" "POST"
    "--timeout-in-millis" (str timeout-in-millis)
    "--uri" uri))))

    (defn sync-method
    "Setup method based via accreate-only changes"
    [rest-api-id resource-id http-method authorization-type]
    (if-let [method (aws "apigateway" "get-method"
    "--rest-api-id" rest-api-id
    "--resource-id" resource-id
    "--http-method" http-method)]

    (let [kvs {:httpMethod http-method :authorizationType authorization-type}]
    (when-not (submap? kvs method)
    (throw (ex-info "method incompatible with config" {:kvs kvs :method method}))))

    (aws "apigateway" "put-method"
    "--rest-api-id" rest-api-id
    "--resource-id" resource-id
    "--http-method" http-method
    "--authorization-type" authorization-type)))

    (defn sync-resource
    "Setup resource based via accreate-only changes"
    [client-id region group rest-api-id api-resources parent-id path-part methods]
    (let [api-resource (or (find-by (submap? {:parentId parent-id :pathPart path-part}) api-resources)
    (aws "apigateway" "create-resource"
    "--rest-api-id" rest-api-id
    "--parent-id" parent-id
    "--path-part" path-part))
    resource-id (:id api-resource)]

    (doseq [{:keys [http-method authorization-type integration]} methods]

    (let [{:keys [lambda-name timeout-in-millis]} integration
    function-name (str group "-" (name lambda-name))
    function (aws "lambda" "get-function" "--function-name" function-name)
    function-arn (get-in function [:Configuration :FunctionArn])]

    (sync-method rest-api-id resource-id http-method authorization-type)
    (sync-integration region rest-api-id resource-id http-method function-arn timeout-in-millis)
    (sync-permission region client-id rest-api-id path-part function-name function-arn)))))

    (defn get-rest-api-id
    [app-name]
    (let [ids (aws "apigateway" "get-rest-apis" "--query" (str "items[?name==`" app-name "`].id"))]
    (when (next ids)
    (throw (ex-info "Multiple matching rest-apis" {:name app-name :rest-api-ids ids})))
    (first ids)))

    (defn sync-rest-api
    "Setup REST API based via accreate-only changes"
    [api-config]
    (let [{:keys [app-name resources]} api-config
    {:keys [client-id region group]} (get-stack-context app-name)
    rest-api-id (or (get-rest-api-id app-name)
    (:id (aws "apigateway" "create-rest-api"
    "--name" app-name
    "--binary-media-types" "*/*"
    "--endpoint-configuration" "{\"types\" : [\"REGIONAL\"]}")))
    api-resources (:items (aws "apigateway" "get-resources" "--rest-api-id" rest-api-id))
    parent-id (:id (find-by (submap? {:path "/"}) api-resources))]

    (doseq [{:keys [path-part methods]} resources]
    (sync-resource client-id region group rest-api-id api-resources parent-id path-part methods))))

    (defn deploy-uri
    [app-name stage-name]
    (let [stacks (aws "cloudformation" "describe-stacks")
    app-stack (find-by (submap? {:StackName app-name}) (:Stacks stacks))
    [_ _ _ region & _] (string/split (:StackId app-stack) #":")
    rest-api-id (get-rest-api-id app-name)]
    (str "https://" rest-api-id ".execute-api." region ".amazonaws.com/" stage-name "/")))

    (defn deploy-api
    [app-name stage-name]
    (let [rest-api-id (get-rest-api-id app-name)]
    (aws "apigateway" "create-deployment" "--rest-api-id" rest-api-id "--stage-name" stage-name)))
    26 changes: 26 additions & 0 deletions aws_utils.clj
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,26 @@
    (ns deploy.aws-utils
    (:require [clojure.data.json :as json]
    [clojure.string :as string]
    [clojure.java.shell :as shell]))

    (defn submap?
    ([a] (partial submap? a))
    ([a b] (= a (select-keys b (keys a)))))

    (defn find-by [pred ms] (first (filter pred ms)))

    (defn aws [& args]
    (apply println "aws" args)
    (let [ret (apply shell/sh "aws" args)]
    (when (= 0 (:exit ret))
    (json/read-str (:out ret) :key-fn keyword :eof-error? false))))

    (defn get-stack-context
    [app-name]
    (let [app-stack (aws "cloudformation" "describe-stacks" "--query" (str "Stacks[?StackName==`" app-name "`] | [0]"))
    {:keys [StackId Outputs]} app-stack
    [_ _ _ region client-id & _] (string/split StackId #":")
    group (:OutputValue (find-by (submap? {:OutputKey "CodeDeployDeploymentGroup"}) Outputs))]
    {:client-id client-id
    :region region
    :group group}))
    17 changes: 0 additions & 17 deletions ion-config.edn
    View file
    Original file line number Diff line number Diff line change
    @@ -1,17 +0,0 @@
    {:allow [arthur.core/hello-world-web
    arthur.core/echo-web]
    :lambdas {:hello-world-web
    {:fn arthur.core/hello-world-web
    :integration :api-gateway/proxy
    :api-gateway/resources [{:path-part "hello-world"
    :methods [{:http-method "ANY"
    :authorization-type "NONE"
    :integration {:timeout-in-millis 29000}}]}]}
    :echo-web
    {:fn arthur.core/echo-web
    :integration :api-gateway/proxy
    :api-gateway/resources [{:path-part "echo"
    :methods [{:http-method "ANY"
    :authorization-type "NONE"
    :integration {:timeout-in-millis 29000}}]}]}}
    :app-name "og-condense-sandbox"}
    24 changes: 24 additions & 0 deletions ion_release.clj
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,24 @@
    (ns deploy.ion-release
    (:require [datomic.ion.dev :as ion-dev]
    [deploy.aws-utils :as aws-utils]))

    (defn release
    "Do push and deploy of app. Supports stable and unstable releases. Returns when deploy finishes running."
    [ion-config args]
    (try
    (let [group (:group (aws-utils/get-stack-context (:app-name ion-config)))
    push-data (ion-dev/push args)
    deploy-args (merge (select-keys args [:creds-profile :region :uname :group])
    (select-keys push-data [:rev])
    {:group group})]
    (let [deploy-data (ion-dev/deploy deploy-args)
    deploy-status-args (merge (select-keys args [:creds-profile :region])
    (select-keys deploy-data [:execution-arn]))]
    (loop []
    (let [status-data (ion-dev/deploy-status deploy-status-args)]
    (if (= "RUNNING" (:code-deploy-status status-data))
    (do (Thread/sleep 5000) (recur))
    status-data)))))
    (catch Exception e
    {:deploy-status "ERROR"
    :message (.getMessage e)})))
    186 changes: 9 additions & 177 deletions user.clj
    View file
    Original file line number Diff line number Diff line change
    @@ -1,181 +1,13 @@
    (ns user
    (:require [datomic.ion.dev :as ion-dev]
    [clojure.java.io :as io]
    [clojure.edn :as edn]
    [clojure.java.shell :as shell]
    [clojure.data.json :as json]
    [clojure.string :as string]))
    (:require [clojure.edn :as edn]
    [deploy.api-release :as api-release]
    [deploy.ion-release :as ion-release]))

    (def ion-config (edn/read-string (slurp "resources/datomic/ion-config.edn")))
    (def stage-name "dev")
    (def api-config (edn/read-string (slurp "resources/datomic/api-config.edn")))

    (defn submap?
    ([a] (partial submap? a))
    ([a b] (= a (select-keys b (keys a)))))

    (defn find-by [pred ms] (first (filter pred ms)))

    (defn aws [& args]
    (apply println "aws" args)
    (let [ret (apply shell/sh "aws" args)]
    (when (= 0 (:exit ret))
    (json/read-str (:out ret) :key-fn keyword :eof-error? false))))

    (defn stack-outputs [stack]
    (into {} (map (juxt (comp keyword :OutputKey) :OutputValue) (:Outputs stack))))


    (defn get-rest-apis []
    (:items (aws "apigateway" "get-rest-apis")))

    (defn create-rest-api []
    (aws "apigateway" "create-rest-api"
    "--name" (:app-name ion-config)
    "--binary-media-types" "*/*"
    "--endpoint-configuration" "{\"types\" : [\"REGIONAL\"]}"))

    (defn find-rest-api
    []
    (find-by (submap? {:name (:app-name ion-config)}) (get-rest-apis)))

    (defn upsert-rest-api
    []
    (let [rest-apis (get-rest-apis)
    name (:app-name ion-config)
    matches (filter (submap? {:name name}) rest-apis)
    [match & extra-matches] matches]
    (when extra-matches
    (throw (ex-info "Multiple matching rest-apis" {:name name :matches matches})))
    (or match (create-rest-api))))

    (defn get-resources
    [rest-api]
    (:items (aws "apigateway" "get-resources" "--rest-api-id" (:id rest-api))))

    (defn sync-rest-api
    "Setup REST API based on ion-config via accreate-only changes. Reports if breaking changes are required."
    []

    (let [stacks (aws "cloudformation" "describe-stacks")
    app-stack (find-by (submap? {:StackName (:app-name ion-config)}) (:Stacks stacks))
    [_ _ _ region client-id & _] (string/split (:StackId app-stack) #":")
    {:keys [CodeDeployDeploymentGroup]} (stack-outputs app-stack)
    rest-api (upsert-rest-api)
    resources (get-resources rest-api)
    rest-api-id (:id rest-api)
    parent-id (:id (find-by (submap? {:path "/"}) resources))]

    (doseq [[lambda-name lambda-config] (:lambdas ion-config)
    :when (submap? {:integration :api-gateway/proxy} lambda-config)
    {:keys [path-part methods]} (:api-gateway/resources lambda-config)]

    (let [resource (or (find-by (submap? {:parentId parent-id :pathPart path-part}) resources)
    (aws "apigateway" "create-resource"
    "--rest-api-id" rest-api-id
    "--parent-id" parent-id
    "--path-part" path-part))

    function-name (str CodeDeployDeploymentGroup "-" (name lambda-name))
    function (aws "lambda" "get-function" "--function-name" function-name)
    FunctionArn (get-in function [:Configuration :FunctionArn])]


    (doseq [{:keys [http-method authorization-type]} methods]

    (if-let [method (aws "apigateway" "get-method"
    "--rest-api-id" rest-api-id
    "--resource-id" (:id resource)
    "--http-method" http-method)]

    (let [kvs {:httpMethod http-method :authorizationType authorization-type}]
    (when-not (submap? kvs method)
    (throw (ex-info "method incompatible with config" {:kvs kvs :method method}))))

    (aws "apigateway" "put-method"
    "--rest-api-id" rest-api-id
    "--resource-id" (:id resource)
    "--http-method" http-method
    "--authorization-type" authorization-type))


    (when-not (aws "apigateway" "get-integration"
    "--rest-api-id" rest-api-id
    "--resource-id" (:id resource)
    "--http-method" http-method)

    (let [service_api (str "2015-03-31/functions/" FunctionArn "/invocations") ; TODO: Okay to hardcode this date?
    uri (str "arn:aws:apigateway:" region ":lambda:path/" service_api)]

    (aws "apigateway" "put-integration"
    "--rest-api-id" rest-api-id
    "--resource-id" (:id resource)
    "--http-method" http-method
    "--type" "AWS_PROXY"
    "--integration-http-method" "POST"
    "--timeout-in-millis" "29000"
    "--uri" uri))))


    (let [source-arn (str "arn:aws:execute-api:" region ":" client-id ":" rest-api-id "/*/*/" path-part)
    statement-id (str "ion-" (hash (str function-name source-arn)))
    {:keys [Policy]} (aws "lambda" "get-policy" "--function-name" function-name)
    policy-data (when Policy (json/read-str Policy :key-fn keyword))]

    (if-let [statement (find-by (submap? {:Sid statement-id}) (:Statement policy-data))]

    (let [kvs {:Condition {:ArnLike source-arn}
    :Resource FunctionArn}]
    (when-not (submap? kvs statement)
    (throw (ex-info "statement incompatible with config" {:kvs kvs :statement statement}))))

    (aws "lambda" "add-permission"
    "--function-name" function-name
    "--action" "lambda:InvokeFunction"
    "--principal" "apigateway.amazonaws.com"
    "--statement-id" statement-id
    "--source-arn" source-arn)))))))

    (defn deploy-api []
    (let [rest-api (find-rest-api)]
    (aws "apigateway" "create-deployment" "--rest-api-id" (:id rest-api) "--stage-name" stage-name)))

    (defn deploy-uri []
    (let [stacks (aws "cloudformation" "describe-stacks")
    app-stack (find-by (submap? {:StackName (:app-name ion-config)}) (:Stacks stacks))
    [_ _ _ region & _] (string/split (:StackId app-stack) #":")
    {:keys [id]} (find-rest-api)]
    (str "https://" id ".execute-api." region ".amazonaws.com/" stage-name "/")))

    (defn compute-group
    []
    (let [stacks (aws "cloudformation" "describe-stacks")
    app-stack (find-by (submap? {:StackName (:app-name ion-config)}) (:Stacks stacks))
    {:keys [CodeDeployDeploymentGroup]} (stack-outputs app-stack)]
    CodeDeployDeploymentGroup))

    (defn release
    "Do push and deploy of app. Supports stable and unstable releases. Returns when deploy finishes running."
    [args]
    (try
    (let [group (compute-group)
    push-data (ion-dev/push args)
    deploy-args (merge (select-keys args [:creds-profile :region :uname])
    (select-keys push-data [:rev])
    {:group group})]
    (let [deploy-data (ion-dev/deploy deploy-args)
    deploy-status-args (merge (select-keys args [:creds-profile :region])
    (select-keys deploy-data [:execution-arn]))]
    (loop []
    (let [status-data (ion-dev/deploy-status deploy-status-args)]
    (if (= "RUNNING" (:code-deploy-status status-data))
    (do (Thread/sleep 5000) (recur))
    status-data)))))
    (catch Exception e
    {:deploy-status "ERROR"
    :message (.getMessage e)})))

    (comment
    (release {})
    (sync-rest-api)
    (deploy-api)
    (deploy-uri))
    (defn deploy []
    (ion-release/release ion-config {})
    (api-release/sync-rest-api api-config)
    (api-release/deploy-api api-config "dev")
    (api-release/deploy-uri api-config "dev"))
  9. @olivergeorge olivergeorge revised this gist Jul 23, 2018. No changes.
  10. @olivergeorge olivergeorge revised this gist Jul 23, 2018. 1 changed file with 5 additions and 0 deletions.
    5 changes: 5 additions & 0 deletions README.md
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,5 @@
    Experiment to populate an AWS API Gateway for Datomic web service ions.

    Thoughts

    * Should be possible to separate from ion-config.edn (say rest-api.edn)
  11. @olivergeorge olivergeorge revised this gist Jul 23, 2018. 1 changed file with 5 additions and 11 deletions.
    16 changes: 5 additions & 11 deletions user.clj
    View file
    Original file line number Diff line number Diff line change
    @@ -77,12 +77,8 @@

    function-name (str CodeDeployDeploymentGroup "-" (name lambda-name))
    function (aws "lambda" "get-function" "--function-name" function-name)
    FunctionArn (get-in function [:Configuration :FunctionArn])
    ]
    FunctionArn (get-in function [:Configuration :FunctionArn])]

    (let [kvs {}]
    (when-not (submap? kvs resource)
    (throw (ex-info "resource incompatible with config" {:kvs kvs :resource resource}))))

    (doseq [{:keys [http-method authorization-type]} methods]

    @@ -119,17 +115,16 @@
    "--timeout-in-millis" "29000"
    "--uri" uri))))


    (let [source-arn (str "arn:aws:execute-api:" region ":" client-id ":" rest-api-id "/*/*/" path-part)
    statement-id (str "ion-" (hash (str function-name source-arn)))
    {:keys [Policy]} (aws "lambda" "get-policy" "--function-name" function-name)
    policy-data (when Policy (json/read-str Policy :key-fn keyword))
    statement (find-by (submap? {:Sid statement-id}) (:Statement policy-data))]
    policy-data (when Policy (json/read-str Policy :key-fn keyword))]

    (if statement
    (if-let [statement (find-by (submap? {:Sid statement-id}) (:Statement policy-data))]

    (let [kvs {:Condition {:ArnLike source-arn}
    :Resource FunctionArn}]

    (when-not (submap? kvs statement)
    (throw (ex-info "statement incompatible with config" {:kvs kvs :statement statement}))))

    @@ -147,7 +142,7 @@
    (defn deploy-uri []
    (let [stacks (aws "cloudformation" "describe-stacks")
    app-stack (find-by (submap? {:StackName (:app-name ion-config)}) (:Stacks stacks))
    [_ _ _ region client-id & _] (string/split (:StackId app-stack) #":")
    [_ _ _ region & _] (string/split (:StackId app-stack) #":")
    {:keys [id]} (find-rest-api)]
    (str "https://" id ".execute-api." region ".amazonaws.com/" stage-name "/")))

    @@ -184,4 +179,3 @@
    (sync-rest-api)
    (deploy-api)
    (deploy-uri))

  12. @olivergeorge olivergeorge revised this gist Jul 23, 2018. 1 changed file with 4 additions and 9 deletions.
    13 changes: 4 additions & 9 deletions user.clj
    View file
    Original file line number Diff line number Diff line change
    @@ -102,14 +102,10 @@
    "--authorization-type" authorization-type))


    (if-let [integration (aws "apigateway" "get-integration"
    "--rest-api-id" rest-api-id
    "--resource-id" (:id resource)
    "--http-method" http-method)]

    (let [kvs {}]
    (when-not (submap? kvs integration)
    (throw (ex-info "method incompatible with config" {:kvs kvs :integration integration}))))
    (when-not (aws "apigateway" "get-integration"
    "--rest-api-id" rest-api-id
    "--resource-id" (:id resource)
    "--http-method" http-method)

    (let [service_api (str "2015-03-31/functions/" FunctionArn "/invocations") ; TODO: Okay to hardcode this date?
    uri (str "arn:aws:apigateway:" region ":lambda:path/" service_api)]
    @@ -123,7 +119,6 @@
    "--timeout-in-millis" "29000"
    "--uri" uri))))


    (let [source-arn (str "arn:aws:execute-api:" region ":" client-id ":" rest-api-id "/*/*/" path-part)
    statement-id (str "ion-" (hash (str function-name source-arn)))
    {:keys [Policy]} (aws "lambda" "get-policy" "--function-name" function-name)
  13. @olivergeorge olivergeorge created this gist Jul 23, 2018.
    17 changes: 17 additions & 0 deletions ion-config.edn
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,17 @@
    {:allow [arthur.core/hello-world-web
    arthur.core/echo-web]
    :lambdas {:hello-world-web
    {:fn arthur.core/hello-world-web
    :integration :api-gateway/proxy
    :api-gateway/resources [{:path-part "hello-world"
    :methods [{:http-method "ANY"
    :authorization-type "NONE"
    :integration {:timeout-in-millis 29000}}]}]}
    :echo-web
    {:fn arthur.core/echo-web
    :integration :api-gateway/proxy
    :api-gateway/resources [{:path-part "echo"
    :methods [{:http-method "ANY"
    :authorization-type "NONE"
    :integration {:timeout-in-millis 29000}}]}]}}
    :app-name "og-condense-sandbox"}
    192 changes: 192 additions & 0 deletions user.clj
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,192 @@
    (ns user
    (:require [datomic.ion.dev :as ion-dev]
    [clojure.java.io :as io]
    [clojure.edn :as edn]
    [clojure.java.shell :as shell]
    [clojure.data.json :as json]
    [clojure.string :as string]))

    (def ion-config (edn/read-string (slurp "resources/datomic/ion-config.edn")))
    (def stage-name "dev")

    (defn submap?
    ([a] (partial submap? a))
    ([a b] (= a (select-keys b (keys a)))))

    (defn find-by [pred ms] (first (filter pred ms)))

    (defn aws [& args]
    (apply println "aws" args)
    (let [ret (apply shell/sh "aws" args)]
    (when (= 0 (:exit ret))
    (json/read-str (:out ret) :key-fn keyword :eof-error? false))))

    (defn stack-outputs [stack]
    (into {} (map (juxt (comp keyword :OutputKey) :OutputValue) (:Outputs stack))))


    (defn get-rest-apis []
    (:items (aws "apigateway" "get-rest-apis")))

    (defn create-rest-api []
    (aws "apigateway" "create-rest-api"
    "--name" (:app-name ion-config)
    "--binary-media-types" "*/*"
    "--endpoint-configuration" "{\"types\" : [\"REGIONAL\"]}"))

    (defn find-rest-api
    []
    (find-by (submap? {:name (:app-name ion-config)}) (get-rest-apis)))

    (defn upsert-rest-api
    []
    (let [rest-apis (get-rest-apis)
    name (:app-name ion-config)
    matches (filter (submap? {:name name}) rest-apis)
    [match & extra-matches] matches]
    (when extra-matches
    (throw (ex-info "Multiple matching rest-apis" {:name name :matches matches})))
    (or match (create-rest-api))))

    (defn get-resources
    [rest-api]
    (:items (aws "apigateway" "get-resources" "--rest-api-id" (:id rest-api))))

    (defn sync-rest-api
    "Setup REST API based on ion-config via accreate-only changes. Reports if breaking changes are required."
    []

    (let [stacks (aws "cloudformation" "describe-stacks")
    app-stack (find-by (submap? {:StackName (:app-name ion-config)}) (:Stacks stacks))
    [_ _ _ region client-id & _] (string/split (:StackId app-stack) #":")
    {:keys [CodeDeployDeploymentGroup]} (stack-outputs app-stack)
    rest-api (upsert-rest-api)
    resources (get-resources rest-api)
    rest-api-id (:id rest-api)
    parent-id (:id (find-by (submap? {:path "/"}) resources))]

    (doseq [[lambda-name lambda-config] (:lambdas ion-config)
    :when (submap? {:integration :api-gateway/proxy} lambda-config)
    {:keys [path-part methods]} (:api-gateway/resources lambda-config)]

    (let [resource (or (find-by (submap? {:parentId parent-id :pathPart path-part}) resources)
    (aws "apigateway" "create-resource"
    "--rest-api-id" rest-api-id
    "--parent-id" parent-id
    "--path-part" path-part))

    function-name (str CodeDeployDeploymentGroup "-" (name lambda-name))
    function (aws "lambda" "get-function" "--function-name" function-name)
    FunctionArn (get-in function [:Configuration :FunctionArn])
    ]

    (let [kvs {}]
    (when-not (submap? kvs resource)
    (throw (ex-info "resource incompatible with config" {:kvs kvs :resource resource}))))

    (doseq [{:keys [http-method authorization-type]} methods]

    (if-let [method (aws "apigateway" "get-method"
    "--rest-api-id" rest-api-id
    "--resource-id" (:id resource)
    "--http-method" http-method)]

    (let [kvs {:httpMethod http-method :authorizationType authorization-type}]
    (when-not (submap? kvs method)
    (throw (ex-info "method incompatible with config" {:kvs kvs :method method}))))

    (aws "apigateway" "put-method"
    "--rest-api-id" rest-api-id
    "--resource-id" (:id resource)
    "--http-method" http-method
    "--authorization-type" authorization-type))


    (if-let [integration (aws "apigateway" "get-integration"
    "--rest-api-id" rest-api-id
    "--resource-id" (:id resource)
    "--http-method" http-method)]

    (let [kvs {}]
    (when-not (submap? kvs integration)
    (throw (ex-info "method incompatible with config" {:kvs kvs :integration integration}))))

    (let [service_api (str "2015-03-31/functions/" FunctionArn "/invocations") ; TODO: Okay to hardcode this date?
    uri (str "arn:aws:apigateway:" region ":lambda:path/" service_api)]

    (aws "apigateway" "put-integration"
    "--rest-api-id" rest-api-id
    "--resource-id" (:id resource)
    "--http-method" http-method
    "--type" "AWS_PROXY"
    "--integration-http-method" "POST"
    "--timeout-in-millis" "29000"
    "--uri" uri))))


    (let [source-arn (str "arn:aws:execute-api:" region ":" client-id ":" rest-api-id "/*/*/" path-part)
    statement-id (str "ion-" (hash (str function-name source-arn)))
    {:keys [Policy]} (aws "lambda" "get-policy" "--function-name" function-name)
    policy-data (when Policy (json/read-str Policy :key-fn keyword))
    statement (find-by (submap? {:Sid statement-id}) (:Statement policy-data))]

    (if statement

    (let [kvs {:Condition {:ArnLike source-arn}
    :Resource FunctionArn}]

    (when-not (submap? kvs statement)
    (throw (ex-info "statement incompatible with config" {:kvs kvs :statement statement}))))

    (aws "lambda" "add-permission"
    "--function-name" function-name
    "--action" "lambda:InvokeFunction"
    "--principal" "apigateway.amazonaws.com"
    "--statement-id" statement-id
    "--source-arn" source-arn)))))))

    (defn deploy-api []
    (let [rest-api (find-rest-api)]
    (aws "apigateway" "create-deployment" "--rest-api-id" (:id rest-api) "--stage-name" stage-name)))

    (defn deploy-uri []
    (let [stacks (aws "cloudformation" "describe-stacks")
    app-stack (find-by (submap? {:StackName (:app-name ion-config)}) (:Stacks stacks))
    [_ _ _ region client-id & _] (string/split (:StackId app-stack) #":")
    {:keys [id]} (find-rest-api)]
    (str "https://" id ".execute-api." region ".amazonaws.com/" stage-name "/")))

    (defn compute-group
    []
    (let [stacks (aws "cloudformation" "describe-stacks")
    app-stack (find-by (submap? {:StackName (:app-name ion-config)}) (:Stacks stacks))
    {:keys [CodeDeployDeploymentGroup]} (stack-outputs app-stack)]
    CodeDeployDeploymentGroup))

    (defn release
    "Do push and deploy of app. Supports stable and unstable releases. Returns when deploy finishes running."
    [args]
    (try
    (let [group (compute-group)
    push-data (ion-dev/push args)
    deploy-args (merge (select-keys args [:creds-profile :region :uname])
    (select-keys push-data [:rev])
    {:group group})]
    (let [deploy-data (ion-dev/deploy deploy-args)
    deploy-status-args (merge (select-keys args [:creds-profile :region])
    (select-keys deploy-data [:execution-arn]))]
    (loop []
    (let [status-data (ion-dev/deploy-status deploy-status-args)]
    (if (= "RUNNING" (:code-deploy-status status-data))
    (do (Thread/sleep 5000) (recur))
    status-data)))))
    (catch Exception e
    {:deploy-status "ERROR"
    :message (.getMessage e)})))

    (comment
    (release {})
    (sync-rest-api)
    (deploy-api)
    (deploy-uri))