cmdoptesc · November 22, 2019 02:56 · Nov 22, 2019
diff --git a/parallel_lambda_test.py b/parallel_lambda_test.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+# parallel_lambda_test.py
+
+import random
+import pendulum
+import time
+from concurrent import futures
+
+
+# The problem is that the AWS boto3 lambda invoke function is not truly asynchronous
+# even with InvocationType='Event', so to achieve concurrency, we rely on the
+# ThreadPoolExecutor, which depending on the vCPU size can invoke ~60 lambdas/sec
+
+def fakeLambda(batch, rank, invoke_time):
+    sleep_time = random.randint(3, 8)
+    time.sleep(sleep_time)
+    curr_time = pendulum.now(tz="UTC")
+    print(f"    ..finished. {batch} - #{rank}, took {sleep_time} sec - invoke {invoke_time.strftime('%H:%M:%S')}, now: {curr_time.strftime('%H:%M:%S')}")
+
+
+def loopLoop():
+    with futures.ThreadPoolExecutor() as executor:
+        batch = 0
+
+        while True:
+            curr_time = pendulum.now(tz="UTC")
+            print(f"Batch {batch} - {curr_time.strftime('%H:%M:%S')}")
+
+            for i in range(5):
+                executor.submit(fakeLambda, batch, i, curr_time)
+
+            time.sleep(5)
+            batch += 1
+
+
+def main():
+    loopLoop()
+
+
+if __name__ == "__main__":
+    main()