#!/usr/bin/env python3 # parallel_lambda_test.py import random import pendulum import time from concurrent import futures # The problem is that the AWS boto3 lambda invoke function is not truly asynchronous # even with InvocationType='Event', so to achieve concurrency, we rely on the # ThreadPoolExecutor, which depending on the vCPU size can invoke ~60 lambdas/sec def fakeLambda(batch, rank, invoke_time): sleep_time = random.randint(3, 8) time.sleep(sleep_time) curr_time = pendulum.now(tz="UTC") print(f" ..finished. {batch} - #{rank}, took {sleep_time} sec - invoke {invoke_time.strftime('%H:%M:%S')}, now: {curr_time.strftime('%H:%M:%S')}") def loopLoop(): with futures.ThreadPoolExecutor() as executor: batch = 0 while True: curr_time = pendulum.now(tz="UTC") print(f"Batch {batch} - {curr_time.strftime('%H:%M:%S')}") for i in range(5): executor.submit(fakeLambda, batch, i, curr_time) time.sleep(5) batch += 1 def main(): loopLoop() if __name__ == "__main__": main()