saswata-dutta · August 8, 2023 10:26 · Aug 8, 2023
diff --git a/s3_select_key_prefix.py b/s3_select_key_prefix.py
@@ -0,0 +1,63 @@
+import boto3
+import datetime
+
+
+s3 = boto3.client("s3")
+bucket = "???"
+prefix_base = "actionType=cancel"
+query = """SELECT subjectid, s."timestamp" FROM s3object s where clientid = '???' and status = 'UNCANCELLABLE'"""
+
+
+def fetch_result(key):
+    resp = s3.select_object_content(
+        Bucket=bucket,
+        Key=key,
+        ExpressionType="SQL",
+        Expression=query,
+        InputSerialization={"Parquet": {}},
+        OutputSerialization={"CSV": {}},
+    )
+    for event in resp["Payload"]:
+        if "Records" in event:
+            records = event["Records"]["Payload"].decode("utf-8")
+            print(records.strip())
+
+
+def fetch_keys(prefix):
+    resp = s3.list_objects_v2(Bucket=bucket, Prefix=prefix)
+    return [it["Key"] for it in resp["Contents"]]
+
+
+def to_date(str_date):
+    return datetime.datetime.strptime(str_date, "%Y-%m-%d")
+
+
+def date_range(start, end):
+    start = to_date(start)
+    end = to_date(end)
+    days = (end - start).days + 1
+    return [start + datetime.timedelta(days=i) for i in range(days)]
+
+
+def to_prefix(date):
+    suffix = date.strftime("year=%Y/month=%m/day=%d")
+    return f"{prefix_base}/{suffix}/"
+
+
+def prefix_range(start, end):
+    prefixes = [to_prefix(d) for d in date_range(start, end)]
+    return prefixes
+
+
+def fetch_keys_range(start, end):
+    prefixes = prefix_range(start, end)
+    return [k for p in prefixes for k in fetch_keys(p)]
+
+
+def query_range(start, end):
+    keys = fetch_keys_range(start, end)
+    for k in keys:
+        fetch_result(k)
+
+
+query_range("2022-08-15", "2022-08-19")
No results found