craigpatten · September 10, 2015 03:15 · Sep 10, 2015 · Sep 10, 2015
diff --git a/inhale.py b/inhale.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""
+Yank data from Swift and do [something] with it.
+"""
+
+import json
+import lzma
+import os
+import sys
+
+from multiprocessing import Pool
+
+import swiftclient
+
+
+class ObjectStorage:
+    def __init__(self, swift_config, container, prefix=None):
+        self.swift = swiftclient.client.Connection(**swift_config)
+        self.container = container
+        self.prefix = prefix
+
+    def list_container(self):
+        listing = self.swift.get_container(self.container,
+                                           prefix=self.prefix,
+                                           full_listing=True)
+        return [item["name"] for item in listing[1]]
+
+    def process(self, name):
+        compressed = self.swift.get_object(self.container, name)[1]
+        raw = lzma.decompress(compressed)
+        data = json.loads(raw.decode("utf-8"))
+        # do something with the data ...
+
+
+if __name__ == "__main__":
+    swift_config = {
+        "auth_version": 2,
+        "authurl": os.getenv("OS_AUTH_URL"),
+        "user": os.getenv("OS_USERNAME"),
+        "key": os.getenv("OS_PASSWORD"),
+        "tenant_name": os.getenv("OS_TENANT_NAME")
+    }
+
+    if len(sys.argv) != 4:
+        print("args: workers container object-name-prefix")
+        sys.exit(1)
+
+    workers = int(sys.argv[1])
+    container = sys.argv[2]
+    prefix = sys.argv[3]
+
+    os = ObjectStorage(swift_config, container, prefix)
+
+    names = os.list_container()
+
+    with Pool(workers) as pool:
+        pool.map(os.process, names)