LokiSharp · November 26, 2018 06:52 · Nov 26, 2018
diff --git a/get_wsgr_manifest.py b/get_wsgr_manifest.py
@@ -0,0 +1,108 @@
+import os
+import json
+import requests
+import re
+import threading
+import queue
+from tqdm import tqdm
+from pprint import pprint
+from hashlib import md5
+
+
+def md5sum(file):
+    m = md5()
+    a_file = open(file, 'rb')
+    m.update(a_file.read())
+    a_file.close()
+    return m.hexdigest()
+
+
+def check_file_md5(file_path, item):
+    if not (md5sum(file_path) == item['md5']):
+        print(file_path)
+        return False
+    else:
+        return True
+
+
+def download(file_path, item):
+    m = re.split('/', item['name'])
+    dir_path = folder_path + item['name'][:-len(m[-1])]
+    try:
+        if not os.path.exists(dir_path):
+            os.makedirs(dir_path)
+    except:
+        pass
+    finally:
+        r = s.get(load_dict['packageUrl'] + item['name'])
+        with open(folder_path + item['name'], "wb") as file:
+            file.write(r.content)
+        check_file_md5(file_path, item)
+
+
+def main(item):
+    try:
+        file_path = folder_path + item['name']
+        if not os.path.exists(file_path):
+            download(file_path, item)
+        elif not check_file_md5(file_path, item):
+            download(file_path, item)
+    except Exception as e:
+        print(e)
+
+
+def process_data(q):
+    while not exitFlag:
+        queueLock.acquire()
+        if not workQueue.empty():
+            item = q.get()
+            queueLock.release()
+            main(item)
+        else:
+            queueLock.release()
+
+
+class myThread (threading.Thread):
+    def __init__(self, threadID, name, q):
+        threading.Thread.__init__(self)
+        self.threadID = threadID
+        self.name = name
+        self.q = q
+
+    def run(self):
+        print("Start: " + self.name)
+        process_data(self.q)
+        print("End: " + self.name)
+
+
+if __name__ == '__main__':
+    exitFlag = 0
+
+    s = requests.Session()
+    folder_path = './data/'
+    with open('./warshipgirlsr.manifest') as f:
+        load_dict = json.load(f)
+    threadList = [f"Thread-{i}" for i in range(100)]
+    queueLock = threading.Lock()
+    workQueue = queue.Queue()
+    threadID = 0
+    threads = []
+    for tName in threadList:
+        thread = myThread(threadID, tName, workQueue)
+        thread.start()
+        threads.append(thread)
+        threadID += 1
+
+    queueLock.acquire()
+    for item in load_dict['hot']:
+        workQueue.put(item)
+    queueLock.release()
+
+    while not workQueue.empty():
+        pass
+
+    exitFlag = 1
+
+    for t in threads:
+        t.join()
+    print("End Of All")
No results found