-
-
Save benblackcake/6a167293ab97140d36db3fd4911d2f85 to your computer and use it in GitHub Desktop.
Threaded generator wrapper for Python
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # A simple generator wrapper, not sure if it's good for anything at all. | |
| # With basic python threading | |
| from threading import Thread | |
| try: | |
| from queue import Queue | |
| except ImportError: | |
| from Queue import Queue | |
| # ... or use multiprocessing versions | |
| # WARNING: use sentinel based on value, not identity | |
| from multiprocessing import Process, Queue as MpQueue | |
| class ThreadedGenerator(object): | |
| """ | |
| Generator that runs on a separate thread, returning values to calling | |
| thread. Care must be taken that the iterator does not mutate any shared | |
| variables referenced in the calling thread. | |
| """ | |
| def __init__(self, iterator, | |
| sentinel=object(), | |
| queue_maxsize=0, | |
| daemon=False, | |
| Thread=Thread, | |
| Queue=Queue): | |
| self._iterator = iterator | |
| self._sentinel = sentinel | |
| self._queue = Queue(maxsize=queue_maxsize) | |
| self._thread = Thread( | |
| name=repr(iterator), | |
| target=self._run | |
| ) | |
| self._thread.daemon = daemon | |
| def __repr__(self): | |
| return 'ThreadedGenerator({!r})'.format(self._iterator) | |
| def _run(self): | |
| try: | |
| for value in self._iterator: | |
| self._queue.put(value) | |
| finally: | |
| self._queue.put(self._sentinel) | |
| def __iter__(self): | |
| self._thread.start() | |
| for value in iter(self._queue.get, self._sentinel): | |
| yield value | |
| self._thread.join() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment