Created
March 10, 2015 08:15
-
-
Save kunnet/dd727036f3fb623c4304 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from subprocess import Popen, PIPE | |
| from threading import Thread | |
| from Queue import Queue, Empty | |
| class CommandlineExecutor(object): | |
| STDERR = 0 | |
| STDOUT = 1 | |
| def __init__(self, command, output_file=None, preview_lines=10): | |
| self.io_q_stdout = Queue() | |
| self.io_q_stderr = Queue() | |
| self.result_lines = 0 | |
| self.proc = Popen(shlex.split(command), stdout=PIPE, stderr=PIPE) | |
| Thread(target=self.io_stream_watcher, name='stdout-watcher', | |
| args=('STDOUT', self.proc.stdout, output_file, preview_lines)).start() | |
| Thread(target=self.io_stream_watcher, name='stderr-watcher', | |
| args=('STDERR', self.proc.stderr)).start() | |
| def io_stream_watcher(self, identifier, stream, filepath=None, preview_lines=10): | |
| if identifier == 'STDOUT': | |
| if filepath: | |
| with open(filepath, 'w') as datafile: | |
| for line in stream: | |
| datafile.write(line.decode('UTF-8', 'ignore').encode('gbk', 'ignore')) | |
| if self.result_lines < preview_lines: | |
| self.io_q_stdout.put(line) | |
| self.result_lines += 1 | |
| else: | |
| for line in stream: | |
| self.io_q_stdout.put(line) | |
| elif identifier == 'STDERR': | |
| for line in stream: | |
| self.io_q_stderr.put(line) | |
| if not stream.closed: | |
| stream.close() | |
| def check_output(self, output_type): | |
| if output_type == self.STDERR: | |
| queue = self.io_q_stderr | |
| else: | |
| queue = self.io_q_stdout | |
| output_string = '' | |
| while True: | |
| try: | |
| item = queue.get(True, 1) | |
| except Empty: | |
| if self.proc.poll() is not None: | |
| break | |
| else: | |
| output_string += item | |
| return output_string | |
| preview_lines = 20 # 结果预览行数 | |
| beeline_command='cat large_file' | |
| filepath='output.txt' | |
| ce = CommandlineExecutor(beeline_command, output_file=filepath, preview_lines=preview_lines) | |
| err = ce.check_output(ce.STDERR) # 取错误输出结果 | |
| if err: | |
| raise Exception(err) | |
| else: | |
| result_preview = ce.check_output(ce.STDOUT) # 取标准输出结果 | |
| count = ce.result_lines | |
| print result_preview,'OK',count |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment