Skip to content

⛏️sproc: subprocesseses for subhumanses ⛏

Useful for handling long-running proceesses that write to both stdout and stderr.

Simple Example

import sproc

CMD = 'my-unix-command "My Cool File.txt" No-file.txt'

for ok, line in sproc.Sub(CMD) as sp:
    if ok:
         print(' ', line)
    else:
         print('!', line)

if sp.returncode:
    print('Error code', sp.returncode)

# Return two lists of text lines and a returncode
out_lines, err_lines, returncode = sproc.run(CMD)

# Call callback functions with lines of text read from stdout and stderr
returncode = sproc.call(CMD, save_results, print_errors)

# Log stdout and stderr, with prefixes
returncode = sproc.log(CMD)

API Documentation

Sub

Sub is a class to Iterate over lines or chunks of text from a subprocess.

If by_lines is true, use readline() to get each new item; if false, use read1().

Parameters:

Name Type Description Default
cmd Cmd

The command to run in a subprocess

 required
by_lines bool

If by_lines is true, Sub uses readline() to get each new item; otherwise, it uses read1() to get each chunk as it comes.

 True
kwargs Mapping

The arguments to subprocess.Popen.

If kwargs['shell'] is true, Popen expects a string, and so if cmd is not a string, it is joined using shlex.

If kwargs['shell'] is false, Popen expects a list of strings, and so if cmd is a string, it is split using shlex.

 {}
Source code in sproc/sproc.py 
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
 
class Sub:
    """
    Sub is a class to Iterate over lines or chunks of text from a subprocess.

    If `by_lines` is true, use readline() to get each new item;
    if false, use read1().

    Args:
      cmd:  The command to run in a subprocess

      by_lines:  If `by_lines` is true, `Sub` uses readline() to get each new
          item;  otherwise, it uses read1() to get each chunk as it comes.

      kwargs: The arguments to subprocess.Popen.

          If `kwargs['shell']` is true, `Popen` expects a string,
          and so if `cmd` is not a string, it is joined using `shlex`.

          If `kwargs['shell']` is false, `Popen` expects a list of strings,
          and so if `cmd` is a string, it is split using `shlex`.
    """
    @functools.wraps(subprocess.Popen)
    def __init__(self, cmd: Cmd, *, by_lines: bool = True, **kwargs: Mapping):
        if 'stdout' in kwargs or 'stderr' in kwargs:
            raise ValueError('Cannot set stdout or stderr')

        self.cmd = cmd
        self.by_lines = by_lines
        self.kwargs = dict(kwargs, **DEFAULTS)
        self._threads = []

        shell = kwargs.get('shell', False)
        if isinstance(cmd, str):
            if not shell:
                self.cmd = shlex.split(cmd)
        else:
            if shell:
                self.cmd = shlex.join(cmd)

    @property
    def returncode(self):
        return self.proc and self.proc.returncode

    def __iter__(self):
        """
        Yields a sequence of `ok, line` pairs from `stdout` and `stderr` of
        a subprocess, where `ok` is `True` if `line` came from `stdout`
        and `False` if it came from `stderr`.

        After iteration is done, the `.returncode` property contains
        the error code from the subprocess, an integer where 0 means no error.
        """
        queue = Queue()

        with subprocess.Popen(self.cmd, **self.kwargs) as self.proc:
            for ok in False, True:
                self._start_thread(ok, lambda o, s: queue.put((o, s)))

            finished = 0
            while finished < 2:
                ok, line = queue.get()
                if line:
                    yield ok, line
                else:
                    finished += 1

    def call(self, out: Callback = None, err: Callback = None):
        """
        Run the subprocess, and call function `out` with lines from
        `stdout` and function `err` with lines from `stderr`.

        Blocks until the subprocess is complete: the callbacks to `out` and
        'err` are on the current thread.

        Args:
            out: if not None, `out` is called for each line from the
                subprocess's stdout

            err: if not None, `err` is called for each line from the
                subprocess's stderr,
        """
        callback = self._callback(out, err)
        for ok, line in self:
            callback(ok, line)

        return self.returncode

    def call_async(self, out: Callback = None, err: Callback = None):
        # DEPRECATED: now called "call_in_thread"
        return self.call_in_thread(out, err)

    def call_in_thread(self, out: Callback = None, err: Callback = None):
        """
        Run the subprocess, and asynchronously call function `out` with lines
        from `stdout`, and function `err` with lines from `stderr`.

        Does not block - immediately returns.

        Args:
            out: If not None, `out` is called for each line from the
                subprocess's stdout

            err: If not None, `err` is called for each line from the
                subprocess's stderr,
    """
        with subprocess.Popen(self.cmd, **self.kwargs) as self.proc:
            callback = self._callback(out, err)
            for ok in False, True:
                self._start_thread(ok, callback)

    def run(self):
        """
        Reads lines from `stdout` and `stderr` into two lists `out` and `err`,
        then returns a tuple `(out, err, returncode)`
        """
        out, err = [], []
        self.call(out.append, err.append)
        return out, err, self.returncode

    def log(self, out: str = '  ', err: str = '! ', print: Callable = print):
        """
        Read lines from `stdin` and `stderr` and prints them with prefixes

        Returns the shell integer error code from the subprocess, where 0 means
        no error.

        Args:
            out: The contents of `out` prepends strings from stdout
            err: The contents of `err` prepends strings from stderr
            print: A function that accepts individual strings
        """
        return self.call(lambda x: print(out + x), lambda x: print(err + x))

    def join(self, timeout: Optional[int] = None):
        """Join the stream handling threads"""
        for t in self._threads:
            t.join(timeout)

    def kill(self):
        """Kill the running process, if any"""
        self.proc and self.proc.kill()

    def _start_thread(self, ok, callback):
        def read_stream():
            try:
                stream = self.proc.stdout if ok else self.proc.stderr
                line = '.'
                while line or self.proc.poll() is None:
                    if self.by_lines:
                        line = stream.readline()
                    else:
                        line = stream.read1()

                    if line:
                        if not isinstance(line, str):
                            line = line.decode('utf8')
                        callback(ok, line)
            finally:
                callback(ok, None)

        th = Thread(target=read_stream, daemon=True)
        th.start()
        self._threads.append(th)

    def _callback(self, out, err):
        if out and err:
            return lambda ok, line: line and (out(line) if ok else err(line))
        if out:
            return lambda ok, line: line and ok and out(line)
        if err:
            return lambda ok, line: line and not ok and err(line)
        else:
            return lambda ok, line: None

__iter__()

Yields a sequence of ok, line pairs from stdout and stderr of a subprocess, where ok is True if line came from stdout and False if it came from stderr.

After iteration is done, the .returncode property contains the error code from the subprocess, an integer where 0 means no error.

Source code in sproc/sproc.py 
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
 
def __iter__(self):
    """
    Yields a sequence of `ok, line` pairs from `stdout` and `stderr` of
    a subprocess, where `ok` is `True` if `line` came from `stdout`
    and `False` if it came from `stderr`.

    After iteration is done, the `.returncode` property contains
    the error code from the subprocess, an integer where 0 means no error.
    """
    queue = Queue()

    with subprocess.Popen(self.cmd, **self.kwargs) as self.proc:
        for ok in False, True:
            self._start_thread(ok, lambda o, s: queue.put((o, s)))

        finished = 0
        while finished < 2:
            ok, line = queue.get()
            if line:
                yield ok, line
            else:
                finished += 1

call(out=None, err=None)

Run the subprocess, and call function out with lines from stdout and function err with lines from stderr.

Blocks until the subprocess is complete: the callbacks to out and 'err` are on the current thread.

Parameters:

Name Type Description Default
out Callback

if not None, out is called for each line from the subprocess's stdout

 None
err Callback

if not None, err is called for each line from the subprocess's stderr,

 None
Source code in sproc/sproc.py 
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
def call(self, out: Callback = None, err: Callback = None):
    """
    Run the subprocess, and call function `out` with lines from
    `stdout` and function `err` with lines from `stderr`.

    Blocks until the subprocess is complete: the callbacks to `out` and
    'err` are on the current thread.

    Args:
        out: if not None, `out` is called for each line from the
            subprocess's stdout

        err: if not None, `err` is called for each line from the
            subprocess's stderr,
    """
    callback = self._callback(out, err)
    for ok, line in self:
        callback(ok, line)

    return self.returncode

call_in_thread(out=None, err=None)

Run the subprocess, and asynchronously call function out with lines from stdout, and function err with lines from stderr.

Does not block - immediately returns.

Parameters:

Name Type Description Default
out Callback

If not None, out is called for each line from the subprocess's stdout

 None
err Callback

If not None, err is called for each line from the subprocess's stderr,

 None
Source code in sproc/sproc.py 
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
 
def call_in_thread(self, out: Callback = None, err: Callback = None):
    """
    Run the subprocess, and asynchronously call function `out` with lines
    from `stdout`, and function `err` with lines from `stderr`.

    Does not block - immediately returns.

    Args:
        out: If not None, `out` is called for each line from the
            subprocess's stdout

        err: If not None, `err` is called for each line from the
            subprocess's stderr,
"""
    with subprocess.Popen(self.cmd, **self.kwargs) as self.proc:
        callback = self._callback(out, err)
        for ok in False, True:
            self._start_thread(ok, callback)

join(timeout=None)

Join the stream handling threads

Source code in sproc/sproc.py 
183
184
185
186
 
def join(self, timeout: Optional[int] = None):
    """Join the stream handling threads"""
    for t in self._threads:
        t.join(timeout)

kill()

Kill the running process, if any

Source code in sproc/sproc.py 
188
189
190
 
def kill(self):
    """Kill the running process, if any"""
    self.proc and self.proc.kill()

log(out=' ', err='! ', print=print)

Read lines from stdin and stderr and prints them with prefixes

Returns the shell integer error code from the subprocess, where 0 means no error.

Parameters:

Name Type Description Default
out str

The contents of out prepends strings from stdout

 ' '
err str

The contents of err prepends strings from stderr

 '! '
print Callable

A function that accepts individual strings

 print
Source code in sproc/sproc.py 
169
170
171
172
173
174
175
176
177
178
179
180
181
 
def log(self, out: str = '  ', err: str = '! ', print: Callable = print):
    """
    Read lines from `stdin` and `stderr` and prints them with prefixes

    Returns the shell integer error code from the subprocess, where 0 means
    no error.

    Args:
        out: The contents of `out` prepends strings from stdout
        err: The contents of `err` prepends strings from stderr
        print: A function that accepts individual strings
    """
    return self.call(lambda x: print(out + x), lambda x: print(err + x))

run()

Reads lines from stdout and stderr into two lists out and err, then returns a tuple (out, err, returncode)

Source code in sproc/sproc.py 
160
161
162
163
164
165
166
167
 
def run(self):
    """
    Reads lines from `stdout` and `stderr` into two lists `out` and `err`,
    then returns a tuple `(out, err, returncode)`
    """
    out, err = [], []
    self.call(out.append, err.append)
    return out, err, self.returncode

call(cmd, out=None, err=None, **kwargs)

Parameters:

Name Type Description Default
cmd Cmd

The command to run in a subprocess

 required
out Callback

if not None, out is called for each line from the subprocess's stdout

 None
err Callback

if not None, err is called for each line from the subprocess's stderr,

 None
kwargs Mapping

The arguments to subprocess.Popen.

 {}
Source code in sproc/sproc.py 
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
 
def call(
    cmd: Cmd, out: Callback = None, err: Callback = None, **kwargs: Mapping
):
    """
    Args:
      cmd:  The command to run in a subprocess

      out: if not None, `out` is called for each line from the
          subprocess's stdout

      err: if not None, `err` is called for each line from the
          subprocess's stderr,

      kwargs: The arguments to subprocess.Popen.
    """
    return Sub(cmd, **kwargs).call(out, err)

call_in_thread(cmd, out=None, err=None, **kwargs)

Parameters:

Name Type Description Default
cmd Cmd

The command to run in a subprocess

 required
out Callback

if not None, out is called for each line from the subprocess's stdout

 None
err Callback

if not None, err is called for each line from the subprocess's stderr,

 None
kwargs Mapping

The arguments to subprocess.Popen.

 {}
Source code in sproc/sproc.py 
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
 
def call_in_thread(
    cmd: Cmd, out: Callback = None, err: Callback = None, **kwargs: Mapping
):
    """
    Args:
      cmd:  The command to run in a subprocess

      out: if not None, `out` is called for each line from the
          subprocess's stdout

      err: if not None, `err` is called for each line from the
          subprocess's stderr,

      kwargs: The arguments to subprocess.Popen.
    """
    return Sub(cmd, **kwargs).call_in_thread(out, err)

log(cmd, out=' ', err='! ', print=print, **kwargs)

Parameters:

Name Type Description Default
cmd Cmd

The command to run in a subprocess

 required
out str

The contents of out prepends strings from stdout

 ' '
err str

The contents of err prepends strings from stderr

 '! '
print Callable

A function that accepts individual strings

 print
Source code in sproc/sproc.py 
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
 
def log(
    cmd: Cmd,
    out: str = '  ',
    err: str = '! ',
    print: Callable = print,
    **kwargs
):
    """
    Args:
        cmd:  The command to run in a subprocess
        out: The contents of `out` prepends strings from stdout
        err: The contents of `err` prepends strings from stderr
        print: A function that accepts individual strings
    """
    return Sub(cmd, **kwargs).log(out, err, print)

About this project