Python subprocess32 模块，PIPE 实例源码

def _load(self, rules):
        """Load IPTables rules

        Args:
            list: IPTables rules

        Returns:
            (int, Optional[str]): A tuple where the first object is the return code and the second is an optional error string associated to the return code.

        """

        tmpfile = tempfile.NamedTemporaryFile(
            dir=self._sessions_dir, delete=False)
        tmpfile.write("\n".join(rules))
        tmpfile.close()
        os.chmod(tmpfile.name, 0755)
        proc = subprocess.Popen(tmpfile.name, shell=True,
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out, err = proc.communicate()
        os.remove(tmpfile.name)

        return proc.returncode, err

def clone(cls, remote_url, path):
        """Clone the remote and return a GitVcs object pointed at the new repo.

        :param str remote_url: the URL to clone from
        :param str path: path to clone to

        :rtype: GitVcs
        :returns: a GitVcs object for the new cloned repo

        :raises tigerhost.vcs.base.CommandError:
        """
        args = ['git', 'clone', '--recursive', remote_url, path]
        proc = Popen(args, stdout=PIPE, stderr=PIPE)
        (stdout, stderr) = proc.communicate()
        if proc.returncode != 0:
            raise CommandError(args[0], proc.returncode, stdout, stderr)
        return cls(path=path)

def handle_eval(self, record):
        # This gives a file name / directory name that no other thread can use
        my_unique_filename = my_gen.next_filename()
        my_unique_filename = str(my_unique_filename) + ".txt"

        # Print to the input file
        f = open(my_unique_filename, 'w')
        f.write(array2str(record.params[0]))
        f.close()

        # Run the objective function and pass the filename of the input file
        self.process = Popen(['./sphere_ext_files', my_unique_filename], stdout=PIPE)
        out = self.process.communicate()[0]

        # Parse the output
        try:
            val = float(out)  # This raises ValueError if out is not a float
            self.finish_success(record, val)
            os.remove(my_unique_filename)  # Remove input file
        except ValueError:
            logging.warning("Function evaluation crashed/failed")
            self.finish_failure(record)
            os.remove(my_unique_filename)  # Remove input file

def _get_ntfs_drives_win(self):
        '''Return list of ntfs drives using fsutil fsinfo's volumeinfo. 
        Result after slpit('\r\n'):

        ['Volume Name : Le Shwa',
         'Volume Serial Number : 0xd4d56c89',
         'Max Component Length : 255',
         'File System Name : NTFS',     --> index #3 --> split(':') --> index #1 
         'Is ReadWrite',....       ]'''

        ntfs_drives=[]
        win_drive_list=(chr(a)+ u':' for a in range(ord('A'), ord('Z')) if self.exists(chr(a)+':'))
        for drive in self.drives:
            volume_info=Popen(('fsutil', 'fsinfo', 'volumeInfo', drive), stdout=PIPE).communicate()[0]
            file_system=volume_info.split('\r\n')[3].split(' : ')[1]
            if file_system=='NTFS':
                ntfs_drives.append(drive)

        return ntfs_drives

def open_process(command, cwd=None, shell=True, _popen_lock=threading.Lock()):
    kwargs = {
        "shell": shell,
        "stdout": subprocess.PIPE,
        "stderr": subprocess.STDOUT,
        "stdin": subprocess.PIPE,
        "bufsize": 1,  # Line buffered
        "universal_newlines": True,
    }

    if cwd is not None:
        kwargs["cwd"] = cwd

    # Prevent signal propagation from parent process
    try:
        # Windows
        kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
    except AttributeError:
        # Unix
        kwargs["preexec_fn"] = os.setpgrp

    with _popen_lock:  # Work around Python 2 Popen race condition
        return subprocess.Popen(command, **kwargs)

def _local_ping(self, count):
        cmd = self._prepare_cmd(count)
        p = subprocess.Popen(cmd,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        # check that process is alive
        if p.poll() is not None:
            stdout, stderr = p.communicate()
            raise Exception(
                'Command {!r} unexpectedly exit with message {}'.format(
                    cmd, stdout, stderr))
        result = PingResult()
        yield result
        if count:
            p.wait()
        # Check if process still alive
        elif p.poll() is None:
            p.send_signal(signal.SIGINT)
        stdout, stderr = p.communicate()
        result.stdout = stdout

def do_run(cmd):
        try:
            cwd = os.getcwd() if inherit_cwd else None
            if not async:
                if stdin:
                    return subprocess.check_output(cmd, shell=True,
                        stderr=stderr, stdin=subprocess.PIPE, env=env_dict, cwd=cwd)
                output = subprocess.check_output(cmd, shell=True, stderr=stderr, env=env_dict, cwd=cwd)
                return output.decode(DEFAULT_ENCODING)
            # subprocess.Popen is not thread-safe, hence use a mutex here..
            try:
                mutex_popen.acquire()
                stdin_arg = subprocess.PIPE if stdin else None
                stdout_arg = open(outfile, 'wb') if isinstance(outfile, six.string_types) else outfile
                process = subprocess.Popen(cmd, shell=True, stdin=stdin_arg, bufsize=-1,
                    stderr=stderr, stdout=stdout_arg, env=env_dict, cwd=cwd)
                return process
            finally:
                mutex_popen.release()
        except subprocess.CalledProcessError as e:
            if print_error:
                print("ERROR: '%s': %s" % (cmd, e.output))
            raise e

def rm(filename, sudo=False):
    """
    Remove a file on the disk, not us os.rm because we want to add timeout to
    the command. It's possible that the os call got hang when the disk has
    some problems
    """
    cmd_args = []
    if sudo:
        cmd_args += ['sudo']
    cmd_args += ['/bin/rm', filename]
    log.debug("Executing cmd: {}".format(str(cmd_args)))
    proc = subprocess.Popen(cmd_args, stdin=subprocess.PIPE,
                            stdout=subprocess.PIPE)
    try:
        (stdout, stderr) = proc.communicate(timeout=10)
    except subprocess.TimeoutExpired:
        proc.kill()
        raise OSCError('SHELL_TIMEOUT', {'cmd': ' '.join(cmd_args)})

def do_run(cmd):
        try:
            cwd = os.getcwd() if inherit_cwd else None
            if not async:
                if stdin:
                    return subprocess.check_output(cmd, shell=True,
                        stderr=stderr, stdin=subprocess.PIPE, env=env_dict, cwd=cwd)
                output = subprocess.check_output(cmd, shell=True, stderr=stderr, env=env_dict, cwd=cwd)
                return output.decode(DEFAULT_ENCODING)
            # subprocess.Popen is not thread-safe, hence use a mutex here..
            try:
                mutex_popen.acquire()
                stdin_arg = subprocess.PIPE if stdin else None
                stdout_arg = open(outfile, 'wb') if isinstance(outfile, six.string_types) else outfile
                process = subprocess.Popen(cmd, shell=True, stdin=stdin_arg, bufsize=-1,
                    stderr=stderr, stdout=stdout_arg, env=env_dict, cwd=cwd)
                return process
            finally:
                mutex_popen.release()
        except subprocess.CalledProcessError as e:
            if print_error:
                print("ERROR: '%s': %s" % (cmd, e.output))
            raise e

def nginx_server_process(prefix='', echo=False):
    args = [
        os.path.join(prefix, 'nginx'),
        '-c', resource_filename('snovault', 'nginx-dev.conf'),
        '-g', 'daemon off;'
    ]
    process = subprocess.Popen(
        args,
        close_fds=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
    )

    if not echo:
        process.stdout.close()

    if echo:
        print('Started: http://localhost:8000')

    return process

def __init__(self, args):
        self.args = args
        self.process = subprocess32.Popen(
            args,
            stdin=subprocess32.PIPE,
            stdout=subprocess32.PIPE,
            stderr=subprocess32.PIPE
        )
        self.err = None

def start_jaer(jaer_path, jaer_exec="jAERViewer1.5_linux.sh"):
    """Start jAER from Python.

    This script is written for Linux usage,
    An error will be raised if it's Windows OS.
    Instead, windows user needs to manually setup jAER.

    Parameters
    ----------
    jaer_path : string
        absolute save path of jAER.
        e.g. /Users/dgyHome/Documents/workspace/jaer/trunk
    jaer_exec : string
        The executable of jAER. Version 1.5 is assumed.

    Returns
    -------
    An opened jAER viewer.
    """
    # Check OS type
    if os.name != "posix":
        raise ValueError("The Operating System is not a POSIX platform")

    commands = "cd "+jaer_path+"; bash "+jaer_exec
    process = subprocess.Popen(commands, stdout=subprocess.PIPE, shell=True)

    return process

def test_rejects_client_that_does_not_specifiy_a_supported_protocol(server):

    node_script = '''
        module.paths.push('{0}')
        WebSocket = require('ws')
        const client = new WebSocket('ws://localhost:{1}/socket')
        client.on('close', (code) => {{
            console.log(JSON.stringify(code))
          }}
        );
    '''.format(
        os.path.join(os.path.dirname(__file__), 'node_modules'), TEST_PORT)

    p = subprocess.Popen(
        ['node', '-e', node_script],
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT)
    q = queue.Queue()
    t = threading.Thread(target=enqueue_output, args=(p.stdout, q))
    t.daemon = True
    t.start()
    time.sleep(.2)
    ret_values = []
    while True:
        try:
            _line = q.get_nowait()
            if isinstance(_line, bytes):
                line = _line.decode()
            line = json.loads(line)
            ret_values.append(line)
        except ValueError:
            pass
        except queue.Empty:
            break
    assert ret_values[0] == 1002 or 1006

def _test(self, rules_file, queue):
        # Import the firewall rules in a detached network namespace
        unshare(CLONE_NEWNET)
        proc = subprocess.Popen(rules_file,
                                shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out, err = proc.communicate()

        if proc.returncode:
            queue.put((False, err.strip()))
        else:
            queue.put((True, None))

def run(self, name):
        """Run a check

        Raises:
            CheckError

        """

        if not self.exists(name):
            raise CheckNotFound("Check not found")

        if self._checks[name]['type'] == 'exec':
            proc = subprocess.Popen(
                self._checks[name]['command'], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            try:
                out, err = proc.communicate(timeout=self._checks[name]['timeout'])
            except subprocess.TimeoutExpired as e:
                raise CheckError("Timed out")
            except Exception as e:
                raise CheckError(e.message)

            if proc.returncode:
                raise CheckError("Command failed with exitstatus {} [{}]".format(
                    proc.returncode, err.strip()))
        elif self._checks[name]['type'] == 'tcp':
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.settimeout(self._checks[name]['timeout'])
            try:
                result = sock.connect_ex(
                    (self._checks[name]['host'], self._checks[name]['port']))
                sock.close()
                if result != 0:
                    raise Exception("Connection failed (Errno: {})".format(result))
            except socket.timeout as e:
                raise CheckError("Timed out")
            except Exception as e:
                raise CheckError(e.message)
            finally:
                sock.close()

def _call_phantom(self, token, arns, output_file):
        """
        shells out to phantomjs.
        - Writes ARNs to a file that phantomjs will read as an input.
        - Phantomjs exchanges the token for session cookies.
        - Phantomjs then navigates to the IAM page and executes JavaScript
        to call GenerateServiceLastAccessedDetails for each ARN.
        - Every 10 seconds, Phantomjs calls GetServiceLastAccessedDetails
        - Phantom saves output to a file that is used by `persist()`

        :return: Exit code from phantomjs subprocess32
        """

        path = os.path.dirname(__file__)
        console_js = os.path.join(path, 'awsconsole.js')

        with tempfile.NamedTemporaryFile() as f:
            json.dump(arns, f)
            f.seek(0)
            try:
                p = subprocess32.Popen([
                    self.current_app.config.get('PHANTOMJS'),
                    console_js,
                    token,
                    f.name,
                    output_file],
                    stdout=subprocess32.PIPE, stderr=subprocess32.STDOUT)
                output, errs = p.communicate(timeout=1200)  # 20 mins
                self.current_app.logger.debug('Phantom Output: \n{}'.format(output))
                self.current_app.logger.debug('Phantom Errors: \n{}'.format(errs))
            except subprocess32.TimeoutExpired:
                self.current_app.logger.error('PhantomJS timed out')
                return 1  # return code 1 for timeout
            except CalledProcessError:
                self.current_app.logger.error('PhantomJS exited: {}'
                                              ''.format(p.returncode))
                return p.returncode
            else:
                self.current_app.logger.info('PhantomJS exited: 0')
                return 0

def raw_cmd(self, *args):
        '''adb command. return the subprocess.Popen object.'''
        cmd_line = [self.adb()] + self.adb_host_port_options + list(args)
        if os.name != "nt":
            cmd_line = [" ".join(cmd_line)]
        return subprocess.Popen(cmd_line, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

def background_test(self):
        self.running = True
        proc = subprocess.Popen('echo hello', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        while True:
            line = proc.stdout.readline()
            if line == '':
                break
            print line
            for client in ProgressHandler.clients:
                client.write_message(line)
            self.output = self.output + line
        self.running = False

def start_app(self, bundle_id):
        '''
        Start app by bundle_id
        Args:
            - bundle_id(string): ex com.netease.my
        Returns:
            idevicedebug subprocess instance
        '''
        idevicedebug = must_look_exec('idevicedebug')

        # run in background
        kwargs = {'stdout': subprocess.PIPE, 'stderr': subprocess.PIPE}
        if sys.platform != 'darwin':
            kwargs['close_fds'] = True
        return subprocess.Popen([idevicedebug, "--udid", self.udid, 'run', bundle_id], **kwargs)

def _init_instruments(self, bundle_id):
        self._bootstrap = os.path.join(__dir__, 'bootstrap.sh')
        self._bundle_id = bundle_id
        self._env.update({'UDID': self.udid, 'BUNDLE_ID': self._bundle_id})
        # 1. remove pipe
        # subprocess.check_output([self._bootstrap, 'reset'], env=self._env)
        # 2. start instruments
        self._proc = subprocess.Popen([self._bootstrap, 'instruments'], env=self._env, stdout=subprocess.PIPE)
        self.sleep(5.0)
        self._wait_instruments()

def __init__(self, device_url, bundle_id=None):
        DeviceMixin.__init__(self)
        self.__device_url = device_url
        self.__scale = None

        self._wda = wda.Client(device_url)
        self._session = None
        self._bundle_id = None

        if bundle_id:
            self.start_app(bundle_id)

        # ioskit.Device.__init__(self, udid)

        # # xcodebuild -project  -scheme WebDriverAgentRunner -destination "id=1002c0174e481a651d71e3d9a89bd6f90d253446" test
        # # Test Case '-[UITestingUITests testRunner]' started.
        # xproj_dir = os.getenv('WEBDRIVERAGENT_DIR')
        # if not xproj_dir:
        #     raise RuntimeError("env-var WEBDRIVERAGENT_DIR need to be set")

        # proc = self._xcproc = subprocess.Popen(['/usr/bin/xcodebuild',
        #     '-project', 'WebDriverAgent.xcodeproj',
        #     '-scheme', 'WebDriverAgentRunner',
        #     '-destination', 'id='+self.udid, 'test'],
        #     stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=xproj_dir, bufsize=1, universal_newlines=True)
        # for line in iter(proc.stdout.readline, b""):
        #     print 'STDOUT:', line.strip()
        #     if 'TEST FAILED' in line:
        #         raise RuntimeError("webdriver start test failed, maybe need to unlock the keychain, try\n" + 
        #             '$ security unlock-keychain ~/Library/Keychains/login.keychain')
        #     elif "Successfully wrote Manifest cache" in line:
        #         print 'GOOD ^_^, wait 5s'
        #         time.sleep(5.0)
        #         break

def raw_cmd(self, *args):
        '''adb command. return the subprocess.Popen object.'''
        cmd_line = [self.adb()] + self.adb_host_port_options + list(args)
        if os.name != "nt":
            cmd_line = [" ".join(cmd_line)]
        return subprocess.Popen(cmd_line, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

def background_test(self):
        self.running = True
        proc = subprocess.Popen('echo hello', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        while True:
            line = proc.stdout.readline()
            if line == '':
                break
            print line
            for client in ProgressHandler.clients:
                client.write_message(line)
            self.output = self.output + line
        self.running = False

def start_app(self, bundle_id):
        '''
        Start app by bundle_id
        Args:
            - bundle_id(string): ex com.netease.my
        Returns:
            idevicedebug subprocess instance
        '''
        idevicedebug = must_look_exec('idevicedebug')

        # run in background
        kwargs = {'stdout': subprocess.PIPE, 'stderr': subprocess.PIPE}
        if sys.platform != 'darwin':
            kwargs['close_fds'] = True
        return subprocess.Popen([idevicedebug, "--udid", self.udid, 'run', bundle_id], **kwargs)

def _init_instruments(self, bundle_id):
        self._bootstrap = os.path.join(__dir__, 'bootstrap.sh')
        self._bundle_id = bundle_id
        self._env.update({'UDID': self.udid, 'BUNDLE_ID': self._bundle_id})
        # 1. remove pipe
        # subprocess.check_output([self._bootstrap, 'reset'], env=self._env)
        # 2. start instruments
        self._proc = subprocess.Popen([self._bootstrap, 'instruments'], env=self._env, stdout=subprocess.PIPE)
        self.sleep(5.0)
        self._wait_instruments()

def __init__(self, device_url, bundle_id=None):
        DeviceMixin.__init__(self)
        self.__device_url = device_url
        self.__display = None
        self.__scale = None

        self._wda = wda.Client(device_url)
        self._session = None
        self._bundle_id = None

        if bundle_id:
            self.start_app(bundle_id)
        # ioskit.Device.__init__(self, udid)

        # # xcodebuild -project  -scheme WebDriverAgentRunner -destination "id=1002c0174e481a651d71e3d9a89bd6f90d253446" test
        # # Test Case '-[UITestingUITests testRunner]' started.
        # xproj_dir = os.getenv('WEBDRIVERAGENT_DIR')
        # if not xproj_dir:
        #     raise RuntimeError("env-var WEBDRIVERAGENT_DIR need to be set")

        # proc = self._xcproc = subprocess.Popen(['/usr/bin/xcodebuild',
        #     '-project', 'WebDriverAgent.xcodeproj',
        #     '-scheme', 'WebDriverAgentRunner',
        #     '-destination', 'id='+self.udid, 'test'],
        #     stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=xproj_dir, bufsize=1, universal_newlines=True)
        # for line in iter(proc.stdout.readline, b""):
        #     print 'STDOUT:', line.strip()
        #     if 'TEST FAILED' in line:
        #         raise RuntimeError("webdriver start test failed, maybe need to unlock the keychain, try\n" + 
        #             '$ security unlock-keychain ~/Library/Keychains/login.keychain')
        #     elif "Successfully wrote Manifest cache" in line:
        #         print 'GOOD ^_^, wait 5s'
        #         time.sleep(5.0)
        #         break

def compile_problem(solution_spec):
    """Compiles a problem submission and generates a problem spec.

    Args:
        solution_spec: Specification string of a solution corresponding to the
            submitted problem.

    Returns:
        (problem_spec, problem_size)
        problem_spec: Specification string of the problem.
        problem_size: Problem size.

    Raises:
        VerificationError: If the solution specification is invalid.
        subprocess.TimeoutExpired: On judge timeout.
        AssertionError: On scrape error.
    """
    with make_temporary_file_with_content(solution_spec) as solution_file:
        proc = subprocess.Popen(
            ['./akatsuki', '--logtostderr', '--compile', solution_file.name],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
        try:
            stdout_output, stderr_output = proc.communicate(
                timeout=_JUDGE_TIMEOUT_SECONDS)
        except subprocess.TimeoutExpired:
            proc.kill()
            proc.wait()
            raise  # report ISE
    if proc.returncode:
        m = _VERIFICATION_ERROR_RE.search(stdout_output)
        assert m, stdout_output  # report ISE
        raise VerificationError(m.group(1))
    problem_spec = stdout_output
    problem_size = sum(len(s) for s in problem_spec.split())
    return (problem_spec, problem_size)

def evaluate_solution(problem_spec, solution_spec):
    """Evaluates a solution submission.

    Args:
        problem_spec: Specification string of a problem.
        solution_spec: Specification string of a solution.

    Returns:
        (resemblance_int, raw_evaluator_output)

    Raises:
        VerificationError: If any of the specifications are invalid.
        subprocess.TimeoutExpired: On judge timeout.
        AssertionError: On scrape error.
    """
    with make_temporary_file_with_content(problem_spec) as problem_file, \
         make_temporary_file_with_content(solution_spec) as solution_file:
        proc = subprocess.Popen(
            ['./akatsuki', '--logtostderr', '--evaluate',
             problem_file.name, solution_file.name],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
        try:
            stdout_output, stderr_output = proc.communicate(
                timeout=_JUDGE_TIMEOUT_SECONDS)
        except subprocess.TimeoutExpired:
            proc.kill()
            proc.wait()
            raise  # report ISE
    if proc.returncode:
        m = _VERIFICATION_ERROR_RE.search(stdout_output)
        assert m, stdout_output  # report ISE
        raise VerificationError(m.group(1))
    m = re.search(r'integer_resemblance: (\d+)', stdout_output)
    assert m, stdout_output  # report ISE
    resemblance_int = int(m.group(1))
    return resemblance_int, stdout_output.decode('utf-8')

def start_dot11decrypt(self, interface, decryption_key): #starts and returns dot11decrypt subprocess and interface.
        print "Starting new dot11decrypt subprocess on " + interface + " with key " + decryption_key
        proc = subprocess32.Popen(['sudo', 'd11decrypt/build/dot11decrypt', interface, decryption_key], stdout=subprocess32.PIPE)
        read = proc.stdout.readline()
        if read[0:14] == "Using device: ":
            print "Currently decrypting packets and releasing to " + read[14:].rstrip()
            print "Process number is " + str(proc.pid)
            return proc, read[14:].rstrip()
        else:
            print read
            raise Exception

def Run(self, command, timeout=None, env=None):
    """Run a command against this SDK installation.

    Args:
      command: string, list or tuple, The command to run (e.g. ['gsutil', 'cp',
        ...])
      timeout: number, Seconds to wait before timing out the command.
      env: dict or None, Extra environmental variables use with this command.

    Returns:
      (stdout, stderr, returncode) returned from the command.

    Raises:
      error.SDKError: If the command cannot be run.
    """
    # Add the passed in variables to the precomputed environment (without
    # altering either dictionary).
    if env:
      env = dict(self._env, **env)
    else:
      env = self._env

    p = subprocess.Popen(
        _PrepareCommand(command), stdout=subprocess.PIPE,
        stderr=subprocess.PIPE, cwd=os.path.dirname(self._sdk_dir), env=env)
    if TIMEOUT_ENABLED:
      out, err = p.communicate(timeout=timeout)
    else:
      if timeout:
        sys.stderr.write(
            'Warning: timeout specified, but subprocess32 is not available.')
      out, err = p.communicate()

    # TODO(magimaster): Change this to raise an error if returncode isn't 0
    return out, err, p.returncode

def run_playbook(playbook, inventory, *args, **kwargs):
    env = ansible_env(os.environ.copy())
    cmd = ['ansible-playbook', '-i', inventory, playbook] + list(args)

    if verbosity():
        cmd += ['-' + ('v' * verbosity())]

    show_timestamp = False
    if 'timestamp' in kwargs:
        show_timestamp = kwargs['timestamp']
        del kwargs['timestamp']

    output = print
    if show_timestamp:
        output = timestamp

    logger.info('running %s', ' '.join(cmd))
    logger.debug('env: %r', env)

    process = Popen(cmd, env=env, stdout=PIPE,
                    bufsize=1, **kwargs)
    for line in iter(process.stdout.readline, b''):
        output(line[:-1])
    # empty output buffers
    process.poll()
    return process.returncode

def checkOthers(self, test_step):
        if os.access(test_step.path, X_OK):
            return

        logger.debug("add +x for %s", test_step.path)
        command = "chmod +x " + test_step.path
        proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        outs, errs = proc.communicate()
        rc = proc.returncode
        if rc:
            raise Exception("command %s failed: rc is %s, output is %s, errs is %s ", command, rc, outs, errs)

def check_call_realtime(args):
    """Run command with arguments and yield the output as they come.

    Stderr is piped into stdout.

    :raises subprocess.CalledProcessError: if exit code is non-zero
    """
    p = subprocess.Popen(args, stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT)
    while p.poll() is None:
        yield p.stdout.read()
    yield p.stdout.read()
    if p.returncode != 0:
        raise subprocess.CalledProcessError(p.returncode, args)

def handle_eval(self, record):
        self.process = Popen(['./sphere_ext', array2str(record.params[0])],
                             stdout=PIPE)
        out = self.process.communicate()[0]
        try:
            val = float(out)  # This raises ValueError if out is not a float
            self.finish_success(record, val)
        except ValueError:
            logging.warning("Function evaluation crashed/failed")
            self.finish_failure(record)

def handle_eval(self, record):
        self.process = Popen(['./sumfun_ext', array2str(record.params[0])],
                             stdout=PIPE)

        val = np.nan
        # Continuously check for new outputs from the subprocess
        while True:
            output = self.process.stdout.readline()
            if output == '' and self.process.poll() is not None:  # No new output
                break
            if output:  # New intermediate output
                try:
                    val = float(output.strip())  # Try to parse output
                    if val > 350:  # Terminate if too large
                        self.process.terminate()
                        self.finish_success(record, 350)
                        return
                except ValueError:  # If the output is nonsense we terminate
                    logging.warning("Incorrect output")
                    self.process.terminate()
                    self.finish_failure(record)
                    return

        rc = self.process.poll()  # Check the return code
        if rc < 0 or np.isnan(val):
            logging.warning("Incorrect output or crashed evaluation")
            self.finish_failure(record)
        else:
            self.finish_success(record, val)

def spawn(self):
        """Spawn the fake executable using subprocess.Popen."""
        self._process = subprocess.Popen(
            [self.path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        self.addCleanup(self._process_kill)

def raw_cmd(self, *args, **kwargs):
        '''adb command. return the subprocess.Popen object.'''
        cmds = [self.adb_path()] + self._host_port_args + list(args)
        kwargs['stdout'] = kwargs.get('stdout', subprocess.PIPE)
        kwargs['stderr'] = kwargs.get('stderr', subprocess.PIPE)
        # if os.name != "nt":
        #     cmd_line = [" ".join(cmd_line)]
        cmds = [strutils.decode(v) for v in cmds]
        return subprocess.Popen(cmds, **kwargs)

def create_journals(self, max_size, delta):
        '''create usn journals to track changes for chosen drives if such don't
        already exist'''

        for drive in self.ntfs_drives:
            #check if a journal exists, else:

            Popen(('fsutil', 'usn', 'createjournal', max_size, delta, drive), stdout=PIPE).communicate()[0]

        #experiment

def get_next_max_usn(self, drive):
        '''On windows/ntfs this is 'next usn' - the usn index to latest change made. 
        Also returns max_usn since enumdata requires an upper boundary. Not needed
        when using readjournal.
        fsutil usn queryjournal result: 

        Usn Journal ID   : 0x01d2a26e17dbc5e8
        First Usn        : 0x0000000000000000
        Next Usn         : 0x0000000000acddf0     <--- index #2
        Lowest Valid Usn : 0x0000000000000000
        Max Usn          : 0x7fffffffffff0000     <--- index #4
        .
        .                                      '''

        if 'win' in sys.platform and drive == '/':
            ''' using '/' on windows works for scandir but not for fsutil'''
            drive='c:'
        else:
            # Removing trailing slashes
            drive=drive.split(':')[0]+':'  

        journal_specs=Popen(('fsutil', 'usn', 'queryjournal', drive), stdout=PIPE).communicate()[0].split('\r\n')
        next_usn=journal_specs[2].split(': ')[1]
        max_usn=journal_specs[4].split(': ')[1]

        return next_usn, max_usn         #int(next_usn, 16), int(max_usn, 16)

def run_cmd(self, params):

        def convert_line(line):
            line = to_str(line)
            return line.strip() + '\r\n'

        try:
            self.process = run(self.cmd, async=True, stdin=self.stdin, outfile=self.outfile,
                env_vars=self.env_vars, inherit_cwd=self.inherit_cwd)
            if self.outfile:
                if self.outfile == subprocess.PIPE:
                    # get stdout/stderr from child process and write to parent output
                    for line in iter(self.process.stdout.readline, ''):
                        if not (line and line.strip()) and self.is_killed():
                            break
                        line = convert_line(line)
                        sys.stdout.write(line)
                        sys.stdout.flush()
                    for line in iter(self.process.stderr.readline, ''):
                        if not (line and line.strip()) and self.is_killed():
                            break
                        line = convert_line(line)
                        sys.stderr.write(line)
                        sys.stderr.flush()
                self.process.wait()
            else:
                self.process.communicate()
        except Exception as e:
            if self.process and not self.quiet:
                LOGGER.warning('Shell command error "%s": %s' % (e, self.cmd))
        if self.process and not self.quiet and self.process.returncode != 0:
            LOGGER.warning('Shell command exit code "%s": %s' % (self.process.returncode, self.cmd))

def setUpClass(cls):
        # In Python 3 Click makes a call to locale to determine how the
        # terminal wants to handle unicode. Because we mock Popen to avoid
        # calling the real verifier, we need to get the actual result of
        # locale to provide it to Click during the test run.
        if os.name == 'nt':
            cls.locale = ''  # pragma: no cover
        else:
            cls.locale = Popen(
                ['locale', '-a'], stdout=PIPE, stderr=PIPE).communicate()[0]

def run_simple (self, cmds, x, family) :
        '''
            Run a single command in a subprocess. Line buffer output.
            cmds   -> A list of commands to be run for this family
            x      -> The sequence of the command to run
            family -> The family that goes with these commands
            pee    -> The process
            fifofh -> File handle to fifo
        '''
        pee = None
        try :
            cmd = cmds[x]
        except IndexError :
            return pee, None

        #fifo = os.path.join ("/tmp", "fifo{0}".format (family))
        #if not os.path.exists (fifo) :
            #os.mkfifo (fifo)

        #fifofh = open (fifo, mode='rw+')

        pee = subprocess.Popen (cmd,
                                shell=True,
                                bufsize=1,
                                cwd=os.path.join (self.home, family),
                                stdout=subprocess.PIPE,
                                universal_newlines=True,
                                close_fds=ON_POSIX)
        fifofh = pee.stdout

        return pee, fifofh

def run_cmds (self, cmds, x = 0, ems = None) :
        '''
            Run conversion commands in a subprocess
            cmds -> A dictionary of families that point to a list of commands.
                    cmds['B']['125a2ph5 -n master.ph5 ...', '1302ph5 -n master.ph5 ...']
            x -> The sequence of the current command executing in the list in cmds.
            ems -> The list of families ['A', 'B', 'C' etc]
        '''
        pees = {}
        if ems == None :
            ems = self.nmini
        else :
            ems = [ems]
        #for inst in ('texan', 'rt-130', 'nodal') :
        for m in ems :
            if len (cmds[m]) > x :
                insts = cmds[m][x]
                pees[m] = subprocess.Popen (insts, 
                                            shell=True, 
                                            bufsize=-1,
                                            cwd=os.path.join (self.home, m),
                                            stdout=subprocess.PIPE,
                                            stderr=subprocess.PIPE)

            else :
                pees[m] = None
        if len (ems) > 1 :
            return pees, x
        else :
            return pees[m], x
    #
    ###   Should this be implemented as a closure?
    #

def run_cmd(self, params):

        def convert_line(line):
            line = to_str(line)
            return line.strip() + '\r\n'

        try:
            self.process = run(self.cmd, async=True, stdin=self.stdin, outfile=self.outfile,
                env_vars=self.env_vars, inherit_cwd=self.inherit_cwd)
            if self.outfile:
                if self.outfile == subprocess.PIPE:
                    # get stdout/stderr from child process and write to parent output
                    for line in iter(self.process.stdout.readline, ''):
                        if not (line and line.strip()) and self.is_killed():
                            break
                        line = convert_line(line)
                        sys.stdout.write(line)
                        sys.stdout.flush()
                    for line in iter(self.process.stderr.readline, ''):
                        if not (line and line.strip()) and self.is_killed():
                            break
                        line = convert_line(line)
                        sys.stderr.write(line)
                        sys.stderr.flush()
                self.process.wait()
            else:
                self.process.communicate()
        except Exception as e:
            if self.process and not self.quiet:
                LOGGER.warning('Shell command error "%s": %s' % (e, self.cmd))
        if self.process and not self.quiet and self.process.returncode != 0:
            LOGGER.warning('Shell command exit code "%s": %s' % (self.process.returncode, self.cmd))

def spawn(self, engine):
        self.engine = engine
        self.process = subprocess.Popen(self.command, stdout=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=1, universal_newlines=True)
        self._receiving_thread.start()

def test_send_subscription_fail_message_to_client_with_invalid_query(server):

    node_script = '''
        module.paths.push('{0}')
        WebSocket = require('ws')
        const SubscriptionClient =
        require('subscriptions-transport-ws').SubscriptionClient
        const client = new SubscriptionClient('ws://localhost:{1}/socket')
        setTimeout(function () {{
            client.subscribe({{
                query: `subscription useInfo($id: String) {{
                  user(id: $id) {{
                    id
                    birthday
                  }}
                }}`,
                operationName: 'useInfo',
                variables: {{
                  id: 3,
                }},

                }}, function (error, result) {{
                }}
            );
        }}, 100);
            client.client.onmessage = (message) => {{
                let msg = JSON.parse(message.data)
                console.log(JSON.stringify({{[msg.type]: msg}}))
            }};
    '''.format(
        os.path.join(os.path.dirname(__file__), 'node_modules'), TEST_PORT)

    p = subprocess.Popen(
        ['node', '-e', node_script],
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT)
    time.sleep(.2)
    q = queue.Queue()
    t = threading.Thread(target=enqueue_output, args=(p.stdout, q))
    t.daemon = True
    t.start()
    time.sleep(.2)
    ret_values = {}
    while True:
        try:
            _line = q.get_nowait()
            if isinstance(_line, bytes):
                line = _line.decode()
            line = json.loads(line)
            ret_values[list(line.keys())[0]] = line[list(line.keys())[0]]
        except ValueError:
            pass
        except queue.Empty:
            break
    assert ret_values['type'] == SUBSCRIPTION_FAIL
    assert len(ret_values['payload']['errors']) > 0


# TODO: troubleshoot this a bit...passes, but receives extra messages which I'm
# filtering out w/ the "AttributeError" exception clause

def test_correctly_sets_the_context_in_on_subscribe(server):
    node_script = '''
        module.paths.push('{0}')
        WebSocket = require('ws')
        const SubscriptionClient =
        require('subscriptions-transport-ws').SubscriptionClient
        const CTX = 'testContext';
        const client = new SubscriptionClient('ws://localhost:{1}/socket')
        client.subscribe({{
            query: `subscription context {{
                context
            }}`,
            variables: {{}},
            context: CTX,
            }}, (error, result) => {{
                client.unsubscribeAll();
                if (error) {{
                  console.log(JSON.stringify(error));
                }}
                if (result) {{
                  console.log(JSON.stringify({{
                    client: {{
                      result: result,
                    }}
                  }}));
                }} else {{
                  // pass
                }}
            }}
        );
    '''.format(
        os.path.join(os.path.dirname(__file__), 'node_modules'), TEST_PORT)

    p = subprocess.Popen(
        ['node', '-e', node_script],
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT)
    time.sleep(.2)
    requests.post(
        'http://localhost:{0}/publish'.format(TEST_PORT), json=['context', {}])
    q = queue.Queue()
    t = threading.Thread(target=enqueue_output, args=(p.stdout, q))
    t.daemon = True
    t.start()
    time.sleep(.2)
    ret_values = {}
    while True:
        try:
            _line = q.get_nowait()
            if isinstance(_line, bytes):
                line = _line.decode()
            line = json.loads(line)
            ret_values[list(line.keys())[0]] = line[list(line.keys())[0]]
        except ValueError:
            pass
        except queue.Empty:
            break
    client = ret_values['client']
    assert client['result']['context']
    assert client['result']['context'] == 'testContext'

def test_rejects_unparsable_message(server):

    node_script = '''
        module.paths.push('{0}');
        WebSocket = require('ws');
        const GRAPHQL_SUBSCRIPTIONS = 'graphql-subscriptions';
        const client = new WebSocket('ws://localhost:{1}/socket',
        GRAPHQL_SUBSCRIPTIONS);
        client.onmessage = (message) => {{
            let msg = JSON.parse(message.data)
            console.log(JSON.stringify({{[msg.type]: msg}}))
            client.close();
        }};
        client.onopen = () => {{
            client.send('HI');
        }}
    '''.format(
        os.path.join(os.path.dirname(__file__), 'node_modules'), TEST_PORT)

    p = subprocess.Popen(
        ['node', '-e', node_script],
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT)
    q = queue.Queue()
    t = threading.Thread(target=enqueue_output, args=(p.stdout, q))
    t.daemon = True
    t.start()
    time.sleep(.2)
    ret_values = {}
    while True:
        try:
            _line = q.get_nowait()
            if isinstance(_line, bytes):
                line = _line.decode()
            line = json.loads(line)
            ret_values[list(line.keys())[0]] = line[list(line.keys())[0]]
        except ValueError:
            pass
        except queue.Empty:
            break
    assert ret_values['subscription_fail']
    assert len(ret_values['subscription_fail']['payload']['errors']) > 0

def test_rejects_nonsense_message(server):

    node_script = '''
        module.paths.push('{0}');
        WebSocket = require('ws');
        const GRAPHQL_SUBSCRIPTIONS = 'graphql-subscriptions';
        const client = new WebSocket('ws://localhost:{1}/socket',
        GRAPHQL_SUBSCRIPTIONS);
        client.onmessage = (message) => {{
            let msg = JSON.parse(message.data)
            console.log(JSON.stringify({{[msg.type]: msg}}))
            client.close();
        }};
        client.onopen = () => {{
            client.send(JSON.stringify({{}}));
        }}
    '''.format(
        os.path.join(os.path.dirname(__file__), 'node_modules'), TEST_PORT)

    p = subprocess.Popen(
        ['node', '-e', node_script],
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT)
    q = queue.Queue()
    t = threading.Thread(target=enqueue_output, args=(p.stdout, q))
    t.daemon = True
    t.start()
    time.sleep(.2)
    ret_values = {}
    while True:
        try:
            _line = q.get_nowait()
            if isinstance(_line, bytes):
                line = _line.decode()
            line = json.loads(line)
            ret_values[list(line.keys())[0]] = line[list(line.keys())[0]]
        except ValueError:
            pass
        except queue.Empty:
            break
    assert ret_values['subscription_fail']
    assert len(ret_values['subscription_fail']['payload']['errors']) > 0