Python subprocess32 模块，TimeoutExpired() 实例源码

def adb_pushfile(adb, filepath, remote_path):
    filesize = os.path.getsize(filepath)
    pb = tqdm.tqdm(unit='B', unit_scale=True, total=filesize)
    p = adb.raw_cmd('push', filepath, remote_path)

    while True:
        try:
            p.wait(0.5)
        except subprocess.TimeoutExpired:
            pb.n = get_file_size(adb, remote_path)
            pb.refresh()
            # log.info("Progress %dM/%dM", get_file_size(remote_path) >>20, filesize >>20)
            pass
        except (KeyboardInterrupt, SystemExit):
            p.kill()
            raise
        except:
            raise
        else:
            # log.info("Success pushed into device")
            break
    pb.close()

def adb_pushfile(adb, filepath, remote_path):
    filesize = os.path.getsize(filepath)
    pb = tqdm.tqdm(unit='B', unit_scale=True, total=filesize)
    p = adb.raw_cmd('push', filepath, remote_path)

    while True:
        try:
            p.wait(0.5)
        except subprocess.TimeoutExpired:
            pb.n = get_file_size(adb, remote_path)
            pb.refresh()
            # log.info("Progress %dM/%dM", get_file_size(remote_path) >>20, filesize >>20)
            pass
        except (KeyboardInterrupt, SystemExit):
            p.kill()
            raise
        except:
            raise
        else:
            # log.info("Success pushed into device")
            break
    pb.close()

def kill_drone():
    global DRONE_RUNNING
    LOG.info('stopping drone')
    if DRONE_RUNNING is False:
        LOG.warning('drone is not running, nothing to do')
        return
    LOG.info('trying to stop drone gracefully')
    DRONE.terminate()
    try:
        DRONE.wait(timeout=10)
        LOG.info('drone exited gracefully')
    except subprocess.TimeoutExpired:
        LOG.info('could not terminate drone properly, kill it.')
        DRONE.kill()
        DRONE.wait(timeout=10)
        LOG.info('drone has been killed')
    DRONE_RUNNING = False

def rm(filename, sudo=False):
    """
    Remove a file on the disk, not us os.rm because we want to add timeout to
    the command. It's possible that the os call got hang when the disk has
    some problems
    """
    cmd_args = []
    if sudo:
        cmd_args += ['sudo']
    cmd_args += ['/bin/rm', filename]
    log.debug("Executing cmd: {}".format(str(cmd_args)))
    proc = subprocess.Popen(cmd_args, stdin=subprocess.PIPE,
                            stdout=subprocess.PIPE)
    try:
        (stdout, stderr) = proc.communicate(timeout=10)
    except subprocess.TimeoutExpired:
        proc.kill()
        raise OSCError('SHELL_TIMEOUT', {'cmd': ' '.join(cmd_args)})

def run(self, name):
        """Run a check

        Raises:
            CheckError

        """

        if not self.exists(name):
            raise CheckNotFound("Check not found")

        if self._checks[name]['type'] == 'exec':
            proc = subprocess.Popen(
                self._checks[name]['command'], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            try:
                out, err = proc.communicate(timeout=self._checks[name]['timeout'])
            except subprocess.TimeoutExpired as e:
                raise CheckError("Timed out")
            except Exception as e:
                raise CheckError(e.message)

            if proc.returncode:
                raise CheckError("Command failed with exitstatus {} [{}]".format(
                    proc.returncode, err.strip()))
        elif self._checks[name]['type'] == 'tcp':
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.settimeout(self._checks[name]['timeout'])
            try:
                result = sock.connect_ex(
                    (self._checks[name]['host'], self._checks[name]['port']))
                sock.close()
                if result != 0:
                    raise Exception("Connection failed (Errno: {})".format(result))
            except socket.timeout as e:
                raise CheckError("Timed out")
            except Exception as e:
                raise CheckError(e.message)
            finally:
                sock.close()

def _call_phantom(self, token, arns, output_file):
        """
        shells out to phantomjs.
        - Writes ARNs to a file that phantomjs will read as an input.
        - Phantomjs exchanges the token for session cookies.
        - Phantomjs then navigates to the IAM page and executes JavaScript
        to call GenerateServiceLastAccessedDetails for each ARN.
        - Every 10 seconds, Phantomjs calls GetServiceLastAccessedDetails
        - Phantom saves output to a file that is used by `persist()`

        :return: Exit code from phantomjs subprocess32
        """

        path = os.path.dirname(__file__)
        console_js = os.path.join(path, 'awsconsole.js')

        with tempfile.NamedTemporaryFile() as f:
            json.dump(arns, f)
            f.seek(0)
            try:
                p = subprocess32.Popen([
                    self.current_app.config.get('PHANTOMJS'),
                    console_js,
                    token,
                    f.name,
                    output_file],
                    stdout=subprocess32.PIPE, stderr=subprocess32.STDOUT)
                output, errs = p.communicate(timeout=1200)  # 20 mins
                self.current_app.logger.debug('Phantom Output: \n{}'.format(output))
                self.current_app.logger.debug('Phantom Errors: \n{}'.format(errs))
            except subprocess32.TimeoutExpired:
                self.current_app.logger.error('PhantomJS timed out')
                return 1  # return code 1 for timeout
            except CalledProcessError:
                self.current_app.logger.error('PhantomJS exited: {}'
                                              ''.format(p.returncode))
                return p.returncode
            else:
                self.current_app.logger.info('PhantomJS exited: 0')
                return 0

def _launch_webdriver(self):
        print("start chromedriver instance")
        p = subprocess.Popen(['chromedriver', '--port='+str(self._port)])
        try:
            p.wait(timeout=2.0)
            return False
        except subprocess.TimeoutExpired:
            return True

def compile_problem(solution_spec):
    """Compiles a problem submission and generates a problem spec.

    Args:
        solution_spec: Specification string of a solution corresponding to the
            submitted problem.

    Returns:
        (problem_spec, problem_size)
        problem_spec: Specification string of the problem.
        problem_size: Problem size.

    Raises:
        VerificationError: If the solution specification is invalid.
        subprocess.TimeoutExpired: On judge timeout.
        AssertionError: On scrape error.
    """
    with make_temporary_file_with_content(solution_spec) as solution_file:
        proc = subprocess.Popen(
            ['./akatsuki', '--logtostderr', '--compile', solution_file.name],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
        try:
            stdout_output, stderr_output = proc.communicate(
                timeout=_JUDGE_TIMEOUT_SECONDS)
        except subprocess.TimeoutExpired:
            proc.kill()
            proc.wait()
            raise  # report ISE
    if proc.returncode:
        m = _VERIFICATION_ERROR_RE.search(stdout_output)
        assert m, stdout_output  # report ISE
        raise VerificationError(m.group(1))
    problem_spec = stdout_output
    problem_size = sum(len(s) for s in problem_spec.split())
    return (problem_spec, problem_size)

def evaluate_solution(problem_spec, solution_spec):
    """Evaluates a solution submission.

    Args:
        problem_spec: Specification string of a problem.
        solution_spec: Specification string of a solution.

    Returns:
        (resemblance_int, raw_evaluator_output)

    Raises:
        VerificationError: If any of the specifications are invalid.
        subprocess.TimeoutExpired: On judge timeout.
        AssertionError: On scrape error.
    """
    with make_temporary_file_with_content(problem_spec) as problem_file, \
         make_temporary_file_with_content(solution_spec) as solution_file:
        proc = subprocess.Popen(
            ['./akatsuki', '--logtostderr', '--evaluate',
             problem_file.name, solution_file.name],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
        try:
            stdout_output, stderr_output = proc.communicate(
                timeout=_JUDGE_TIMEOUT_SECONDS)
        except subprocess.TimeoutExpired:
            proc.kill()
            proc.wait()
            raise  # report ISE
    if proc.returncode:
        m = _VERIFICATION_ERROR_RE.search(stdout_output)
        assert m, stdout_output  # report ISE
        raise VerificationError(m.group(1))
    m = re.search(r'integer_resemblance: (\d+)', stdout_output)
    assert m, stdout_output  # report ISE
    resemblance_int = int(m.group(1))
    return resemblance_int, stdout_output.decode('utf-8')

def run_proc_with_quit(proc_id, quit_dict, args, logfile=None, append=False, env=None, cwd=None):
    if logfile is None:
        logfile = os.devnull

    mode = 'ab' if append else 'wb'
    with open(logfile, mode) as logf:
        if proc_id in quit_dict:
            return None
        proc = subprocess.Popen(args, stdout=logf, stderr=subprocess.STDOUT,
                                env=env, cwd=cwd)
        retcode = None
        num_kill = 0
        timeout = 0.05
        while retcode is None and num_kill <= 2:
            try:
                retcode = proc.wait(timeout=timeout)
            except subprocess.TimeoutExpired:
                if proc_id in quit_dict:
                    if num_kill == 0:
                        proc.terminate()
                        timeout = quit_dict[proc_id]
                    elif num_kill == 1:
                        proc.kill()
                    num_kill += 1

        return proc.returncode

def run_in_subprocess(cmd, check_output=False, ** kwargs):
    """
    Execute command using default subprocess configuration.

    Parameters
    ----------
    cmd : string
        Command to be executed in subprocess.
    kwargs : keywords
        Options to pass to subprocess.Popen.

    Returns
    -------
    proc : Popen subprocess
        Subprocess used to run command.
    """

    logger.debug('Executing command: {0}'.format(cmd))
    config = DEFAULT_SUBPROCESS_CONFIG.copy()
    config.update(kwargs)
    if not check_output:
        if omniduct_config.logging_level < 20:
            config['stdout'] = None
            config['stderr'] = None
        else:
            config['stdout'] = open(os.devnull, 'w')
            config['stderr'] = open(os.devnull, 'w')
    timeout = config.pop('timeout', None)

    process = subprocess.Popen(cmd, **config)
    try:
        stdout, stderr = process.communicate(None, timeout=timeout)
    except subprocess.TimeoutExpired:
        os.killpg(os.getpgid(process.pid), signal.SIGINT)  # send signal to the process group, recurively killing all children
        output, unused_err = process.communicate()
        raise subprocess.TimeoutExpired(process.args, timeout, output=output)
    return SubprocessResults(returncode=process.returncode, stdout=stdout or '', stderr=stderr or '')

def subproc_call(cmd, timeout=None):
    try:
        output = subprocess.check_output(
                cmd, stderr=subprocess.STDOUT,
                shell=True, timeout=timeout)
        return output
    except subprocess.TimeoutExpired as e:
        logger.warn("Command timeout!")
        logger.warn(e.output)
    except subprocess.CalledProcessError as e:
        logger.warn("Commnad failed: {}".format(e.returncode))
        logger.warn(e.output)

def test_verification_timeout(self):
        self.mock_Popen.return_value.communicate.side_effect = TimeoutExpired(
            [], 30)

        result = self.runner.invoke(verify.main, self.default_opts)
        self.assertEqual(result.exit_code, -1)
        self.assertIsInstance(result.exception, TimeoutExpired)
        self.assertProcess(*self.default_call)
        self.mock_Popen.return_value.communicate.assert_called_once_with(
            timeout=30)

def place_file(self, id, source_path):
    target_dir = os.path.join(IMAGERY_PATH, id)
    if not os.path.exists(target_dir):
        os.mkdir(target_dir)
    output_file = os.path.abspath(os.path.join(target_dir, 'index.tif'))

    # rewrite with gdal_translate
    gdal_translate = [
        'gdal_translate',
        source_path,
        output_file,
        '-co', 'TILED=yes',
        '-co', 'COMPRESS=DEFLATE',
        '-co', 'PREDICTOR=2',
        '-co', 'BLOCKXSIZE=512',
        '-co', 'BLOCKYSIZE=512',
        '-co', 'NUM_THREADS=ALL_CPUS',
    ]

    started_at = datetime.utcnow()

    self.update_state(state='RUNNING',
                      meta={
                        'name': 'preprocess',
                        'started_at': started_at.isoformat(),
                        'status': 'Rewriting imagery'
                      })

    try:
        returncode = subprocess.call(gdal_translate, timeout=TASK_TIMEOUT)
    except subprocess.TimeoutExpired as e:
        raise Exception(json.dumps({
            'name': 'preprocess',
            'started_at': started_at.isoformat(),
            'command': ' '.join(gdal_translate),
            'status': 'Timed out'
        }))

    if returncode != 0:
        raise Exception(json.dumps({
            'name': 'preprocess',
            'started_at': started_at.isoformat(),
            'command': ' '.join(gdal_translate),
            'return_code': returncode,
            'status': 'Failed'
        }))

    if not source_path.startswith(('/vsicurl', 'http://', 'https://')):
        # delete original
        os.unlink(source_path)

    return {
        'name': 'preprocess',
        'completed_at': datetime.utcnow().isoformat(),
        'started_at': started_at,
        'status': 'Image pre-processing completed'
    }

def create_warped_vrt(self, id):
    raster_path = os.path.abspath(os.path.join(IMAGERY_PATH, id, 'index.tif'))
    vrt_path = os.path.abspath(os.path.join(IMAGERY_PATH, id, 'index.vrt'))
    meta = get_metadata(id)
    approximate_zoom = meta['meta']['approximateZoom']

    # create a warped VRT to reproject on the fly
    gdalwarp = [
        'gdalwarp',
        raster_path,
        vrt_path,
        '-r', 'cubic',
        '-t_srs', 'epsg:3857',
        '-overwrite',
        '-of', 'VRT',
        '-te', '-20037508.34', '-20037508.34', '20037508.34', '20037508.34',
        '-ts', str(2 ** approximate_zoom * 256), str(2 ** approximate_zoom * 256),
    ]

    # add an alpha band (for NODATA) if one wasn't already included
    if meta['meta']['bandCount'] < 4:
        gdalwarp.append('-dstalpha')

    started_at = datetime.utcnow()

    self.update_state(state='RUNNING',
                      meta={
                        'name': 'warped-vrt',
                        'started_at': started_at.isoformat(),
                        'status': 'Creating warped VRT'
                      })

    try:
        returncode = subprocess.call(gdalwarp, timeout=TASK_TIMEOUT)
    except subprocess.TimeoutExpired as e:
        raise Exception(json.dumps({
            'name': 'warped-vrt',
            'started_at': started_at.isoformat(),
            'command': ' '.join(gdalwarp),
            'status': 'Timed out'
        }))

    if returncode != 0:
        raise Exception(json.dumps({
            'name': 'warped-vrt',
            'started_at': started_at.isoformat(),
            'command': ' '.join(gdalwarp),
            'return_code': returncode,
            'status': 'Failed'
        }))

    return {
        'completed_at': datetime.utcnow().isoformat(),
        'started_at': started_at,
        'status': 'Warped VRT creation completed'
    }

def _bootup_vm(self, cores, memory):
        """Boot up the VM as, internal helper funtion.

        Note that it opens temporarily file as self._vm_pidfile.
        """
        LOG.debug("Spawning up VM to run jobs within")
        drive = "file={0._disk},media=disk,discard=unmap,snapshot={0._snapshot},if=virtio".format(self)
        netdev = ("user,id=fakenet0,net=172.16.6.0/24,restrict={0._restrict_net},"
                  "hostfwd=tcp:127.0.0.1:{0._ssh_port}-:22,").format(self)
        self._vm_pidfile = tempfile.NamedTemporaryFile(mode='r', prefix="worker-vm", suffix="pid")

        kvm_command = ["kvm", "-name", self._vm_name,
                       "-sandbox", self._sandbox,
                       "-machine", "pc-i440fx-1.7,accel=kvm,usb=off",
                       "-cpu", "SandyBridge",
                       "-smp", "{}".format(cores),
                       "-m", "{}M".format(memory),
                       "-snapshot",
                       "-drive", drive,
                       "-netdev", netdev,
                       "-net", "nic,netdev=fakenet0,model=virtio",
                       "-daemonize",
                       "-pidfile", self._vm_pidfile.name,
                       "-vnc", "none"]
        try:
            kvm_process = subprocess.Popen(kvm_command, stdout=subprocess.PIPE,
                                           stderr=subprocess.PIPE)
        except OSError as e:
            LOG.error("Is KVM installed? Popen raised %s", e)
            raise EnvironmentError("Unable to start VM, KVM process failed %s", e)

        stdout, stderr = None, None
        try:
            stdout, stderr = kvm_process.communicate(timeout=self._kvm_timeout)
            LOG.debug("stdout: %s", stdout)
            LOG.debug("stderr: %s", stderr)
        except subprocess.TimeoutExpired:
            LOG.error("VM did not start within %s seconds, killing it", self._kvm_timeout)
            LOG.debug("stdout: %s", stdout)
            LOG.debug("stderr: %s", stderr)
            kvm_process.terminate()
            if self.vm_pid is not None:
                os.kill(self.vm_pid, signal.SIGTERM)

            LOG.warning("5 seconds grace period before forcefully killing VM")
            time.sleep(5)
            kvm_process.kill()
            if self.vm_pid is not None:
                os.kill(self.vm_pid, signal.SIGKILL)

            raise EnvironmentError("KVM start did not boot up properly")