Python tornado.httpclient 模块，HTTPError() 实例源码

def _handle_exception(self, typ, value, tb):
        if self.final_callback:
            self._remove_timeout()
            if isinstance(value, StreamClosedError):
                if value.real_error is None:
                    value = HTTPError(599, "Stream closed")
                else:
                    value = value.real_error
            self._run_callback(HTTPResponse(self.request, 599, error=value,
                                            request_time=self.io_loop.time() - self.start_time,
                                            ))

            if hasattr(self, "stream"):
                # TODO: this may cause a StreamClosedError to be raised
                # by the connection's Future.  Should we cancel the
                # connection more gracefully?
                self.stream.close()
            return True
        else:
            # If our callback has already been called, we are probably
            # catching an exception that is not caused by us but rather
            # some child of our callback. Rather than drop it on the floor,
            # pass it along, unless it's just the stream being closed.
            return isinstance(value, StreamClosedError)

def _handle_exception(self, typ, value, tb):
        if self.final_callback:
            self._remove_timeout()
            if isinstance(value, StreamClosedError):
                if value.real_error is None:
                    value = HTTPError(599, "Stream closed")
                else:
                    value = value.real_error
            self._run_callback(HTTPResponse(self.request, 599, error=value,
                                            request_time=self.io_loop.time() - self.start_time,
                                            ))

            if hasattr(self, "stream"):
                # TODO: this may cause a StreamClosedError to be raised
                # by the connection's Future.  Should we cancel the
                # connection more gracefully?
                self.stream.close()
            return True
        else:
            # If our callback has already been called, we are probably
            # catching an exception that is not caused by us but rather
            # some child of our callback. Rather than drop it on the floor,
            # pass it along, unless it's just the stream being closed.
            return isinstance(value, StreamClosedError)

def _handle_exception(self, typ, value, tb):
        if self.final_callback:
            self._remove_timeout()
            if isinstance(value, StreamClosedError):
                if value.real_error is None:
                    value = HTTPError(599, "Stream closed")
                else:
                    value = value.real_error
            self._run_callback(HTTPResponse(self.request, 599, error=value,
                                            request_time=self.io_loop.time() - self.start_time,
                                            ))

            if hasattr(self, "stream"):
                # TODO: this may cause a StreamClosedError to be raised
                # by the connection's Future.  Should we cancel the
                # connection more gracefully?
                self.stream.close()
            return True
        else:
            # If our callback has already been called, we are probably
            # catching an exception that is not caused by us but rather
            # some child of our callback. Rather than drop it on the floor,
            # pass it along, unless it's just the stream being closed.
            return isinstance(value, StreamClosedError)

def request(method,url,params=None,data=None,context=None):
    http_client = httpclient.HTTPClient()        
    if params is not None and len(params.keys())>0:
        url+='?'
        for key in params.keys():
            url+="%s=%s&"%(key,params[key])
        url=url[:-1]

    if context is not None:
        url = url.replace('http://','https://',1)
    try:
        request = httpclient.HTTPRequest(url=url,
                                         method =method,
                                         ssl_options=context,
                                         body=data)
        response = http_client.fetch(request)
        http_client.close()
    except httpclient.HTTPError as e:
        if e.response is None:
            return tornado_response(500,str(e))

        return tornado_response(e.response.code,e.response.body)
    if response is None:
        return None
    return tornado_response(response.code,response.body)

def test_perm_roles_decorator(http_client, base_url, app_base_handlers, monkeypatch):
    with pytest.raises(HTTPError) as e:
        await http_client.fetch(base_url + '/test/api_test_model_dec/', method='GET')

    async def success_get_roles(self):
        return ['admin']

    monkeypatch.setattr(DecTestHandler, 'get_roles', success_get_roles)
    res = await http_client.fetch(base_url + '/test/api_test_model_dec/', method='GET')
    assert res.code == 200

    async def error_is_auth(self):
        return False

    monkeypatch.setattr(DecTestHandler, 'is_auth', error_is_auth)

    with pytest.raises(HTTPError) as e:
        await http_client.fetch(base_url + '/test/api_test_model_dec/', method='GET')

def ping(self, path=None):
        request = HTTPRequest(
            url=self.endpoint + (path or self.api_path),
            method='GET',
            headers=self.headers,
            follow_redirects=False,
            request_timeout=100
        )
        try:
            yield self.outbound_client.fetch(request)
        except HTTPError as ex:  # pragma: no cover
            if ex.code == 307:
                raise_from(MasterRedirect(
                    urlparse(ex.response.headers['location']).netloc), None)
        except ConnectionRefusedError as ex:  # pragma: no cover
            log.debug("Problem reaching: %s" % self.endpoint)
            raise ex
        except Exception as ex:  # pragma: no cover
            log.debug("Unhandled exception when connecting to %s",
                      self.endpoint)
            raise ex

def test_bad_request_error(req, msg):
    inbound = HTTPInbound()
    inbound.start(None)

    client = AsyncHTTPClient()

    req.url = 'http://localhost:%s' % inbound.port
    req.method = 'POST'
    req.body = ''

    with pytest.raises(HTTPError) as e:
        yield client.fetch(req)

    e = e.value
    assert e.code >= 400 and e.code <= 500
    assert e.response.body == msg

def get_entity(repository_id, entity_type, entity_id):
    """
    Get an entity from a repository

    :param repository_id: the repository that contains the entity
    :param entity_type: a valid entity type (asset, offer or agreement)
    :param entity_id: the entity's ID
    :returns: the entity data from the repository service
    :raises: tornado.httpclient.HTTPError
    """
    endpoint = yield entity_endpoint(repository_id, entity_type)
    if not endpoint:
        raise Return(None)

    try:
        result = yield endpoint[entity_id].get()
    except httpclient.HTTPError as err:
        if err.code == 404:
            raise Return(None)
        else:
            raise

    raise Return(result['data'])

def entity_endpoint(repository_id, entity_type):
    """
    Return an chub.API endpoint for the repository & entity type

    :param repository_id: the repository that contains the entity
    :param entity_type: a valid entity type (asset, offer or agreement)
    :returns: chub.API instance
    :raises: tornado.httpclient.HTTPError
    """
    try:
        repository = yield common.get_repository(repository_id)
    except httpclient.HTTPError as err:
        if err.code == 404:
            raise Return(None)
        else:
            raise

    url = repository['data']['service']['location']
    api = yield common.service_client('repository', url)
    endpoint = api.repository.repositories[repository_id][_pluralise(entity_type)]

    raise Return(endpoint)

def get_asset_offers_from_one_repo(repository, data):
    repository_id = repository['repository_id']
    try:
        repo = yield common.get_repository(repository_id)
    except httpclient.HTTPError as err:
        # Return None if could not get the URL, might be stale index data
        if err.code == 404:
            raise Return(None)
        else:
            raise

    url = repo['data']['service']['location']
    api = yield common.service_client('repository', url)
    endpoint = api.repository.repositories[repository_id].search.offers

    result = yield common.repository_request(endpoint, repository, data)
    raise Return(result)

def get(self):
        """
        Retrieve licensors
        """
        source_id_type = self.get_argument('source_id_type', None)
        source_id = self.get_argument('source_id', None)
        if not source_id_type or not source_id:
            raise HTTPError(400, 'Must have "source_id_type" and "source_id" parameters')

        try:
            translated_id = common.translate_id_pair(
                {'source_id_type': source_id_type, 'source_id': source_id})
        except ValueError:
            raise HTTPError(400, '{} is an invalid hub key'.format(source_id))

        try:
            result = yield licensors.get_asset_licensors(
                translated_id['source_id_type'],
                translated_id['source_id']
            )
            self.finish(result)
        except httpclient.HTTPError as exc:
            body = json.loads(exc.response.body)
            raise HTTPError(exc.response.code, body)

def test_post_with_bad_data(service_client, get_repositories, get_repository):
    get_repositories.return_value = future_repositories
    get_repository.return_value = future_repository

    mock_response = Mock()
    mock_response.body = '{"errors": [{"source_id_type": "", "message": "not supported asset id type"}]}'
    mock_response.code = 400
    exc = httpclient.HTTPError(400, response=mock_response)
    service_client.return_value = make_future(MagicMock())
    client = yield service_client()
    endpoint = client.repository.repositories[''].search.offers
    endpoint.post.side_effect = exc

    handler = _create_offers_handler()

    # MUT
    handler.request.body = ('[{"source_id":' +
                            '"https://openpermissions.org/s0/hub1/asset/exampleco/ExampleCoPictureID/1",' +
                            '"source_id_type":""}]')

    with pytest.raises(HTTPError) as excinfo:
        handler.post().result()

    assert excinfo.value.status_code == mock_response.code
    assert excinfo.value.errors == json.loads(mock_response.body)

def _handle_exception(self, typ, value, tb):
        if self.final_callback:
            self._remove_timeout()
            if isinstance(value, StreamClosedError):
                if value.real_error is None:
                    value = HTTPError(599, "Stream closed")
                else:
                    value = value.real_error
            self._run_callback(HTTPResponse(self.request, 599, error=value,
                                            request_time=self.io_loop.time() - self.start_time,
                                            ))

            if hasattr(self, "stream"):
                # TODO: this may cause a StreamClosedError to be raised
                # by the connection's Future.  Should we cancel the
                # connection more gracefully?
                self.stream.close()
            return True
        else:
            # If our callback has already been called, we are probably
            # catching an exception that is not caused by us but rather
            # some child of our callback. Rather than drop it on the floor,
            # pass it along, unless it's just the stream being closed.
            return isinstance(value, StreamClosedError)

def _handle_exception(self, typ, value, tb):
        if self.final_callback:
            self._remove_timeout()
            if isinstance(value, StreamClosedError):
                value = HTTPError(599, "Stream closed")
            self._run_callback(HTTPResponse(self.request, 599, error=value,
                                            request_time=self.io_loop.time() - self.start_time,
                                            ))

            if hasattr(self, "stream"):
                # TODO: this may cause a StreamClosedError to be raised
                # by the connection's Future.  Should we cancel the
                # connection more gracefully?
                self.stream.close()
            return True
        else:
            # If our callback has already been called, we are probably
            # catching an exception that is not caused by us but rather
            # some child of our callback. Rather than drop it on the floor,
            # pass it along, unless it's just the stream being closed.
            return isinstance(value, StreamClosedError)

def pushMessage2Client(self):
        from tornado.httpclient import HTTPClient, HTTPError
        from tornado.escape import json_decode
        self.logger.info("Print finished.")
        http_client = HTTPClient()
        #data = {"device_id": self.profile['boxid']}
        data = {"device_id": self.profile['boxid']}
        (headers, body) = self.createFormData(data)
        try:
            response = http_client.fetch("http://yun.mohou.com/api/cloud/push-message", method='POST', headers=headers, body=body, request_timeout=10)
            data = json_decode(response.body)
            self.logger.info("Response result: %s." % str(response.body))
            if data['code'] == 0:
                return 0
            else:
                return 1
        except HTTPError as err:
            self.logger.error("HTTPError: " + str(err))
            return 2
        except Exception as ex:
            self.logger.error("Exception: " + str(ex))
            return 2

def test_create_service_404_error(self, _get_service):
        client = MagicMock()
        organisation_id = 'organisation-id'
        name = 'service-name'
        location = 'https://example.com'
        service_type = 'external'

        client.accounts.organisations = {'organisation-id': MagicMock()}
        client.accounts.organisations[organisation_id].services.post.side_effect = HTTPError(404, 'Not Found')

        with pytest.raises(click.ClickException) as exc:
            _create_service(client, organisation_id, name, location, service_type)

        client.accounts.organisations[organisation_id].services.post.assert_called_once_with(
            name='service-name', location='https://example.com', service_type='external')
        assert not _get_service.called
        assert exc.value.message == ('\x1b[31mOrganisation organisation-id cannot be found. '
                                     'Please check organisation_id.\x1b[0m')

def test_create_service_already_created(self, _get_service):
        client = MagicMock()
        organisation_id = 'organisation-id'
        name = 'service-name'
        location = 'https://example.com'
        service_type = 'external'

        client.accounts.organisations = {'organisation-id': MagicMock()}
        client.accounts.organisations[organisation_id].services.post.side_effect = HTTPError(400, 'Service already Created')

        result = _create_service(client, organisation_id, name, location, service_type)

        client.accounts.organisations[organisation_id].services.post.assert_called_once_with(
            name='service-name', location='https://example.com', service_type='external')
        _get_service.assert_called_once_with(client, 'organisation-id', 'service-name')
        assert result == 'service-id'

def test_create_service_http_error_no_service(self, _get_service):
        client = MagicMock()
        organisation_id = 'organisation-id'
        name = 'service-name'
        location = 'https://example.com'
        service_type = 'external'

        client.accounts.organisations = {'organisation-id': MagicMock()}
        client.accounts.organisations[organisation_id].services.post.side_effect = HTTPError(400, 'Error Message')

        with pytest.raises(HTTPError) as exc:
            _create_service(client, organisation_id, name, location, service_type)

        client.accounts.organisations[organisation_id].services.post.assert_called_once_with(
            name='service-name', location='https://example.com', service_type='external')
        _get_service.assert_called_once_with(client, 'organisation-id', 'service-name')
        assert exc.value.message == 'HTTP 400: Error Message'

def _get_service(client, organisation_id, name):
    """
    Get service belonging to organisation which matches given service name
    :param client: Accounts Service API Client
    :param organisation_id: Id of Organisation
    :param name: Service Name
    :return: Service Id
    """
    try:
        response = client.accounts.services.get(organisation_id=organisation_id)
    except httpclient.HTTPError as exc:
        if exc.code == 404:
            # If error is a 404 then this means the organisation_id is not recognised. Raise this error immediately
            msg = ('Organisation {} cannot be found. '
                   'Please check organisation_id.'.format(organisation_id))
            raise click.ClickException(click.style(msg, fg='red'))
        else:
            raise exc

    services = [s for s in response['data'] if s['name'] == name]

    if services:
        return services[0]['id']
    return None

def _get_client_secret(client, service_id):
    """
    Get client secret for service
    :param client: Accounts Service API Client
    :param service_id: Service ID
    :return: Client secret (if available)
    """
    try:
        response = client.accounts.services[service_id].secrets.get()
    except httpclient.HTTPError as exc:
        if exc.code == 404:
            # If error is a 404 then this means the service_id is not recognised. Raise this error immediately
            msg = ('Service {} cannot be found.'.format(service_id))
            raise click.ClickException(click.style(msg, fg='red'))
        else:
            raise exc

    client_secrets = response['data']
    if client_secrets:
        return client_secrets[0]
    return None

def _handle_exception(self, typ, value, tb):
        if self.final_callback:
            self._remove_timeout()
            if isinstance(value, StreamClosedError):
                if value.real_error is None:
                    value = HTTPError(599, "Stream closed")
                else:
                    value = value.real_error
            self._run_callback(HTTPResponse(self.request, 599, error=value,
                                            request_time=self.io_loop.time() - self.start_time,
                                            ))

            if hasattr(self, "stream"):
                # TODO: this may cause a StreamClosedError to be raised
                # by the connection's Future.  Should we cancel the
                # connection more gracefully?
                self.stream.close()
            return True
        else:
            # If our callback has already been called, we are probably
            # catching an exception that is not caused by us but rather
            # some child of our callback. Rather than drop it on the floor,
            # pass it along, unless it's just the stream being closed.
            return isinstance(value, StreamClosedError)

def unregister(self, urlpath):
        """Unregisters a previously registered urlpath.
        If the urlpath is not found in the reverse proxy, it will not raise
        an error, but it will log the unexpected circumstance.

        Parameters
        ----------
        urlpath: str
            The absolute path of the url (e.g. /my/internal/service/"
        """
        self.log.info("Deregistering {} redirection".format(urlpath))

        try:
            yield self._reverse_proxy.api_request(urlpath, method='DELETE')
        except httpclient.HTTPError as e:
            if e.code == 404:
                self.log.warning("Could not find urlpath {} when removing"
                                 " container. In any case, the reverse proxy"
                                 " does not map the url. Continuing".format(
                                     urlpath))
            else:
                raise e

def _handle_exception(self, typ, value, tb):
        if self.final_callback:
            self._remove_timeout()
            if isinstance(value, StreamClosedError):
                if value.real_error is None:
                    value = HTTPError(599, "Stream closed")
                else:
                    value = value.real_error
            self._run_callback(HTTPResponse(self.request, 599, error=value,
                                            request_time=self.io_loop.time() - self.start_time,
                                            ))

            if hasattr(self, "stream"):
                # TODO: this may cause a StreamClosedError to be raised
                # by the connection's Future.  Should we cancel the
                # connection more gracefully?
                self.stream.close()
            return True
        else:
            # If our callback has already been called, we are probably
            # catching an exception that is not caused by us but rather
            # some child of our callback. Rather than drop it on the floor,
            # pass it along, unless it's just the stream being closed.
            return isinstance(value, StreamClosedError)

def _on_timeout(self, key, info=None):
        """Timeout callback of request.

        Construct a timeout HTTPResponse when a timeout occurs.

        :arg object key: A simple object to mark the request.
        :info string key: More detailed timeout information.
        """
        request, callback, timeout_handle = self.waiting[key]
        self.queue.remove((key, request, callback))

        error_message = "Timeout {0}".format(info) if info else "Timeout"
        timeout_response = HTTPResponse(
            request, 599, error=HTTPError(599, error_message),
            request_time=self.io_loop.time() - request.start_time)
        self.io_loop.add_callback(callback, timeout_response)
        del self.waiting[key]

def _handle_exception(self, typ, value, tb):
        if self.final_callback:
            self._remove_timeout()
            if isinstance(value, StreamClosedError):
                if value.real_error is None:
                    value = HTTPError(599, "Stream closed")
                else:
                    value = value.real_error
            self._run_callback(HTTPResponse(self.request, 599, error=value,
                                            request_time=self.io_loop.time() - self.start_time,
                                            ))

            if hasattr(self, "stream"):
                # TODO: this may cause a StreamClosedError to be raised
                # by the connection's Future.  Should we cancel the
                # connection more gracefully?
                self.stream.close()
            return True
        else:
            # If our callback has already been called, we are probably
            # catching an exception that is not caused by us but rather
            # some child of our callback. Rather than drop it on the floor,
            # pass it along, unless it's just the stream being closed.
            return isinstance(value, StreamClosedError)

def _on_timeout(self, key, info=None):
        """Timeout callback of request.

        Construct a timeout HTTPResponse when a timeout occurs.

        :arg object key: A simple object to mark the request.
        :info string key: More detailed timeout information.
        """
        request, callback, timeout_handle = self.waiting[key]
        self.queue.remove((key, request, callback))

        error_message = "Timeout {0}".format(info) if info else "Timeout"
        timeout_response = HTTPResponse(
            request, 599, error=HTTPError(599, error_message),
            request_time=self.io_loop.time() - request.start_time)
        self.io_loop.add_callback(callback, timeout_response)
        del self.waiting[key]

def _handle_exception(self, typ, value, tb):
        if self.final_callback:
            self._remove_timeout()
            if isinstance(value, StreamClosedError):
                if value.real_error is None:
                    value = HTTPError(599, "Stream closed")
                else:
                    value = value.real_error
            self._run_callback(HTTPResponse(self.request, 599, error=value,
                                            request_time=self.io_loop.time() - self.start_time,
                                            ))

            if hasattr(self, "stream"):
                # TODO: this may cause a StreamClosedError to be raised
                # by the connection's Future.  Should we cancel the
                # connection more gracefully?
                self.stream.close()
            return True
        else:
            # If our callback has already been called, we are probably
            # catching an exception that is not caused by us but rather
            # some child of our callback. Rather than drop it on the floor,
            # pass it along, unless it's just the stream being closed.
            return isinstance(value, StreamClosedError)

def get_posts_url_from_page(page_url):
    """
    ????????????URL
    :param page_url
    :return:
    """
    try:
        response = yield httpclient.AsyncHTTPClient().fetch(page_url, headers=headers)
        soup = BeautifulSoup(response.body, 'html.parser')
        posts_tag = soup.find_all('div', class_="post floated-thumb")
        urls = []
        for index, archive in enumerate(posts_tag):
            meta = archive.find("div", class_="post-meta")
            url = meta.p.a['href']
            urls.append(url)
        raise gen.Return(urls)
    except httpclient.HTTPError as e:
        print('Exception: %s %s' % (e, page_url))
        raise gen.Return([])

def execute_oauth_request(self, request, raise_error=True):
        """
        Execute an OAuth request

        Retry with a new set of tokens when the OAuth access token is expired or rejected
        """
        if self.credentials.is_expired():
            self.log.info("The OAuth token for %s expired at %s", self.user,
                          ctime(self.credentials.expires_at))
            yield self.refresh_tokens()

        self.authorize_request(request)

        try:
            return (yield self.execute_request(request, raise_error))
        except HTTPError as e:
            if e.response.code != 401:
                raise

            # Try once more with a new set of tokens
            self.log.info("The OAuth token for %s was rejected", self.user)
            yield self.refresh_tokens()
            self.authorize_request(request)
            return (yield self.execute_request(request, raise_error))

def execute_request(self, request, raise_error=True):
        """
        Execute an HTTP request and log the error, if any
        """

        self.log.debug("%s %s", request.method, request.url)
        http_client = AsyncHTTPClient()
        request.headers.update({
            'User-Agent': 'jupyterhub-carina/' + __version__
        })
        try:
            return (yield http_client.fetch(request, raise_error=raise_error))
        except HTTPError as e:
            self.log.exception('An error occurred executing %s %s:\n(%s) %s',
                               request.method, request.url, e.response.code, e.response.body)
            raise

def _get_repository(repository_id):
    """Get a repository from the accounts service

    :param repository_id: str
    :returns: repository resource
    :raises: koi.exceptions.HTTPError
    """
    client = API(options.url_accounts, ssl_options=ssl_server_options())

    try:
        repo = yield client.accounts.repositories[repository_id].get()
        raise Return(repo)
    except httpclient.HTTPError as exc:
        if exc.code == 404:
            msg = 'Unknown repository ID'
        else:
            msg = 'Unexpected error'

        raise exceptions.HTTPError(exc.code, msg, source='accounts')

def _get_provider_by_name(provider):
    """Get a provider from the accounts service

    :param provider: str
    :returns: organisation resource
    :raises: koi.exceptions.HTTPError
    """
    client = API(options.url_accounts, ssl_options=ssl_server_options())

    try:
        res = yield client.accounts.organisations.get(name=provider)
        raise Return(res['data'][0])
    except httpclient.HTTPError as exc:
        if exc.code == 404:
            msg = 'Unknown provider'
        else:
            msg = 'Unexpected error ' + exc.message

        raise exceptions.HTTPError(exc.code, msg, source='accounts')

def _get_provider(provider_id):
    """Get a provider from the accounts service

    :param provider_id: str
    :returns: organisation resource
    :raises: koi.exceptions.HTTPError
    """
    client = API(options.url_accounts, ssl_options=ssl_server_options())

    try:
        org = yield client.accounts.organisations[provider_id].get()
        raise Return(org)
    except httpclient.HTTPError as exc:
        if exc.code == 404:
            msg = 'Unknown provider ID'
        else:
            msg = 'Unexpected error'

        raise exceptions.HTTPError(exc.code, msg, source='accounts')

def _get_ids(repository_id, entity_id):
    """Get ids from the repository service

    :param provider_id: str
    :returns: organisation resource
    :raises: koi.exceptions.HTTPError
    """
    repository = yield _get_repository(repository_id)
    repository_url = repository['data']['service']['location']

    token = yield get_token(
        options.url_auth, options.service_id,
        options.client_secret, scope=Read(),
        ssl_options=ssl_server_options()
    )
    client = API(repository_url, token=token, ssl_options=ssl_server_options())

    try:
        res = yield client.repository.repositories[repository_id].assets[entity_id].ids.get()
        raise Return(res['data'])
    except httpclient.HTTPError as exc:
        raise exceptions.HTTPError(exc.code, str(exc), source='repository')

def _get_offers_by_type_and_id(source_id_type, source_id):
    """ get asset offers for given type and id
    :param source_id_type: str
    :param source_id: str
    :returns: list of offers json
    :raises: koi.exceptions.HTTPError
    """
    client = API(options.url_query, ssl_options=ssl_server_options())

    try:
        req_body = '[{"source_id_type": "' + source_id_type + '", "source_id": "' + source_id + '"}]'

        client.query.search.offers.prepare_request(headers={'Content-Type': 'application/json'},
                                                body=req_body.strip())
        res = yield client.query.search.offers.post()
        raise Return(res['data'])        
    except httpclient.HTTPError as exc:
        msg = 'Unexpected error ' + exc.message
        raise exceptions.HTTPError(exc.code, msg, source='query')

def _get_providers_by_type_and_id(source_id_type, source_id):
    """ get the matching providers for a given source_id_type and source_id

    :param source_id_type: str
    :param source_id: str        
    :returns: list of organisations
    :raises: koi.exceptsion.HTTPError
    """
    client = API(options.url_query, ssl_options=ssl_server_options())

    try:
        res = yield client.query.licensors.get(source_id_type=source_id_type, source_id=source_id)
        raise Return(res['data'])
    except httpclient.HTTPError as exc:
        if exc.code == 404:
            msg = 'No matching providers found'
        else:
            msg = 'Unexpected error ' + exc.message

        raise exceptions.HTTPError(exc.code, msg, source='query')

def fetch(url, *args, **kwargs):
    retry_max = kwargs.pop('retry_max', 5)
    retry = 0
    while True:
        try:
            result = yield http_client.fetch(url, *args, **kwargs)
            return result
        except HTTPError as e:
            if e.code == 599:
                retry += 1
                if retry < retry_max:
                    logger.error(
                        "Timeout in request: "
                        "sleeping for {}: {}".format(2**retry, url)
                    )
                    yield gen.sleep(2**retry)
                    continue
            raise

def find_faces_url(url, hash_face=False, upsample=1):
    """
    Given a URL to an image, find all the faces.  The returned list has
    dictionaries with the following fields:
        rect -> bounding rectangle around the face
        score -> score of the face detection (higher == better)
        pose -> index of sub-detector matched which roughly corresponds to pos
                (0 is best)
    """
    try:
        image_req = yield httpclient.fetch(url, request_timeout=30.0)
    except HTTPError as e:
        print("Exception while fetching image URL: {}: {}".format(url, e))
        return []
    if image_req.code != 200:
        return []
    image_fd = BytesIO(image_req.body)
    return (yield find_faces_buffer(image_fd, hash_face=hash_face,
                                    upsample=upsample))

def facebook_paginate(data, max_results=500):
    paginated_data = []
    while True:
        paginated_data.extend(data['data'])
        if max_results is not None and len(paginated_data) >= max_results:
            break
        try:
            paginate_url = data['paging']['next']
        except KeyError:
            break
        try:
            request = yield httpclient.fetch(paginate_url,
                                             validate_cert=False,
                                             request_timeout=10.0)
        except HTTPError:
            logger.exception("Exception while paginating facebook")
            break
        data = json.loads(request.body)
    return paginated_data

def send_async_request(url, headers=None, method="GET", body=None):
    headers = headers or {}
    if body or method.upper() == "POST":
        if 'Content-Type' not in headers:
            headers["Content-Type"] = "application/x-www-form-urlencoded"
    req = httpclient.HTTPRequest(
        url, method=method, body=body, headers=headers, allow_nonstandard_methods=True)
    http_request = AsyncHTTPClient()
    response = ""
    try:
        response = yield http_request.fetch(req)
    except httpclient.HTTPError as e:
        print("Error:" + str(e))
    except Exception as e:
        print("Error:" + str(e))
    else:
        return response
    finally:
        http_request.close()

def send_sync_request(url, headers=None, method="GET", body=None):
    headers = headers or {}
    if body or method.upper() == "POST":
        if "Content-Type" not in headers:
            headers["Content-Type"] = "application/x-www-form-urlencoded"
    req = httpclient.HTTPRequest(
        url, method=method, body=body, headers=headers, allow_nonstandard_methods=True)
    http_client = httpclient.HTTPClient()
    response = ""
    try:
        response = http_client.fetch(req)
        return response
    except httpclient.HTTPError as e:
        print("Error:" + str(e))
        raise httpclient.HTTPError()
    except Exception as e:
        print("Error:" + str(e))
        raise Exception
    finally:
        http_client.close()

def _on_timeout(self, key, info=None):
        """Timeout callback of request.

        Construct a timeout HTTPResponse when a timeout occurs.

        :arg object key: A simple object to mark the request.
        :info string key: More detailed timeout information.
        """
        request, callback, timeout_handle = self.waiting[key]
        self.queue.remove((key, request, callback))

        error_message = "Timeout {0}".format(info) if info else "Timeout"
        timeout_response = HTTPResponse(
            request, 599, error=HTTPError(599, error_message),
            request_time=self.io_loop.time() - request.start_time)
        self.io_loop.add_callback(callback, timeout_response)
        del self.waiting[key]

def _handle_exception(self, typ, value, tb):
        if self.final_callback:
            self._remove_timeout()
            if isinstance(value, StreamClosedError):
                if value.real_error is None:
                    value = HTTPError(599, "Stream closed")
                else:
                    value = value.real_error
            self._run_callback(HTTPResponse(self.request, 599, error=value,
                                            request_time=self.io_loop.time() - self.start_time,
                                            ))

            if hasattr(self, "stream"):
                # TODO: this may cause a StreamClosedError to be raised
                # by the connection's Future.  Should we cancel the
                # connection more gracefully?
                self.stream.close()
            return True
        else:
            # If our callback has already been called, we are probably
            # catching an exception that is not caused by us but rather
            # some child of our callback. Rather than drop it on the floor,
            # pass it along, unless it's just the stream being closed.
            return isinstance(value, StreamClosedError)

def send_email(to, subject, html):
    if isinstance(to, unicode):
        to = to.encode('utf-8')
    if isinstance(subject, unicode):
        subject = subject.encode('utf-8')
    if isinstance(html, unicode):
        html = html.encode('utf-8')

    data = {
        'from': CONFIG.EMAIL_SENDER,
        'to': to,
        'subject': subject,
        'html': html
    }
    data = urlencode(data)
    request = HTTPRequest(
        url=_MAILGUN_API_URL,
        method='POST',
        auth_username='api',
        auth_password=CONFIG.MAILGUN_API_KEY,
        body=data
    )
    client = AsyncHTTPClient()
    try:
        yield client.fetch(request)
    except HTTPError as e:
        try:
            response = e.response.body
        except AttributeError:
            response = None
        logging.exception('failed to send email:\nto: %s\nsubject: %s\nhtml: %s\nresponse: %s', to, subject, html, response)

def _on_timeout(self, key):
        request, callback, timeout_handle = self.waiting[key]
        self.queue.remove((key, request, callback))
        timeout_response = HTTPResponse(
            request, 599, error=HTTPError(599, "Timeout"),
            request_time=self.io_loop.time() - request.start_time)
        self.io_loop.add_callback(callback, timeout_response)
        del self.waiting[key]

def _on_timeout(self):
        self._timeout = None
        if self.final_callback is not None:
            raise HTTPError(599, "Timeout")

def on_connection_close(self):
        if self.final_callback is not None:
            message = "Connection closed"
            if self.stream.error:
                raise self.stream.error
            try:
                raise HTTPError(599, message)
            except HTTPError:
                self._handle_exception(*sys.exc_info())

def test_websocket_http_fail(self):
        with self.assertRaises(HTTPError) as cm:
            yield self.ws_connect('/notfound')
        self.assertEqual(cm.exception.code, 404)

def test_check_origin_invalid(self):
        port = self.get_http_port()

        url = 'ws://127.0.0.1:%d/echo' % port
        # Host is 127.0.0.1, which should not be accessible from some other
        # domain
        headers = {'Origin': 'http://somewhereelse.com'}

        with self.assertRaises(HTTPError) as cm:
            yield websocket_connect(HTTPRequest(url, headers=headers),
                                    io_loop=self.io_loop)

        self.assertEqual(cm.exception.code, 403)

def test_check_origin_invalid_subdomains(self):
        port = self.get_http_port()

        url = 'ws://localhost:%d/echo' % port
        # Subdomains should be disallowed by default.  If we could pass a
        # resolver to websocket_connect we could test sibling domains as well.
        headers = {'Origin': 'http://subtenant.localhost'}

        with self.assertRaises(HTTPError) as cm:
            yield websocket_connect(HTTPRequest(url, headers=headers),
                                    io_loop=self.io_loop)

        self.assertEqual(cm.exception.code, 403)