Python eventlet 模块，net() 实例源码

def monkey_patch():
    """Greening the world (eventlet)

    The approach is to monkeypatch the Standard Library. For more details
    see http://eventlet.net/doc/patching.html
    """
    if os.name == 'nt':
        # eventlet monkey patching the os and thread modules causes
        # subprocess.Popen to fail on Windows when using pipes due
        # to missing non-blocking IO support.
        #
        # bug report on eventlet:
        # https://bitbucket.org/eventlet/eventlet/issue/132/
        #       eventletmonkey_patch-breaks
        eventlet.monkey_patch(os=False, thread=False)
    else:
        eventlet.monkey_patch()

def producer(start_url):
    """Recursively crawl starting from *start_url*.  Returns a set of
    urls that were found."""
    pool = eventlet.GreenPool()
    seen = set()
    q = eventlet.Queue()
    q.put(start_url)
    # keep looping if there are new urls, or workers that may produce more urls
    while True:
        while not q.empty():
            url = q.get()
            # limit requests to eventlet.net so we don't crash all over the internet
            if url not in seen and 'eventlet.net' in url:
                seen.add(url)
                pool.spawn_n(fetch, url, q)
        pool.waitall()
        if q.empty():
            break

    return seen

def _yield_before_after():
    # Yield to any other co-routines...
    #
    # See: http://eventlet.net/doc/modules/greenthread.html
    # for how this zero sleep is really a cooperative yield to other potential
    # co-routines...
    eventlet.sleep(0)
    try:
        yield
    finally:
        eventlet.sleep(0)

def _yield_before_after():
    # Yield to any other co-routines...
    #
    # See: http://eventlet.net/doc/modules/greenthread.html
    # for how this zero sleep is really a cooperative yield to other potential
    # co-routines...
    eventlet.sleep(0)
    try:
        yield
    finally:
        eventlet.sleep(0)

def fetch(url, seen, pool):
    """Fetch a url, stick any found urls into the seen set, and
    dispatch any new ones to the pool."""
    print("fetching", url)
    data = ''
    with eventlet.Timeout(5, False):
        data = urllib2.urlopen(url).read()
    for url_match in url_regex.finditer(data):
        new_url = url_match.group(0)
        # only send requests to eventlet.net so as not to destroy the internet
        if new_url not in seen and 'eventlet.net' in new_url:
            seen.add(new_url)
            # while this seems stack-recursive, it's actually not:
            # spawned greenthreads start their own stacks
            pool.spawn_n(fetch, new_url, seen, pool)