getfavicon.py

#!/usr/bin/python
# Copyright (c) 2009 Denis Bilenko. See LICENSE for details.
"""Spawn multiple workers and wait for them to complete"""

ursl = []
urls = lines = ['http://www.' + line.strip() for line in open('urllist.txt')]

import gevent
from gevent import monkey

# patches stdlib (including socket and ssl modules) to cooperate with other greenlets
monkey.patch_all()

import urllib2
from socket import setdefaulttimeout
setdefaulttimeout(30)

def print_head(url):
     print ('Starting %s' % url)
     url = url + '/favicon.ico'
     try:
         data = urllib2.urlopen(url).read()
         except Exception, e:
         print 'error', url, e
         return

    fn = 'icons/' + url[+11:].replace("/", "-")
    myFile = file(fn, 'w')
    myFile.write(data)
    myFile.close()

jobs = [gevent.spawn(print_head, url) for url in urls]

gevent.joinall(jobs)