Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import pika, sys, os
- import requests
- from multiprocessing import Pool
- from request_retrier import retry_request_till_success
- import PIL.Image as Image
- import PIL
- dest_folder = "images/source-512/"
- def image_is_invalid(file_path):
- try:
- img = Image.open(file_path)
- img.verify()
- return False
- except PIL.UnidentifiedImageError:
- return True
- def file_exists_and_valid_image(file_path):
- return os.path.exists(file_path) and not image_is_invalid(file_path)
- def download_file(icon_url, channel, method):
- file_name = os.path.basename(icon_url)
- file_path = os.path.join(dest_folder, file_name)
- if file_exists_and_valid_image(file_path):
- print(f"File already exists and is valid image: {file_name}")
- channel.basic_ack(delivery_tag=method.delivery_tag) # REMOVE FROM QUEUE
- return
- icon_url_512 = icon_url.replace("/128/", "/512/")
- response = retry_request_till_success(requests.get)(icon_url_512)
- content = response.content
- with open(file_path, "wb") as f:
- f.write(content)
- file_size = sys.getsizeof(content)
- print(f"File size: {file_size} bytes, Downloaded: {file_name}")
- channel.basic_ack(delivery_tag=method.delivery_tag) # REMOVE FROM QUEUE
- def worker(args):
- try:
- download_file(*args)
- except Exception as e:
- print(f"Error downloading file: {e}")
- def main():
- connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
- channel = connection.channel()
- channel.queue_declare(queue='hello')
- pool = Pool(processes=100)
- def callback(ch, method, properties, body):
- string = body.decode()
- pool.apply_async(worker, ((string, ch, method),))
- channel.basic_consume(queue='hello', on_message_callback=callback, auto_ack=False)
- print(' [*] Waiting for messages. To exit press CTRL+C')
- channel.start_consuming()
- if __name__ == '__main__':
- try:
- main()
- except KeyboardInterrupt:
- print('Interrupted')
- try:
- sys.exit(0)
- except SystemExit:
- os._exit(0)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement