Crawler.py

import urllib.request
import os
x = int(input("Start number:\n"))
end = int(input("End number:\n"))
f1 = open("do.txt", "a")
f2 = open("dont.txt", "a")
if os.stat('dont.txt').st_size == 0:
    f1.write("\n")
    f2.write("\n")
f1.close()
f2.close()
while x < end:
    try:
        urllib.request.urlretrieve ("http://cds.idc.ique.com:16963/cds/download?content_id=" + str(x), "download" + str(x))
        print("http://cds.idc.ique.com:16963/cds/download?content_id=" + str(x) + " downloaded")
        with open("do.txt", "r+") as file:
            for line in file:
                if "\n" + str(x) + "\n" in line:
                    print("http://cds.idc.ique.com:16963/cds/download?content_id=" + str(x) + " already known to exist")
                    break
                else:
                    file.write(str(x) + "\n")
                    break
    except urllib.error.HTTPError:
        print("http://cds.idc.ique.com:16963/cds/download?content_id=" + str(x) + " not found")
        with open("dont.txt", "r+") as file:
            for line in file:
                if "\n" + str(x) + "\n" in line:
                    print("http://cds.idc.ique.com:16963/cds/download?content_id=" + str(x) + " already known to not exist")
                    break
                else:
                    file.write(str(x) + "\n")
                    break
    except ConnectionResetError:
        print("Connection reset error, retrying")
        x -= 1
    except urllib.error.URLError:
        print("URLError, retrying")
        x -= 1
    x += 1
exit = input("Press any button to exit...")