geoip + python3 - scan nginx access logs inclusive gzipped

#!/usr/bin/env python3

# in venv
# pip install python-geoip-python3
# pip install python-geoip-geolite2

import glob
import gzip
from geoip import geolite2
import collections


def get_country(ip):
    """
    Returns the country as ISO 3166-1 alpha-2
    If the lookup fails, it returns None
    """
    result = geolite2.lookup(ip)
    if result:
        return result.country


def scan_ips(nginx_access_log):
    filter_ips = lambda fd: (line.split()[0] for line in fd)
    if nginx_access_log.endswith('.gz'):
        with gzip.open(nginx_access_log, 'rt') as fd:
            yield from filter_ips(fd)
    else:
        with open(nginx_access_log) as fd:
            for line in fd:
                yield from filter_ips(fd)


def scan_all_logs(pattern):
    for file in glob.glob(pattern):
        yield from scan_ips(file)

ips = tuple(scan_all_logs('/var/log/nginx/archiv.vv.ist-im-web.de_access.log*'))

unique_ips = set(ips)
unique_countries = {get_country(ip) for ip in unique_ips}
countries_filtered = {c for c in unique_countries if c}

countries_all = [get_country(ip) for ip in ips]
countries_all = [c for c in countries_all if c]

top10 = collections.Counter(countries_all).most_common(10)

#[('US', 7407),
# ('DE', 1194),
# ('CA', 578),
# ('GR', 264),
# ('FR', 102),
# ('RU', 78),
# ('HK', 60),
# ('CH', 56),
# ('GB', 50),
# ('RO', 36)]