diff --git a/ipset_update.sh b/ipset_update.sh new file mode 100755 index 0000000..8f8f916 --- /dev/null +++ b/ipset_update.sh @@ -0,0 +1,21 @@ +#!/bin/bash +cd "$(dirname "$0")" +echo "Runinng list update" +python ./lists_updater.py + +echo "Create ipset blocklists if not existing" +ipset create blacklist_net -exist hash:net family inet hashsize 16777216 maxelem 16777216 +ipset create blacklist_ipv4 -exist hash:ip family inet hashsize 16777216 maxelem 16777216 +ipset create blacklist_ipv6 -exist hash:net family inet hashsize 16777216 maxelem 16777216 + +echo "Import lists into ipset" +ipset restore < ipset_ipv4.set +#ipset restore < ipset_ipv6.set +ipset restore < ipset_subnets.set + +echo "Saving ipset" +ipset save > /etc/ipset.conf + +rm ./ipset_ipv4.set +rm ./ipset_ipv6.set +rm ./ipset_subnets.set diff --git a/lists.json b/lists.json new file mode 100644 index 0000000..1e7756c --- /dev/null +++ b/lists.json @@ -0,0 +1,22 @@ +{ + "lists":{ + "ipv4": + { + "spamhaus":"https://www.spamhaus.org/drop/drop.txt", + "blocklist":"https://lists.blocklist.de/lists/all.txt" + }, + "ipv6": + { + "bogons":"https://www.team-cymru.org/Services/Bogons/fullbogons-ipv6.txt", + "spamhaus":"https://www.spamhaus.org/drop/dropv6.txt" + }, + "net": + { + "emerging":"https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt", + "bogons":"https://www.team-cymru.org/Services/Bogons/fullbogons-ipv4.txt", + "firehol":"https://iplists.firehol.org/files/firehol_level1.netset", + "kor":"https://www.okean.com/sinokoreacidr.txt", + "cn":"https://www.okean.com/chinacidr.txt" + } + } +} diff --git a/lists_updater.py b/lists_updater.py new file mode 100644 index 0000000..6eb5a1b --- /dev/null +++ b/lists_updater.py @@ -0,0 +1,91 @@ +#!/usr/bin/python +import os +import json +import pprint +import requests +import re +import ipaddress + +print('Reading lists of URL to download') +with open("lists.json","r") as f: + data = json.load(f) + + +def save_to_file(text,filename): + print("= Saved {} elements to {}".format(len(text),filename)) + with open('./{}'.format(filename), mode='w') as myfile: + myfile.write('\n'.join(text)) + +def is_subnet(subnet): + if re.match('^([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?$',subnet): + return True + else: + return False + +def is_ipv4(ip): + if re.match("^(\d{0,3})\.(\d{0,3})\.(\d{0,3})\.(\d{0,3})$", ip): + return True + else: + return False + +def is_ipv6_subnet(ip): + try: + addr = ipaddress.IPv6Network(ip) + except Exception as e: + return False + return True + #if re.match("^(((?=.*(::))(?!.*\3.+\3))\3?|[\dA-F]{1,4}:)([\dA-F]{1,4}(\3|:\b)|\2){5}(([\dA-F]{1,4}(\3|:\b|$)|\2){2}|(((2[0-4]|1\d|[1-9])?\d|25[0-5])\.?\b){4})\Z",ip): + # return True + #else: + # return False + +def download_list(list_name,list_url): + print("- Downloading {}".format(list_name)) + filedata = "" + try: + filedata = requests.get(list_url).content.split('\n') + except Exception as e: + print(e) + return filedata + +def update_ipset_files(): + ipv4_list = [] + ipv6_list = [] + net_list = [] + + for elem in ['ipv4','ipv6','net']: + print("= Update {}".format(elem)) + elem_list = [] + + for key,url in data['lists'][elem].iteritems(): + dl_list = download_list(key,url) + ipv4_tab = [] + ipv6_tab = [] + subnet_tab = [] + + + for line in dl_list: + + if elem == "ipv4": + if is_ipv4(line): + ipv4_tab.append("add blacklist_ipv4 {} -exist".format(line)) + if elem == "ipv6": + if is_ipv6_subnet(line): + ipv6_tab.append("add blacklist_ipv6 {} -exist".format(line)) + if elem == "net": + if is_subnet(line): + subnet_tab.append("add blacklist_net {} -exist".format(line)) + + if elem == "ipv4": ipv4_list.extend(ipv4_tab) + if elem == "ipv6": ipv6_list.extend(ipv6_tab) + if elem == "net": net_list.extend(subnet_tab) + ipv4_list = sorted(set(ipv4_list)) + ipv6_list = sorted(set(ipv6_list)) + inet_list = sorted(set(net_list)) + + save_to_file(ipv4_list,"ipset_ipv4.set") + save_to_file(ipv6_list,"ipset_ipv6.set") + save_to_file(net_list,"ipset_subnets.set") + + +update_ipset_files()