diff --git a/README.md b/README.md index e69de29..4ba5f6e 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,7 @@ += ipset updater + += install + +* git clone this +* launch ./install.sh +* launch ./ipset_update.sh for first time diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..54e51ef --- /dev/null +++ b/install.sh @@ -0,0 +1,4 @@ +#!/bin/bash +echo "30 12 * * * root /opt/ipset_updater/ipset_update.sh >> /var/log/blacklist_update.log" > /etc/cron.d/ipset_update +chmod a+x /etc/cron.d/ipset_update +systemctl restart cron.service diff --git a/ipset_update.sh b/ipset_update.sh index 8f8f916..442a7d3 100755 --- a/ipset_update.sh +++ b/ipset_update.sh @@ -1,21 +1,26 @@ #!/bin/bash cd "$(dirname "$0")" -echo "Runinng list update" +date_good=`date +"%Y-%m-%d %T"` +echo "* ${date_good} Runinng list update" python ./lists_updater.py -echo "Create ipset blocklists if not existing" +echo "* Create ipset blocklists if not already existing" ipset create blacklist_net -exist hash:net family inet hashsize 16777216 maxelem 16777216 ipset create blacklist_ipv4 -exist hash:ip family inet hashsize 16777216 maxelem 16777216 -ipset create blacklist_ipv6 -exist hash:net family inet hashsize 16777216 maxelem 16777216 +#ipset create blacklist_ipv6 -exist hash:net family inet hashsize 16777216 maxelem 16777216 -echo "Import lists into ipset" +echo "* Import lists into ipset" +echo "== Import ipv4 ipset" ipset restore < ipset_ipv4.set +#echo "== Import ipv6 ipset" #ipset restore < ipset_ipv6.set +echo "== Import subnets ipset" ipset restore < ipset_subnets.set -echo "Saving ipset" +echo "* Saving ipset" ipset save > /etc/ipset.conf +echo "* Cleanup ipset files" rm ./ipset_ipv4.set -rm ./ipset_ipv6.set +#rm ./ipset_ipv6.set rm ./ipset_subnets.set diff --git a/lists.json b/lists.json index 1e7756c..240c83d 100644 --- a/lists.json +++ b/lists.json @@ -3,12 +3,15 @@ "ipv4": { "spamhaus":"https://www.spamhaus.org/drop/drop.txt", - "blocklist":"https://lists.blocklist.de/lists/all.txt" - }, - "ipv6": - { - "bogons":"https://www.team-cymru.org/Services/Bogons/fullbogons-ipv6.txt", - "spamhaus":"https://www.spamhaus.org/drop/dropv6.txt" + "blocklist":"https://lists.blocklist.de/lists/all.txt", + "stopforumspam":"https://iplists.firehol.org/files/stopforumspam.ipset", + "haley_ssh":"https://iplists.firehol.org/files/haley_ssh.ipset", + "blocklist_ssh":"https://iplists.firehol.org/files/blocklist_de_ssh.ipset", + "bi_any_0_1d":"https://iplists.firehol.org/files/bi_any_0_1d.ipset", + "bi_any__1_7d":"https://iplists.firehol.org/files/bi_any_1_7d.ipset", + "bi_any_2_1d":"https://iplists.firehol.org/files/bi_any_2_1d.ipset", + "bi_any_2_30d":"https://iplists.firehol.org/files/bi_any_2_30d.ipset", + "bi_any_2_7d":"https://iplists.firehol.org/files/bi_any_2_7d.ipset" }, "net": { @@ -16,7 +19,11 @@ "bogons":"https://www.team-cymru.org/Services/Bogons/fullbogons-ipv4.txt", "firehol":"https://iplists.firehol.org/files/firehol_level1.netset", "kor":"https://www.okean.com/sinokoreacidr.txt", - "cn":"https://www.okean.com/chinacidr.txt" + "cn":"https://www.okean.com/chinacidr.txt", + "ru":"https://iplists.firehol.org/files/ip2location_country/ip2location_country_ru.netset", + "pk":"https://iplists.firehol.org/files/ip2location_country/ip2location_country_pk.netset", + "sa":"https://iplists.firehol.org/files/ip2location_country/ip2location_country_sa.netset", + "cn2":"https://iplists.firehol.org/files/ip2location_country/ip2location_country_cn.netset" } } } diff --git a/lists_updater.py b/lists_updater.py index 6eb5a1b..2196c85 100644 --- a/lists_updater.py +++ b/lists_updater.py @@ -1,11 +1,11 @@ #!/usr/bin/python import os import json -import pprint import requests import re import ipaddress + print('Reading lists of URL to download') with open("lists.json","r") as f: data = json.load(f) @@ -28,16 +28,22 @@ def is_ipv4(ip): else: return False +def is_ipv6(ip): + try: + ip_addr = ipaddress.IPv6Address(ip) + except ipaddress.AddressValueError: + return False + return ip_addr.version == 6 + +def extract_ipv6(source): + return re.findall(regex_ipv6_cidr,source) + def is_ipv6_subnet(ip): try: addr = ipaddress.IPv6Network(ip) except Exception as e: return False return True - #if re.match("^(((?=.*(::))(?!.*\3.+\3))\3?|[\dA-F]{1,4}:)([\dA-F]{1,4}(\3|:\b)|\2){5}(([\dA-F]{1,4}(\3|:\b|$)|\2){2}|(((2[0-4]|1\d|[1-9])?\d|25[0-5])\.?\b){4})\Z",ip): - # return True - #else: - # return False def download_list(list_name,list_url): print("- Downloading {}".format(list_name)) @@ -45,18 +51,19 @@ def download_list(list_name,list_url): try: filedata = requests.get(list_url).content.split('\n') except Exception as e: - print(e) + print("Error downloading {} : {}".format(list_name,e)) return filedata + def update_ipset_files(): ipv4_list = [] ipv6_list = [] net_list = [] - for elem in ['ipv4','ipv6','net']: + for elem in ['ipv4','net']: print("= Update {}".format(elem)) elem_list = [] - + for key,url in data['lists'][elem].iteritems(): dl_list = download_list(key,url) ipv4_tab = [] @@ -75,17 +82,16 @@ def update_ipset_files(): if elem == "net": if is_subnet(line): subnet_tab.append("add blacklist_net {} -exist".format(line)) - + if elem == "ipv4": ipv4_list.extend(ipv4_tab) if elem == "ipv6": ipv6_list.extend(ipv6_tab) if elem == "net": net_list.extend(subnet_tab) ipv4_list = sorted(set(ipv4_list)) ipv6_list = sorted(set(ipv6_list)) inet_list = sorted(set(net_list)) - + save_to_file(ipv4_list,"ipset_ipv4.set") - save_to_file(ipv6_list,"ipset_ipv6.set") + #save_to_file(ipv6_list,"ipset_ipv6.set") save_to_file(net_list,"ipset_subnets.set") - update_ipset_files()