update

README.md

@@ -0,0 +1,7 @@
+= ipset updater
+
+= install
+
+* git clone this
+* launch ./install.sh
+* launch ./ipset_update.sh for first time

install.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+echo "30 12  * * *   root    /opt/ipset_updater/ipset_update.sh >> /var/log/blacklist_update.log" > /etc/cron.d/ipset_update
+chmod a+x /etc/cron.d/ipset_update
+systemctl restart cron.service

ipset_update.sh

@@ -1,21 +1,26 @@
 #!/bin/bash
 cd "$(dirname "$0")"
-echo "Runinng list update"
+date_good=`date +"%Y-%m-%d %T"`
+echo "* ${date_good} Runinng list update"
 python ./lists_updater.py
 
-echo "Create ipset blocklists if not existing"
+echo "* Create ipset blocklists if not already existing"
 ipset create blacklist_net -exist hash:net family inet hashsize 16777216 maxelem 16777216 
 ipset create blacklist_ipv4 -exist hash:ip family inet hashsize 16777216 maxelem 16777216 
-ipset create blacklist_ipv6 -exist hash:net family inet hashsize 16777216 maxelem 16777216
+#ipset create blacklist_ipv6 -exist hash:net family inet hashsize 16777216 maxelem 16777216
 
-echo "Import lists into ipset"
+echo "* Import lists into ipset"
+echo "== Import ipv4 ipset"
 ipset restore < ipset_ipv4.set
+#echo "== Import ipv6 ipset"
 #ipset restore < ipset_ipv6.set
+echo "== Import subnets ipset"
 ipset restore < ipset_subnets.set
 
-echo "Saving ipset"
+echo "* Saving ipset"
 ipset save > /etc/ipset.conf
 
+echo "* Cleanup ipset files"
 rm ./ipset_ipv4.set
-rm ./ipset_ipv6.set
+#rm ./ipset_ipv6.set
 rm ./ipset_subnets.set

lists.json

@@ -3,12 +3,15 @@
       "ipv4":
          {
             "spamhaus":"https://www.spamhaus.org/drop/drop.txt",
-            "blocklist":"https://lists.blocklist.de/lists/all.txt"
-         },
-      "ipv6":
-         {
-            "bogons":"https://www.team-cymru.org/Services/Bogons/fullbogons-ipv6.txt",
-            "spamhaus":"https://www.spamhaus.org/drop/dropv6.txt"
+            "blocklist":"https://lists.blocklist.de/lists/all.txt",
+	    "stopforumspam":"https://iplists.firehol.org/files/stopforumspam.ipset",
+	    "haley_ssh":"https://iplists.firehol.org/files/haley_ssh.ipset",
+	    "blocklist_ssh":"https://iplists.firehol.org/files/blocklist_de_ssh.ipset",
+	    "bi_any_0_1d":"https://iplists.firehol.org/files/bi_any_0_1d.ipset",
+	    "bi_any__1_7d":"https://iplists.firehol.org/files/bi_any_1_7d.ipset",
+	    "bi_any_2_1d":"https://iplists.firehol.org/files/bi_any_2_1d.ipset",
+	    "bi_any_2_30d":"https://iplists.firehol.org/files/bi_any_2_30d.ipset",
+	    "bi_any_2_7d":"https://iplists.firehol.org/files/bi_any_2_7d.ipset"
          },
       "net":
          {
@@ -16,7 +19,11 @@
             "bogons":"https://www.team-cymru.org/Services/Bogons/fullbogons-ipv4.txt",
             "firehol":"https://iplists.firehol.org/files/firehol_level1.netset",
             "kor":"https://www.okean.com/sinokoreacidr.txt",
-            "cn":"https://www.okean.com/chinacidr.txt"
+            "cn":"https://www.okean.com/chinacidr.txt",
+	    "ru":"https://iplists.firehol.org/files/ip2location_country/ip2location_country_ru.netset",
+	    "pk":"https://iplists.firehol.org/files/ip2location_country/ip2location_country_pk.netset",
+	    "sa":"https://iplists.firehol.org/files/ip2location_country/ip2location_country_sa.netset",
+	    "cn2":"https://iplists.firehol.org/files/ip2location_country/ip2location_country_cn.netset"
          }
    }
 }

lists_updater.py

@@ -1,11 +1,11 @@
 #!/usr/bin/python
 import os
 import json
-import pprint
 import requests
 import re
 import ipaddress
 
+
 print('Reading lists of URL to download')
 with open("lists.json","r") as f:
     data = json.load(f)
@@ -28,16 +28,22 @@ def is_ipv4(ip):
     else:
 	return False
 
+def is_ipv6(ip):
+    try:
+        ip_addr = ipaddress.IPv6Address(ip)
+    except ipaddress.AddressValueError:
+        return False
+    return ip_addr.version == 6
+
+def extract_ipv6(source):
+    return re.findall(regex_ipv6_cidr,source)
+
 def is_ipv6_subnet(ip):
     try:
         addr = ipaddress.IPv6Network(ip)
     except Exception as e:
         return False
     return True
-    #if re.match("^(((?=.*(::))(?!.*\3.+\3))\3?|[\dA-F]{1,4}:)([\dA-F]{1,4}(\3|:\b)|\2){5}(([\dA-F]{1,4}(\3|:\b|$)|\2){2}|(((2[0-4]|1\d|[1-9])?\d|25[0-5])\.?\b){4})\Z",ip):
-    #    return True
-    #else:
-    #    return False
 
 def download_list(list_name,list_url):
     print("- Downloading {}".format(list_name))
@@ -45,18 +51,19 @@ def download_list(list_name,list_url):
     try:
         filedata = requests.get(list_url).content.split('\n')
     except Exception as e:
-        print(e)
+        print("Error downloading {} : {}".format(list_name,e))
     return filedata
 
+
 def update_ipset_files():
     ipv4_list = []
     ipv6_list = []
     net_list = []
 
-    for elem in ['ipv4','ipv6','net']:
+    for elem in ['ipv4','net']:
        print("= Update {}".format(elem))
        elem_list = []
-       
+
        for key,url in data['lists'][elem].iteritems():
            dl_list = download_list(key,url)
            ipv4_tab = []
@@ -75,17 +82,16 @@ def update_ipset_files():
                if elem == "net":
                    if is_subnet(line):
                        subnet_tab.append("add blacklist_net {} -exist".format(line))
-                       
+
            if elem == "ipv4": ipv4_list.extend(ipv4_tab)
            if elem == "ipv6": ipv6_list.extend(ipv6_tab)
            if elem == "net": net_list.extend(subnet_tab)
     ipv4_list = sorted(set(ipv4_list))
     ipv6_list = sorted(set(ipv6_list))
     inet_list = sorted(set(net_list))
-    
+
     save_to_file(ipv4_list,"ipset_ipv4.set")
-    save_to_file(ipv6_list,"ipset_ipv6.set")
+    #save_to_file(ipv6_list,"ipset_ipv6.set")
     save_to_file(net_list,"ipset_subnets.set")
 
-
 update_ipset_files()