ipset

ipset_update.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+cd "$(dirname "$0")"
+echo "Runinng list update"
+python ./lists_updater.py
+
+echo "Create ipset blocklists if not existing"
+ipset create blacklist_net -exist hash:net family inet hashsize 16777216 maxelem 16777216 
+ipset create blacklist_ipv4 -exist hash:ip family inet hashsize 16777216 maxelem 16777216 
+ipset create blacklist_ipv6 -exist hash:net family inet hashsize 16777216 maxelem 16777216
+
+echo "Import lists into ipset"
+ipset restore < ipset_ipv4.set
+#ipset restore < ipset_ipv6.set
+ipset restore < ipset_subnets.set
+
+echo "Saving ipset"
+ipset save > /etc/ipset.conf
+
+rm ./ipset_ipv4.set
+rm ./ipset_ipv6.set
+rm ./ipset_subnets.set

lists.json

@@ -0,0 +1,22 @@
+{
+   "lists":{
+      "ipv4":
+         {
+            "spamhaus":"https://www.spamhaus.org/drop/drop.txt",
+            "blocklist":"https://lists.blocklist.de/lists/all.txt"
+         },
+      "ipv6":
+         {
+            "bogons":"https://www.team-cymru.org/Services/Bogons/fullbogons-ipv6.txt",
+            "spamhaus":"https://www.spamhaus.org/drop/dropv6.txt"
+         },
+      "net":
+         {
+            "emerging":"https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt",
+            "bogons":"https://www.team-cymru.org/Services/Bogons/fullbogons-ipv4.txt",
+            "firehol":"https://iplists.firehol.org/files/firehol_level1.netset",
+            "kor":"https://www.okean.com/sinokoreacidr.txt",
+            "cn":"https://www.okean.com/chinacidr.txt"
+         }
+   }
+}

lists_updater.py

@@ -0,0 +1,91 @@
+#!/usr/bin/python
+import os
+import json
+import pprint
+import requests
+import re
+import ipaddress
+
+print('Reading lists of URL to download')
+with open("lists.json","r") as f:
+    data = json.load(f)
+
+
+def save_to_file(text,filename):
+    print("= Saved {} elements to {}".format(len(text),filename))
+    with open('./{}'.format(filename), mode='w') as myfile:
+        myfile.write('\n'.join(text))
+
+def is_subnet(subnet):
+    if re.match('^([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?$',subnet):
+        return True
+    else:
+        return False
+
+def is_ipv4(ip):
+    if re.match("^(\d{0,3})\.(\d{0,3})\.(\d{0,3})\.(\d{0,3})$", ip):
+	return True
+    else:
+	return False
+
+def is_ipv6_subnet(ip):
+    try:
+        addr = ipaddress.IPv6Network(ip)
+    except Exception as e:
+        return False
+    return True
+    #if re.match("^(((?=.*(::))(?!.*\3.+\3))\3?|[\dA-F]{1,4}:)([\dA-F]{1,4}(\3|:\b)|\2){5}(([\dA-F]{1,4}(\3|:\b|$)|\2){2}|(((2[0-4]|1\d|[1-9])?\d|25[0-5])\.?\b){4})\Z",ip):
+    #    return True
+    #else:
+    #    return False
+
+def download_list(list_name,list_url):
+    print("- Downloading {}".format(list_name))
+    filedata = ""
+    try:
+        filedata = requests.get(list_url).content.split('\n')
+    except Exception as e:
+        print(e)
+    return filedata
+
+def update_ipset_files():
+    ipv4_list = []
+    ipv6_list = []
+    net_list = []
+
+    for elem in ['ipv4','ipv6','net']:
+       print("= Update {}".format(elem))
+       elem_list = []
+       
+       for key,url in data['lists'][elem].iteritems():
+           dl_list = download_list(key,url)
+           ipv4_tab = []
+           ipv6_tab = []
+           subnet_tab = []
+
+
+           for line in dl_list:
+
+               if elem == "ipv4":
+                   if is_ipv4(line):
+                       ipv4_tab.append("add blacklist_ipv4 {} -exist".format(line))
+               if elem == "ipv6":
+                   if is_ipv6_subnet(line):
+                       ipv6_tab.append("add blacklist_ipv6 {} -exist".format(line))
+               if elem == "net":
+                   if is_subnet(line):
+                       subnet_tab.append("add blacklist_net {} -exist".format(line))
+                       
+           if elem == "ipv4": ipv4_list.extend(ipv4_tab)
+           if elem == "ipv6": ipv6_list.extend(ipv6_tab)
+           if elem == "net": net_list.extend(subnet_tab)
+    ipv4_list = sorted(set(ipv4_list))
+    ipv6_list = sorted(set(ipv6_list))
+    inet_list = sorted(set(net_list))
+    
+    save_to_file(ipv4_list,"ipset_ipv4.set")
+    save_to_file(ipv6_list,"ipset_ipv6.set")
+    save_to_file(net_list,"ipset_subnets.set")
+
+
+update_ipset_files()