98 lines
2.6 KiB
Python
98 lines
2.6 KiB
Python
#!/usr/bin/python
|
|
import os
|
|
import json
|
|
import requests
|
|
import re
|
|
import ipaddress
|
|
|
|
|
|
print('Reading lists of URL to download')
|
|
with open("lists.json","r") as f:
|
|
data = json.load(f)
|
|
|
|
|
|
def save_to_file(text,filename):
|
|
print("= Saved {} elements to {}".format(len(text),filename))
|
|
with open('./{}'.format(filename), mode='w') as myfile:
|
|
myfile.write('\n'.join(text))
|
|
|
|
def is_subnet(subnet):
|
|
if re.match('^([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?$',subnet):
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def is_ipv4(ip):
|
|
if re.match("^(\d{0,3})\.(\d{0,3})\.(\d{0,3})\.(\d{0,3})$", ip):
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def is_ipv6(ip):
|
|
try:
|
|
ip_addr = ipaddress.IPv6Address(ip)
|
|
except ipaddress.AddressValueError:
|
|
return False
|
|
return ip_addr.version == 6
|
|
|
|
def extract_ipv6(source):
|
|
return re.findall(regex_ipv6_cidr,source)
|
|
|
|
def is_ipv6_subnet(ip):
|
|
try:
|
|
addr = ipaddress.IPv6Network(ip)
|
|
except Exception as e:
|
|
return False
|
|
return True
|
|
|
|
def download_list(list_name,list_url):
|
|
print("- Downloading {}".format(list_name))
|
|
filedata = ""
|
|
try:
|
|
filedata = requests.get(list_url).content.split('\n')
|
|
except Exception as e:
|
|
print("Error downloading {} : {}".format(list_name,e))
|
|
return filedata
|
|
|
|
|
|
def update_ipset_files():
|
|
ipv4_list = []
|
|
ipv6_list = []
|
|
net_list = []
|
|
|
|
for elem in ['ipv4','net']:
|
|
print("= Update {}".format(elem))
|
|
elem_list = []
|
|
|
|
for key,url in data['lists'][elem].iteritems():
|
|
dl_list = download_list(key,url)
|
|
ipv4_tab = []
|
|
ipv6_tab = []
|
|
subnet_tab = []
|
|
|
|
|
|
for line in dl_list:
|
|
|
|
if elem == "ipv4":
|
|
if is_ipv4(line):
|
|
ipv4_tab.append("add blacklist_ipv4 {} -exist".format(line))
|
|
if elem == "ipv6":
|
|
if is_ipv6_subnet(line):
|
|
ipv6_tab.append("add blacklist_ipv6 {} -exist".format(line))
|
|
if elem == "net":
|
|
if is_subnet(line):
|
|
subnet_tab.append("add blacklist_net {} -exist".format(line))
|
|
|
|
if elem == "ipv4": ipv4_list.extend(ipv4_tab)
|
|
if elem == "ipv6": ipv6_list.extend(ipv6_tab)
|
|
if elem == "net": net_list.extend(subnet_tab)
|
|
ipv4_list = sorted(set(ipv4_list))
|
|
ipv6_list = sorted(set(ipv6_list))
|
|
inet_list = sorted(set(net_list))
|
|
|
|
save_to_file(ipv4_list,"ipset_ipv4.set")
|
|
#save_to_file(ipv6_list,"ipset_ipv6.set")
|
|
save_to_file(net_list,"ipset_subnets.set")
|
|
|
|
update_ipset_files()
|