update
parent
58c8ecc682
commit
7dd9ec5fd5
|
@ -0,0 +1,7 @@
|
|||
= ipset updater
|
||||
|
||||
= install
|
||||
|
||||
* git clone this
|
||||
* launch ./install.sh
|
||||
* launch ./ipset_update.sh for first time
|
|
@ -0,0 +1,4 @@
|
|||
#!/bin/bash
|
||||
echo "30 12 * * * root /opt/ipset_updater/ipset_update.sh >> /var/log/blacklist_update.log" > /etc/cron.d/ipset_update
|
||||
chmod a+x /etc/cron.d/ipset_update
|
||||
systemctl restart cron.service
|
|
@ -1,21 +1,26 @@
|
|||
#!/bin/bash
|
||||
cd "$(dirname "$0")"
|
||||
echo "Runinng list update"
|
||||
date_good=`date +"%Y-%m-%d %T"`
|
||||
echo "* ${date_good} Runinng list update"
|
||||
python ./lists_updater.py
|
||||
|
||||
echo "Create ipset blocklists if not existing"
|
||||
echo "* Create ipset blocklists if not already existing"
|
||||
ipset create blacklist_net -exist hash:net family inet hashsize 16777216 maxelem 16777216
|
||||
ipset create blacklist_ipv4 -exist hash:ip family inet hashsize 16777216 maxelem 16777216
|
||||
ipset create blacklist_ipv6 -exist hash:net family inet hashsize 16777216 maxelem 16777216
|
||||
#ipset create blacklist_ipv6 -exist hash:net family inet hashsize 16777216 maxelem 16777216
|
||||
|
||||
echo "Import lists into ipset"
|
||||
echo "* Import lists into ipset"
|
||||
echo "== Import ipv4 ipset"
|
||||
ipset restore < ipset_ipv4.set
|
||||
#echo "== Import ipv6 ipset"
|
||||
#ipset restore < ipset_ipv6.set
|
||||
echo "== Import subnets ipset"
|
||||
ipset restore < ipset_subnets.set
|
||||
|
||||
echo "Saving ipset"
|
||||
echo "* Saving ipset"
|
||||
ipset save > /etc/ipset.conf
|
||||
|
||||
echo "* Cleanup ipset files"
|
||||
rm ./ipset_ipv4.set
|
||||
rm ./ipset_ipv6.set
|
||||
#rm ./ipset_ipv6.set
|
||||
rm ./ipset_subnets.set
|
||||
|
|
21
lists.json
21
lists.json
|
@ -3,12 +3,15 @@
|
|||
"ipv4":
|
||||
{
|
||||
"spamhaus":"https://www.spamhaus.org/drop/drop.txt",
|
||||
"blocklist":"https://lists.blocklist.de/lists/all.txt"
|
||||
},
|
||||
"ipv6":
|
||||
{
|
||||
"bogons":"https://www.team-cymru.org/Services/Bogons/fullbogons-ipv6.txt",
|
||||
"spamhaus":"https://www.spamhaus.org/drop/dropv6.txt"
|
||||
"blocklist":"https://lists.blocklist.de/lists/all.txt",
|
||||
"stopforumspam":"https://iplists.firehol.org/files/stopforumspam.ipset",
|
||||
"haley_ssh":"https://iplists.firehol.org/files/haley_ssh.ipset",
|
||||
"blocklist_ssh":"https://iplists.firehol.org/files/blocklist_de_ssh.ipset",
|
||||
"bi_any_0_1d":"https://iplists.firehol.org/files/bi_any_0_1d.ipset",
|
||||
"bi_any__1_7d":"https://iplists.firehol.org/files/bi_any_1_7d.ipset",
|
||||
"bi_any_2_1d":"https://iplists.firehol.org/files/bi_any_2_1d.ipset",
|
||||
"bi_any_2_30d":"https://iplists.firehol.org/files/bi_any_2_30d.ipset",
|
||||
"bi_any_2_7d":"https://iplists.firehol.org/files/bi_any_2_7d.ipset"
|
||||
},
|
||||
"net":
|
||||
{
|
||||
|
@ -16,7 +19,11 @@
|
|||
"bogons":"https://www.team-cymru.org/Services/Bogons/fullbogons-ipv4.txt",
|
||||
"firehol":"https://iplists.firehol.org/files/firehol_level1.netset",
|
||||
"kor":"https://www.okean.com/sinokoreacidr.txt",
|
||||
"cn":"https://www.okean.com/chinacidr.txt"
|
||||
"cn":"https://www.okean.com/chinacidr.txt",
|
||||
"ru":"https://iplists.firehol.org/files/ip2location_country/ip2location_country_ru.netset",
|
||||
"pk":"https://iplists.firehol.org/files/ip2location_country/ip2location_country_pk.netset",
|
||||
"sa":"https://iplists.firehol.org/files/ip2location_country/ip2location_country_sa.netset",
|
||||
"cn2":"https://iplists.firehol.org/files/ip2location_country/ip2location_country_cn.netset"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,11 +1,11 @@
|
|||
#!/usr/bin/python
|
||||
import os
|
||||
import json
|
||||
import pprint
|
||||
import requests
|
||||
import re
|
||||
import ipaddress
|
||||
|
||||
|
||||
print('Reading lists of URL to download')
|
||||
with open("lists.json","r") as f:
|
||||
data = json.load(f)
|
||||
|
@ -28,16 +28,22 @@ def is_ipv4(ip):
|
|||
else:
|
||||
return False
|
||||
|
||||
def is_ipv6(ip):
|
||||
try:
|
||||
ip_addr = ipaddress.IPv6Address(ip)
|
||||
except ipaddress.AddressValueError:
|
||||
return False
|
||||
return ip_addr.version == 6
|
||||
|
||||
def extract_ipv6(source):
|
||||
return re.findall(regex_ipv6_cidr,source)
|
||||
|
||||
def is_ipv6_subnet(ip):
|
||||
try:
|
||||
addr = ipaddress.IPv6Network(ip)
|
||||
except Exception as e:
|
||||
return False
|
||||
return True
|
||||
#if re.match("^(((?=.*(::))(?!.*\3.+\3))\3?|[\dA-F]{1,4}:)([\dA-F]{1,4}(\3|:\b)|\2){5}(([\dA-F]{1,4}(\3|:\b|$)|\2){2}|(((2[0-4]|1\d|[1-9])?\d|25[0-5])\.?\b){4})\Z",ip):
|
||||
# return True
|
||||
#else:
|
||||
# return False
|
||||
|
||||
def download_list(list_name,list_url):
|
||||
print("- Downloading {}".format(list_name))
|
||||
|
@ -45,18 +51,19 @@ def download_list(list_name,list_url):
|
|||
try:
|
||||
filedata = requests.get(list_url).content.split('\n')
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print("Error downloading {} : {}".format(list_name,e))
|
||||
return filedata
|
||||
|
||||
|
||||
def update_ipset_files():
|
||||
ipv4_list = []
|
||||
ipv6_list = []
|
||||
net_list = []
|
||||
|
||||
for elem in ['ipv4','ipv6','net']:
|
||||
for elem in ['ipv4','net']:
|
||||
print("= Update {}".format(elem))
|
||||
elem_list = []
|
||||
|
||||
|
||||
for key,url in data['lists'][elem].iteritems():
|
||||
dl_list = download_list(key,url)
|
||||
ipv4_tab = []
|
||||
|
@ -75,17 +82,16 @@ def update_ipset_files():
|
|||
if elem == "net":
|
||||
if is_subnet(line):
|
||||
subnet_tab.append("add blacklist_net {} -exist".format(line))
|
||||
|
||||
|
||||
if elem == "ipv4": ipv4_list.extend(ipv4_tab)
|
||||
if elem == "ipv6": ipv6_list.extend(ipv6_tab)
|
||||
if elem == "net": net_list.extend(subnet_tab)
|
||||
ipv4_list = sorted(set(ipv4_list))
|
||||
ipv6_list = sorted(set(ipv6_list))
|
||||
inet_list = sorted(set(net_list))
|
||||
|
||||
|
||||
save_to_file(ipv4_list,"ipset_ipv4.set")
|
||||
save_to_file(ipv6_list,"ipset_ipv6.set")
|
||||
#save_to_file(ipv6_list,"ipset_ipv6.set")
|
||||
save_to_file(net_list,"ipset_subnets.set")
|
||||
|
||||
|
||||
update_ipset_files()
|
||||
|
|
Loading…
Reference in New Issue