pavel.muhortov/my_route
1
0
Fork 0
generated from pavel.muhortov/template-python
Code Issues Pull Requests Projects Releases Wiki Activity

added wwwhmptoday source

Browse Source
This commit is contained in:
Pavel Muhortov 2023-08-31 09:53:56 +03:00
parent e0a9273307
commit 59a7e82f13
2 changed files with 413 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

344
my_route.conf Executable file
View File

@ -0,0 +1,344 @@
[common]
# By default, a database directory is created in the same path where the script is located.
# If you need change it, uncomment the parameter and set the path you want.
#db_root_path = /tmp/my_route.db
#
# By default, logs use the same directory where the script is located.
# If you need change it, uncomment the parameter and set the path you want.
#log_root = /var/log/my_route
#
# The default log level is "INFO".
# If you get errors or want to change the logging level, uncomment the parameter and set the level you want:
# DEBUG, INFO, WARNING, ERROR, CRITICAL.
#log_level = DEBUG
[enable-sources]
# google - it's a json (https://www.gstatic.com/ipranges/goog.json)
# google cloud subnets (https://www.gstatic.com/ipranges/cloud.json) included to goog.json
# + absoutely free
# + simple parsing
# - contain only google ip ranges
google = true
#
# amazon - it's a json (https://ip-ranges.amazonaws.com/ip-ranges.json)
# + absoutely free
# + simple parsing
# - contain only amazon ip ranges
amazon = true
#
# atlassian - it's a json (https://ip-ranges.atlassian.com)
# + absoutely free
# + simple parsing
# - contain only atlassian ip ranges
atlassian = true
#
# herrbischoff - it's a GitHub repository (https://github.com/herrbischoff/country-ip-blocks)
# + absoutely free
# + updated hourly
# - less accuracy
herrbischoff = true
#
# ip2location - it's a geolocation database (https://www.ip2location.com)
# + more accuracy
# - you need account
# - updated monthly
ip2location = true
ip2l_download_token = ip2location_TOKEN
#
# DB1LITECSV - free database
# + absoutely free
# - difficult parsing
# - accuracy only 1.2.3.4/24
#ip2l_database_code = DB1LITECSV
#
# DB1CIDR - commercial database
# + accuracy 1.2.3.4/32
# + simple parsing
# - access to updates only by paid subscription
ip2l_database_code = DB1CIDR
#
# wwwhmptoday - it's compilation from all other sources (https://git.hmp.today/pavel.muhortov/my_route.db)
# + all the benefits of other sources
# + all lists ready to use
# - you need account
wwwhmptoday = true
wwwhmptoday_user = username
wwwhmptoday_pass = password
[enable-gateway]
# List the gateway block names. Only blocks with the TRUE value will be used.
via-192.168.0.1 = true
dev-wg1 = true
[via-192.168.0.1]
# List of CIDR. Only CIDR with the TRUE value will be used.
# If different sources contain the same cidr, use the path part to define your preference.
ipv4/via-192.168.0.1.cidr = true
ipv4/atlassian.cidr = true
ip2location/cidr/ipv4/ru.cidr = true
ipv4/private.cidr
[dev-wg1]
# List of CIDR. Only CIDR with the TRUE value will be used.
# If different sources contain the same cidr, use the path part to define your preference.
ipv4/dev-wg1.cidr = true
ipv4/public.cidr = true
ip2location/cidr/ipv4/-.cidr
ipv4/ad.cidr
ipv4/ae.cidr
ipv4/af.cidr
ipv4/ag.cidr
ipv4/ai.cidr
ipv4/al.cidr
ipv4/am.cidr
ipv4/ao.cidr
herrbischoff/cidr/ipv4/ap.cidr
ipv4/aq.cidr
ipv4/ar.cidr
ipv4/as.cidr
ipv4/at.cidr
ipv4/au.cidr
ipv4/aw.cidr
ipv4/ax.cidr
ipv4/az.cidr
ipv4/ba.cidr
ipv4/bb.cidr
ipv4/bd.cidr
ipv4/be.cidr
ipv4/bf.cidr
ipv4/bg.cidr
ipv4/bh.cidr
ipv4/bi.cidr
ipv4/bj.cidr
ipv4/bl.cidr
ipv4/bm.cidr
ipv4/bn.cidr
ipv4/bo.cidr
ipv4/bq.cidr
ipv4/br.cidr
ipv4/bs.cidr
ipv4/bt.cidr
ip2location/cidr/ipv4/bv.cidr
ipv4/bw.cidr
ipv4/by.cidr
ipv4/bz.cidr
ipv4/ca.cidr
ip2location/cidr/ipv4/cc.cidr
ipv4/cd.cidr
ipv4/cf.cidr
ipv4/cg.cidr
ipv4/ch.cidr
ipv4/ci.cidr
ipv4/ck.cidr
ipv4/cl.cidr
ipv4/cm.cidr
ipv4/cn.cidr
ipv4/co.cidr
ipv4/cr.cidr
ipv4/cu.cidr
ipv4/cv.cidr
ipv4/cw.cidr
ip2location/cidr/ipv4/cx.cidr
ipv4/cy.cidr
ipv4/cz.cidr
ipv4/de.cidr
ipv4/dj.cidr
ipv4/dk.cidr
ipv4/dm.cidr
ipv4/do.cidr
ipv4/dz.cidr
ipv4/ec.cidr
ipv4/ee.cidr
ipv4/eg.cidr
ip2location/cidr/ipv4/eh.cidr
ipv4/er.cidr
ipv4/es.cidr
ipv4/et.cidr
ipv4/fi.cidr
ipv4/fj.cidr
ipv4/fk.cidr
ipv4/fm.cidr
ipv4/fo.cidr
ipv4/fr.cidr
ipv4/ga.cidr
ipv4/gb.cidr
ipv4/gd.cidr
ipv4/ge.cidr
ipv4/gf.cidr
ipv4/gg.cidr
ipv4/gh.cidr
ipv4/gi.cidr
ipv4/gl.cidr
ipv4/gm.cidr
ipv4/gn.cidr
ipv4/gp.cidr
ipv4/gq.cidr
ipv4/gr.cidr
ip2location/cidr/ipv4/gs.cidr
ipv4/gt.cidr
ipv4/gu.cidr
ipv4/gw.cidr
ipv4/gy.cidr
ipv4/hk.cidr
ip2location/cidr/ipv4/hm.cidr
ipv4/hn.cidr
ipv4/hr.cidr
ipv4/ht.cidr
ipv4/hu.cidr
ipv4/id.cidr
ipv4/ie.cidr
ipv4/il.cidr
ipv4/im.cidr
ipv4/in.cidr
ipv4/io.cidr
ipv4/iq.cidr
ipv4/ir.cidr
ipv4/is.cidr
ipv4/it.cidr
ipv4/je.cidr
ipv4/jm.cidr
ipv4/jo.cidr
ipv4/jp.cidr
ipv4/ke.cidr
ipv4/kg.cidr
ipv4/kh.cidr
ipv4/ki.cidr
ipv4/km.cidr
ipv4/kn.cidr
ipv4/kp.cidr
ipv4/kr.cidr
ipv4/kw.cidr
ipv4/ky.cidr
ipv4/kz.cidr
ipv4/la.cidr
ipv4/lb.cidr
ipv4/lc.cidr
ipv4/li.cidr
ipv4/lk.cidr
ipv4/lr.cidr
ipv4/ls.cidr
ipv4/lt.cidr
ipv4/lu.cidr
ipv4/lv.cidr
ipv4/ly.cidr
ipv4/ma.cidr
ipv4/mc.cidr
ipv4/md.cidr
ipv4/me.cidr
ipv4/mf.cidr
ipv4/mg.cidr
ipv4/mh.cidr
ipv4/mk.cidr
ipv4/ml.cidr
ipv4/mm.cidr
ipv4/mn.cidr
ipv4/mo.cidr
ipv4/mp.cidr
ipv4/mq.cidr
ipv4/mr.cidr
ipv4/ms.cidr
ipv4/mt.cidr
ipv4/mu.cidr
ipv4/mv.cidr
ipv4/mw.cidr
ipv4/mx.cidr
ipv4/my.cidr
ipv4/mz.cidr
ipv4/na.cidr
ipv4/nc.cidr
ipv4/ne.cidr
ipv4/nf.cidr
ipv4/ng.cidr
ipv4/ni.cidr
ipv4/nl.cidr
ipv4/no.cidr
ipv4/np.cidr
ipv4/nr.cidr
ipv4/nu.cidr
ipv4/nz.cidr
ipv4/om.cidr
ipv4/pa.cidr
ipv4/pe.cidr
ipv4/pf.cidr
ipv4/pg.cidr
ipv4/ph.cidr
ipv4/pk.cidr
ipv4/pl.cidr
ipv4/pm.cidr
ip2location/cidr/ipv4/pn.cidr
ipv4/pr.cidr
ipv4/ps.cidr
ipv4/pt.cidr
ipv4/pw.cidr
ipv4/py.cidr
ipv4/qa.cidr
ipv4/re.cidr
ipv4/ro.cidr
ipv4/rs.cidr
ipv4/ru.cidr
ipv4/rw.cidr
ipv4/sa.cidr
ipv4/sb.cidr
ipv4/sc.cidr
ipv4/sd.cidr
ipv4/se.cidr
ipv4/sg.cidr
ip2location/cidr/ipv4/sh.cidr
ipv4/si.cidr
ip2location/cidr/ipv4/sj.cidr
ipv4/sk.cidr
ipv4/sl.cidr
ipv4/sm.cidr
ipv4/sn.cidr
ipv4/so.cidr
ipv4/sr.cidr
ipv4/ss.cidr
ipv4/st.cidr
ipv4/sv.cidr
ipv4/sx.cidr
ipv4/sy.cidr
ipv4/sz.cidr
ipv4/tc.cidr
ipv4/td.cidr
ip2location/cidr/ipv4/tf.cidr
ipv4/tg.cidr
ipv4/th.cidr
ipv4/tj.cidr
ipv4/tk.cidr
ipv4/tl.cidr
ipv4/tm.cidr
ipv4/tn.cidr
ipv4/to.cidr
ipv4/tr.cidr
ipv4/tt.cidr
ipv4/tv.cidr
ipv4/tw.cidr
ipv4/tz.cidr
ipv4/ua.cidr
ipv4/ug.cidr
ip2location/cidr/ipv4/um.cidr
ipv4/us.cidr
ipv4/uy.cidr
ipv4/uz.cidr
ipv4/va.cidr
ipv4/vc.cidr
ipv4/ve.cidr
ipv4/vg.cidr
ipv4/vi.cidr
ipv4/vn.cidr
ipv4/vu.cidr
ipv4/wf.cidr
ipv4/ws.cidr
ipv4/ye.cidr
ipv4/yt.cidr
ipv4/za.cidr
ipv4/zm.cidr
ipv4/zw.cidr
herrbischoff/cidr/ipv4/zz.cidr

73
my_route.py
View File

@ -14,7 +14,7 @@ import logging
import urllib.request import urllib.request
from argparse import ArgumentParser from argparse import ArgumentParser
from datetime import datetime from datetime import datetime
from os import path, sep, makedirs, remove, rmdir, walk from os import path, sep, makedirs, remove, rmdir, system, walk
from shutil import copyfile from shutil import copyfile
from sys import platform from sys import platform
from subprocess import Popen, PIPE from subprocess import Popen, PIPE
@ -393,6 +393,8 @@ class Route(Connect):
db_root_path: str, db_root_path: str,
db_source_code: (str, type(None)) = None, db_source_code: (str, type(None)) = None,
download_token: (str, type(None)) = None, download_token: (str, type(None)) = None,
download_user: (str, type(None)) = None,
download_pass: (str, type(None)) = None,
force_download: bool = False, force_download: bool = False,
logger_alias: str = inspect.stack()[0].function logger_alias: str = inspect.stack()[0].function
) -> bool: ) -> bool:
@ -444,6 +446,14 @@ class Route(Connect):
force_download=force_download, force_download=force_download,
logger_alias=logger_alias logger_alias=logger_alias
) )
elif name == 'wwwhmptoday':
return self.__update_source_wwwhmptoday(
db_root_path=db_root_path,
download_user=download_user,
download_pass=download_pass,
force_download=force_download,
logger_alias=logger_alias
)
def __download_db( def __download_db(
self, self,
@ -836,6 +846,49 @@ class Route(Connect):
remove(db_source_file) remove(db_source_file)
return False return False
def __update_source_wwwhmptoday(
self,
db_root_path: str,
download_user: str,
download_pass: str,
force_download: bool = False,
logger_alias: str = inspect.stack()[0].function
) -> bool:
local_logger = logging.getLogger(logger_alias)
if Do.args_valid(locals(), self.__update_source_wwwhmptoday.__annotations__):
if not path.exists(db_root_path + sep + 'sources'):
force_download = True
if force_download:
db_temp_path = path.dirname(db_root_path) + sep + 'tmp'
cmd_gitclone = (''
+ "git clone " + "https://" + download_user + ":" + download_pass
+ "@git.hmp.today/pavel.muhortov/my_route.db.git " + db_temp_path
)
ret_gitclone = system(cmd_gitclone)
if ret_gitclone == 0:
try:
makedirs(db_root_path, exist_ok=True)
for root, dirs, files in walk(db_temp_path, topdown=False):
for file in files:
src_file = path.join(path.realpath(root), file)
dst_file = src_file.replace(db_temp_path, db_root_path)
makedirs(path.dirname(dst_file), exist_ok=True)
copyfile(src=src_file, dst=dst_file)
remove(path.join(path.realpath(root), file))
for directory in dirs:
rmdir(path.join(path.realpath(root), directory))
rmdir(db_temp_path)
return True
except OSError as error:
local_logger.debug(msg='error: ' + '\n' + str(error))
else:
local_logger.warning(msg=''
+ 'git clone returned '+ str(ret_gitclone) + ' code. '
+ 'Restart by interactive and check stdout.'
)
return False
class Do(): class Do():
"""Set of various methods (functions) for routine. """Set of various methods (functions) for routine.
@ -894,7 +947,8 @@ if __name__ == "__main__":
description='Route management by CIDR lists.', description='Route management by CIDR lists.',
epilog='Dependencies: ' epilog='Dependencies: '
'- Python 3 (tested version 3.9.5), ' '- Python 3 (tested version 3.9.5), '
'- privileged rights ' '- privileged rights, '
'- git '
) )
args.add_argument('--config', type=str, args.add_argument('--config', type=str,
default=path.splitext(__file__)[0] + '.conf', default=path.splitext(__file__)[0] + '.conf',
@ -959,7 +1013,9 @@ if __name__ == "__main__":
enable_sources[key] = { enable_sources[key] = {
'enable': value, 'enable': value,
'download_token': None, 'download_token': None,
'db_source_code': None 'db_source_code': None,
'download_user': None,
'download_pass': None
} }
if 'ip2location' in enable_sources: if 'ip2location' in enable_sources:
enable_sources['ip2location']['download_token'] = ( enable_sources['ip2location']['download_token'] = (
@ -968,6 +1024,13 @@ if __name__ == "__main__":
enable_sources['ip2location']['db_source_code'] = ( enable_sources['ip2location']['db_source_code'] = (
conf_sources.data['ip2l_database_code'] conf_sources.data['ip2l_database_code']
) )
if 'wwwhmptoday' in enable_sources:
enable_sources['wwwhmptoday']['download_user'] = (
conf_sources.data['wwwhmptoday_user']
)
enable_sources['wwwhmptoday']['download_pass'] = (
conf_sources.data['wwwhmptoday_pass']
)
logging.basicConfig( logging.basicConfig(
format='%(asctime)s %(levelname)s: %(name)s: %(message)s', format='%(asctime)s %(levelname)s: %(name)s: %(message)s',
@ -983,7 +1046,7 @@ if __name__ == "__main__":
) )
if Do.checkroot(): if not Do.checkroot():
ro = Route(gateways=enable_gateway, db_root_path=db_root_path) ro = Route(gateways=enable_gateway, db_root_path=db_root_path)
if args['update']: if args['update']:
@ -993,6 +1056,8 @@ if __name__ == "__main__":
db_root_path=db_root_path, db_root_path=db_root_path,
db_source_code=enable_sources[key]['db_source_code'], db_source_code=enable_sources[key]['db_source_code'],
download_token=enable_sources[key]['download_token'], download_token=enable_sources[key]['download_token'],
download_user=enable_sources[key]['download_user'],
download_pass=enable_sources[key]['download_pass'],
force_download=args['force'], force_download=args['force'],
logger_alias='update sources ' + key logger_alias='update sources ' + key
) )