From 59a7e82f13727e9fccdefb85846c7bf24755ee55 Mon Sep 17 00:00:00 2001 From: Pavel Muhortov Date: Thu, 31 Aug 2023 09:53:56 +0300 Subject: [PATCH] added wwwhmptoday source --- my_route.conf | 344 ++++++++++++++++++++++++++++++++++++++++++++++++++ my_route.py | 73 ++++++++++- 2 files changed, 413 insertions(+), 4 deletions(-) create mode 100755 my_route.conf diff --git a/my_route.conf b/my_route.conf new file mode 100755 index 0000000..6a029fa --- /dev/null +++ b/my_route.conf @@ -0,0 +1,344 @@ +[common] +# By default, a database directory is created in the same path where the script is located. +# If you need change it, uncomment the parameter and set the path you want. +#db_root_path = /tmp/my_route.db +# +# By default, logs use the same directory where the script is located. +# If you need change it, uncomment the parameter and set the path you want. +#log_root = /var/log/my_route +# +# The default log level is "INFO". +# If you get errors or want to change the logging level, uncomment the parameter and set the level you want: +# DEBUG, INFO, WARNING, ERROR, CRITICAL. +#log_level = DEBUG + + + +[enable-sources] +# google - it's a json (https://www.gstatic.com/ipranges/goog.json) +# google cloud subnets (https://www.gstatic.com/ipranges/cloud.json) included to goog.json +# + absoutely free +# + simple parsing +# - contain only google ip ranges +google = true +# +# amazon - it's a json (https://ip-ranges.amazonaws.com/ip-ranges.json) +# + absoutely free +# + simple parsing +# - contain only amazon ip ranges +amazon = true +# +# atlassian - it's a json (https://ip-ranges.atlassian.com) +# + absoutely free +# + simple parsing +# - contain only atlassian ip ranges +atlassian = true +# +# herrbischoff - it's a GitHub repository (https://github.com/herrbischoff/country-ip-blocks) +# + absoutely free +# + updated hourly +# - less accuracy +herrbischoff = true +# +# ip2location - it's a geolocation database (https://www.ip2location.com) +# + more accuracy +# - you need account +# - updated monthly +ip2location = true +ip2l_download_token = ip2location_TOKEN +# +# DB1LITECSV - free database +# + absoutely free +# - difficult parsing +# - accuracy only 1.2.3.4/24 +#ip2l_database_code = DB1LITECSV +# +# DB1CIDR - commercial database +# + accuracy 1.2.3.4/32 +# + simple parsing +# - access to updates only by paid subscription +ip2l_database_code = DB1CIDR +# +# wwwhmptoday - it's compilation from all other sources (https://git.hmp.today/pavel.muhortov/my_route.db) +# + all the benefits of other sources +# + all lists ready to use +# - you need account +wwwhmptoday = true +wwwhmptoday_user = username +wwwhmptoday_pass = password + + +[enable-gateway] +# List the gateway block names. Only blocks with the TRUE value will be used. +via-192.168.0.1 = true +dev-wg1 = true + + + +[via-192.168.0.1] +# List of CIDR. Only CIDR with the TRUE value will be used. +# If different sources contain the same cidr, use the path part to define your preference. +ipv4/via-192.168.0.1.cidr = true +ipv4/atlassian.cidr = true +ip2location/cidr/ipv4/ru.cidr = true +ipv4/private.cidr + + + +[dev-wg1] +# List of CIDR. Only CIDR with the TRUE value will be used. +# If different sources contain the same cidr, use the path part to define your preference. +ipv4/dev-wg1.cidr = true +ipv4/public.cidr = true +ip2location/cidr/ipv4/-.cidr +ipv4/ad.cidr +ipv4/ae.cidr +ipv4/af.cidr +ipv4/ag.cidr +ipv4/ai.cidr +ipv4/al.cidr +ipv4/am.cidr +ipv4/ao.cidr +herrbischoff/cidr/ipv4/ap.cidr +ipv4/aq.cidr +ipv4/ar.cidr +ipv4/as.cidr +ipv4/at.cidr +ipv4/au.cidr +ipv4/aw.cidr +ipv4/ax.cidr +ipv4/az.cidr +ipv4/ba.cidr +ipv4/bb.cidr +ipv4/bd.cidr +ipv4/be.cidr +ipv4/bf.cidr +ipv4/bg.cidr +ipv4/bh.cidr +ipv4/bi.cidr +ipv4/bj.cidr +ipv4/bl.cidr +ipv4/bm.cidr +ipv4/bn.cidr +ipv4/bo.cidr +ipv4/bq.cidr +ipv4/br.cidr +ipv4/bs.cidr +ipv4/bt.cidr +ip2location/cidr/ipv4/bv.cidr +ipv4/bw.cidr +ipv4/by.cidr +ipv4/bz.cidr +ipv4/ca.cidr +ip2location/cidr/ipv4/cc.cidr +ipv4/cd.cidr +ipv4/cf.cidr +ipv4/cg.cidr +ipv4/ch.cidr +ipv4/ci.cidr +ipv4/ck.cidr +ipv4/cl.cidr +ipv4/cm.cidr +ipv4/cn.cidr +ipv4/co.cidr +ipv4/cr.cidr +ipv4/cu.cidr +ipv4/cv.cidr +ipv4/cw.cidr +ip2location/cidr/ipv4/cx.cidr +ipv4/cy.cidr +ipv4/cz.cidr +ipv4/de.cidr +ipv4/dj.cidr +ipv4/dk.cidr +ipv4/dm.cidr +ipv4/do.cidr +ipv4/dz.cidr +ipv4/ec.cidr +ipv4/ee.cidr +ipv4/eg.cidr +ip2location/cidr/ipv4/eh.cidr +ipv4/er.cidr +ipv4/es.cidr +ipv4/et.cidr +ipv4/fi.cidr +ipv4/fj.cidr +ipv4/fk.cidr +ipv4/fm.cidr +ipv4/fo.cidr +ipv4/fr.cidr +ipv4/ga.cidr +ipv4/gb.cidr +ipv4/gd.cidr +ipv4/ge.cidr +ipv4/gf.cidr +ipv4/gg.cidr +ipv4/gh.cidr +ipv4/gi.cidr +ipv4/gl.cidr +ipv4/gm.cidr +ipv4/gn.cidr +ipv4/gp.cidr +ipv4/gq.cidr +ipv4/gr.cidr +ip2location/cidr/ipv4/gs.cidr +ipv4/gt.cidr +ipv4/gu.cidr +ipv4/gw.cidr +ipv4/gy.cidr +ipv4/hk.cidr +ip2location/cidr/ipv4/hm.cidr +ipv4/hn.cidr +ipv4/hr.cidr +ipv4/ht.cidr +ipv4/hu.cidr +ipv4/id.cidr +ipv4/ie.cidr +ipv4/il.cidr +ipv4/im.cidr +ipv4/in.cidr +ipv4/io.cidr +ipv4/iq.cidr +ipv4/ir.cidr +ipv4/is.cidr +ipv4/it.cidr +ipv4/je.cidr +ipv4/jm.cidr +ipv4/jo.cidr +ipv4/jp.cidr +ipv4/ke.cidr +ipv4/kg.cidr +ipv4/kh.cidr +ipv4/ki.cidr +ipv4/km.cidr +ipv4/kn.cidr +ipv4/kp.cidr +ipv4/kr.cidr +ipv4/kw.cidr +ipv4/ky.cidr +ipv4/kz.cidr +ipv4/la.cidr +ipv4/lb.cidr +ipv4/lc.cidr +ipv4/li.cidr +ipv4/lk.cidr +ipv4/lr.cidr +ipv4/ls.cidr +ipv4/lt.cidr +ipv4/lu.cidr +ipv4/lv.cidr +ipv4/ly.cidr +ipv4/ma.cidr +ipv4/mc.cidr +ipv4/md.cidr +ipv4/me.cidr +ipv4/mf.cidr +ipv4/mg.cidr +ipv4/mh.cidr +ipv4/mk.cidr +ipv4/ml.cidr +ipv4/mm.cidr +ipv4/mn.cidr +ipv4/mo.cidr +ipv4/mp.cidr +ipv4/mq.cidr +ipv4/mr.cidr +ipv4/ms.cidr +ipv4/mt.cidr +ipv4/mu.cidr +ipv4/mv.cidr +ipv4/mw.cidr +ipv4/mx.cidr +ipv4/my.cidr +ipv4/mz.cidr +ipv4/na.cidr +ipv4/nc.cidr +ipv4/ne.cidr +ipv4/nf.cidr +ipv4/ng.cidr +ipv4/ni.cidr +ipv4/nl.cidr +ipv4/no.cidr +ipv4/np.cidr +ipv4/nr.cidr +ipv4/nu.cidr +ipv4/nz.cidr +ipv4/om.cidr +ipv4/pa.cidr +ipv4/pe.cidr +ipv4/pf.cidr +ipv4/pg.cidr +ipv4/ph.cidr +ipv4/pk.cidr +ipv4/pl.cidr +ipv4/pm.cidr +ip2location/cidr/ipv4/pn.cidr +ipv4/pr.cidr +ipv4/ps.cidr +ipv4/pt.cidr +ipv4/pw.cidr +ipv4/py.cidr +ipv4/qa.cidr +ipv4/re.cidr +ipv4/ro.cidr +ipv4/rs.cidr +ipv4/ru.cidr +ipv4/rw.cidr +ipv4/sa.cidr +ipv4/sb.cidr +ipv4/sc.cidr +ipv4/sd.cidr +ipv4/se.cidr +ipv4/sg.cidr +ip2location/cidr/ipv4/sh.cidr +ipv4/si.cidr +ip2location/cidr/ipv4/sj.cidr +ipv4/sk.cidr +ipv4/sl.cidr +ipv4/sm.cidr +ipv4/sn.cidr +ipv4/so.cidr +ipv4/sr.cidr +ipv4/ss.cidr +ipv4/st.cidr +ipv4/sv.cidr +ipv4/sx.cidr +ipv4/sy.cidr +ipv4/sz.cidr +ipv4/tc.cidr +ipv4/td.cidr +ip2location/cidr/ipv4/tf.cidr +ipv4/tg.cidr +ipv4/th.cidr +ipv4/tj.cidr +ipv4/tk.cidr +ipv4/tl.cidr +ipv4/tm.cidr +ipv4/tn.cidr +ipv4/to.cidr +ipv4/tr.cidr +ipv4/tt.cidr +ipv4/tv.cidr +ipv4/tw.cidr +ipv4/tz.cidr +ipv4/ua.cidr +ipv4/ug.cidr +ip2location/cidr/ipv4/um.cidr +ipv4/us.cidr +ipv4/uy.cidr +ipv4/uz.cidr +ipv4/va.cidr +ipv4/vc.cidr +ipv4/ve.cidr +ipv4/vg.cidr +ipv4/vi.cidr +ipv4/vn.cidr +ipv4/vu.cidr +ipv4/wf.cidr +ipv4/ws.cidr +ipv4/ye.cidr +ipv4/yt.cidr +ipv4/za.cidr +ipv4/zm.cidr +ipv4/zw.cidr +herrbischoff/cidr/ipv4/zz.cidr \ No newline at end of file diff --git a/my_route.py b/my_route.py index 966b5b0..0295d76 100755 --- a/my_route.py +++ b/my_route.py @@ -14,7 +14,7 @@ import logging import urllib.request from argparse import ArgumentParser from datetime import datetime -from os import path, sep, makedirs, remove, rmdir, walk +from os import path, sep, makedirs, remove, rmdir, system, walk from shutil import copyfile from sys import platform from subprocess import Popen, PIPE @@ -393,6 +393,8 @@ class Route(Connect): db_root_path: str, db_source_code: (str, type(None)) = None, download_token: (str, type(None)) = None, + download_user: (str, type(None)) = None, + download_pass: (str, type(None)) = None, force_download: bool = False, logger_alias: str = inspect.stack()[0].function ) -> bool: @@ -444,6 +446,14 @@ class Route(Connect): force_download=force_download, logger_alias=logger_alias ) + elif name == 'wwwhmptoday': + return self.__update_source_wwwhmptoday( + db_root_path=db_root_path, + download_user=download_user, + download_pass=download_pass, + force_download=force_download, + logger_alias=logger_alias + ) def __download_db( self, @@ -836,6 +846,49 @@ class Route(Connect): remove(db_source_file) return False + def __update_source_wwwhmptoday( + self, + db_root_path: str, + download_user: str, + download_pass: str, + force_download: bool = False, + logger_alias: str = inspect.stack()[0].function + ) -> bool: + local_logger = logging.getLogger(logger_alias) + if Do.args_valid(locals(), self.__update_source_wwwhmptoday.__annotations__): + if not path.exists(db_root_path + sep + 'sources'): + force_download = True + if force_download: + db_temp_path = path.dirname(db_root_path) + sep + 'tmp' + cmd_gitclone = ('' + + "git clone " + "https://" + download_user + ":" + download_pass + + "@git.hmp.today/pavel.muhortov/my_route.db.git " + db_temp_path + ) + ret_gitclone = system(cmd_gitclone) + + if ret_gitclone == 0: + try: + makedirs(db_root_path, exist_ok=True) + for root, dirs, files in walk(db_temp_path, topdown=False): + for file in files: + src_file = path.join(path.realpath(root), file) + dst_file = src_file.replace(db_temp_path, db_root_path) + makedirs(path.dirname(dst_file), exist_ok=True) + copyfile(src=src_file, dst=dst_file) + remove(path.join(path.realpath(root), file)) + for directory in dirs: + rmdir(path.join(path.realpath(root), directory)) + rmdir(db_temp_path) + return True + except OSError as error: + local_logger.debug(msg='error: ' + '\n' + str(error)) + else: + local_logger.warning(msg='' + + 'git clone returned '+ str(ret_gitclone) + ' code. ' + + 'Restart by interactive and check stdout.' + ) + return False + class Do(): """Set of various methods (functions) for routine. @@ -894,7 +947,8 @@ if __name__ == "__main__": description='Route management by CIDR lists.', epilog='Dependencies: ' '- Python 3 (tested version 3.9.5), ' - '- privileged rights ' + '- privileged rights, ' + '- git ' ) args.add_argument('--config', type=str, default=path.splitext(__file__)[0] + '.conf', @@ -959,7 +1013,9 @@ if __name__ == "__main__": enable_sources[key] = { 'enable': value, 'download_token': None, - 'db_source_code': None + 'db_source_code': None, + 'download_user': None, + 'download_pass': None } if 'ip2location' in enable_sources: enable_sources['ip2location']['download_token'] = ( @@ -968,6 +1024,13 @@ if __name__ == "__main__": enable_sources['ip2location']['db_source_code'] = ( conf_sources.data['ip2l_database_code'] ) + if 'wwwhmptoday' in enable_sources: + enable_sources['wwwhmptoday']['download_user'] = ( + conf_sources.data['wwwhmptoday_user'] + ) + enable_sources['wwwhmptoday']['download_pass'] = ( + conf_sources.data['wwwhmptoday_pass'] + ) logging.basicConfig( format='%(asctime)s %(levelname)s: %(name)s: %(message)s', @@ -983,7 +1046,7 @@ if __name__ == "__main__": ) - if Do.checkroot(): + if not Do.checkroot(): ro = Route(gateways=enable_gateway, db_root_path=db_root_path) if args['update']: @@ -993,6 +1056,8 @@ if __name__ == "__main__": db_root_path=db_root_path, db_source_code=enable_sources[key]['db_source_code'], download_token=enable_sources[key]['download_token'], + download_user=enable_sources[key]['download_user'], + download_pass=enable_sources[key]['download_pass'], force_download=args['force'], logger_alias='update sources ' + key )