Merge branch 'master' of https://git.hmp.today/pavel.muhortov/my_route

.gitignore

@@ -1,165 +1,7 @@
-# ---> Python
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
 # Distribution / packaging / credential / logs
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
 my_route.db/
-parts/
-sdist/
 test/
-tmp/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-*.conf
 *.log
 
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-#  JetBrains specific template is maintainted in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
-
-# IntelliJ related files
-.idea
-*.iml
-
 # VSCode related files
-.vscode/*
+.vscode/

README.md

@@ -4,7 +4,6 @@ Route manager by CIDR lists
 
 * [`my_route`.py](https://git.hmp.today/pavel.muhortov/my_route#my_route-py)
 
-
 ____
 
 ## `my_route`.py
@@ -16,6 +15,7 @@ ____
 >
 > * privileged rights
 > * [Python 3](https://www.python.org/downloads/) (tested version 3.9.5 on [Debian GNU/Linux 11](http://ftp.debian.org/debian/dists/bullseye/))
+> * [git](https://git-scm.com/) (tested version 2.30.2 on [Debian GNU/Linux 11](http://ftp.debian.org/debian/dists/bullseye/))
 
 |  PARAMETERS | DESCRIPTION | DEFAULT|
 |-------------|-------------|--------|
@@ -27,22 +27,22 @@ ____
 |**[-u, --update]**|update cidr file db|`False`|
 |**[-f, --force]**|force download sources for update|`False`|
 
-Example usage:  
+### Example usage my_route.py
 
 ```bash
-# download
+# download my_route.py
 sudo wget https://git.hmp.today/pavel.muhortov/my_route/raw/branch/master/my_route.py -O /usr/local/bin/my_route.py
 sudo chmod +x /usr/local/bin/my_route.py
 ```
 
 ```bash
-# download and edit config file
+# download and edit my_route.conf
 sudo wget https://git.hmp.today/pavel.muhortov/my_route/raw/branch/master/my_route.conf -O /usr/local/bin/my_route.conf
 sudo nano /usr/local/bin/my_route.conf
 ```
 
 ```bash
-# create and edit cidr file or update by sources
+# create and edit cidr files or update by sources
 sudo mkdir /usr/local/bin/my_route.db
 sudo tee /usr/local/bin/my_route.db/sources/custom.cidr > /dev/null <<'EOF'
 34.117.59.81/32     # ipinfo.io
@@ -51,5 +51,5 @@ EOF
 
 ```bash
 # sudo crontab -e
-0 0 * * * /usr/bin/python3 /usr/local/bin/my_route.py --update
+0 0 * * * /usr/bin/python3 /usr/local/bin/my_route.py --update --force
 ```

my_route.conf

@@ -36,14 +36,14 @@ atlassian = true
 #
 # herrbischoff - it's a GitHub repository (https://github.com/herrbischoff/country-ip-blocks)
 # + absoutely free
-# + simple parsing
+# + updated hourly
 # - less accuracy
 herrbischoff = true
 #
 # ip2location - it's a geolocation database (https://www.ip2location.com)
 # + more accuracy
 # - you need account
-# - download the database once a month recommended
+# - updated monthly
 ip2location = true
 ip2l_download_token = ip2location_TOKEN
 #
@@ -58,7 +58,14 @@ ip2l_download_token = ip2location_TOKEN
 # + simple parsing
 # - access to updates only by paid subscription
 ip2l_database_code = DB1CIDR
-
+#
+# wwwhmptoday - it's compilation from all other sources (https://git.hmp.today/pavel.muhortov/my_route.db)
+# + all the benefits of other sources
+# + all lists ready to use
+# - you need account
+wwwhmptoday = true
+wwwhmptoday_user = username
+wwwhmptoday_pass = password
 
 
 [enable-gateway]
@@ -335,3 +342,260 @@ ipv4/za.cidr
 ipv4/zm.cidr
 ipv4/zw.cidr
 herrbischoff/cidr/ipv4/zz.cidr
+[dev-wg1]
+# List of CIDR. Only CIDR with the TRUE value will be used.
+# If different sources contain the same cidr, use the path part to define your preference.
+ipv4/dev-wg1.cidr = true
+ipv4/public.cidr = true
+ip2location/cidr/ipv4/-.cidr
+ipv4/ad.cidr
+ipv4/ae.cidr
+ipv4/af.cidr
+ipv4/ag.cidr
+ipv4/ai.cidr
+ipv4/al.cidr
+ipv4/am.cidr
+ipv4/ao.cidr
+herrbischoff/cidr/ipv4/ap.cidr
+ipv4/aq.cidr
+ipv4/ar.cidr
+ipv4/as.cidr
+ipv4/at.cidr
+ipv4/au.cidr
+ipv4/aw.cidr
+ipv4/ax.cidr
+ipv4/az.cidr
+ipv4/ba.cidr
+ipv4/bb.cidr
+ipv4/bd.cidr
+ipv4/be.cidr
+ipv4/bf.cidr
+ipv4/bg.cidr
+ipv4/bh.cidr
+ipv4/bi.cidr
+ipv4/bj.cidr
+ipv4/bl.cidr
+ipv4/bm.cidr
+ipv4/bn.cidr
+ipv4/bo.cidr
+ipv4/bq.cidr
+ipv4/br.cidr
+ipv4/bs.cidr
+ipv4/bt.cidr
+ip2location/cidr/ipv4/bv.cidr
+ipv4/bw.cidr
+ipv4/by.cidr
+ipv4/bz.cidr
+ipv4/ca.cidr
+ip2location/cidr/ipv4/cc.cidr
+ipv4/cd.cidr
+ipv4/cf.cidr
+ipv4/cg.cidr
+ipv4/ch.cidr
+ipv4/ci.cidr
+ipv4/ck.cidr
+ipv4/cl.cidr
+ipv4/cm.cidr
+ipv4/cn.cidr
+ipv4/co.cidr
+ipv4/cr.cidr
+ipv4/cu.cidr
+ipv4/cv.cidr
+ipv4/cw.cidr
+ip2location/cidr/ipv4/cx.cidr
+ipv4/cy.cidr
+ipv4/cz.cidr
+ipv4/de.cidr
+ipv4/dj.cidr
+ipv4/dk.cidr
+ipv4/dm.cidr
+ipv4/do.cidr
+ipv4/dz.cidr
+ipv4/ec.cidr
+ipv4/ee.cidr
+ipv4/eg.cidr
+ip2location/cidr/ipv4/eh.cidr
+ipv4/er.cidr
+ipv4/es.cidr
+ipv4/et.cidr
+ipv4/fi.cidr
+ipv4/fj.cidr
+ipv4/fk.cidr
+ipv4/fm.cidr
+ipv4/fo.cidr
+ipv4/fr.cidr
+ipv4/ga.cidr
+ipv4/gb.cidr
+ipv4/gd.cidr
+ipv4/ge.cidr
+ipv4/gf.cidr
+ipv4/gg.cidr
+ipv4/gh.cidr
+ipv4/gi.cidr
+ipv4/gl.cidr
+ipv4/gm.cidr
+ipv4/gn.cidr
+ipv4/gp.cidr
+ipv4/gq.cidr
+ipv4/gr.cidr
+ip2location/cidr/ipv4/gs.cidr
+ipv4/gt.cidr
+ipv4/gu.cidr
+ipv4/gw.cidr
+ipv4/gy.cidr
+ipv4/hk.cidr
+ip2location/cidr/ipv4/hm.cidr
+ipv4/hn.cidr
+ipv4/hr.cidr
+ipv4/ht.cidr
+ipv4/hu.cidr
+ipv4/id.cidr
+ipv4/ie.cidr
+ipv4/il.cidr
+ipv4/im.cidr
+ipv4/in.cidr
+ipv4/io.cidr
+ipv4/iq.cidr
+ipv4/ir.cidr
+ipv4/is.cidr
+ipv4/it.cidr
+ipv4/je.cidr
+ipv4/jm.cidr
+ipv4/jo.cidr
+ipv4/jp.cidr
+ipv4/ke.cidr
+ipv4/kg.cidr
+ipv4/kh.cidr
+ipv4/ki.cidr
+ipv4/km.cidr
+ipv4/kn.cidr
+ipv4/kp.cidr
+ipv4/kr.cidr
+ipv4/kw.cidr
+ipv4/ky.cidr
+ipv4/kz.cidr
+ipv4/la.cidr
+ipv4/lb.cidr
+ipv4/lc.cidr
+ipv4/li.cidr
+ipv4/lk.cidr
+ipv4/lr.cidr
+ipv4/ls.cidr
+ipv4/lt.cidr
+ipv4/lu.cidr
+ipv4/lv.cidr
+ipv4/ly.cidr
+ipv4/ma.cidr
+ipv4/mc.cidr
+ipv4/md.cidr
+ipv4/me.cidr
+ipv4/mf.cidr
+ipv4/mg.cidr
+ipv4/mh.cidr
+ipv4/mk.cidr
+ipv4/ml.cidr
+ipv4/mm.cidr
+ipv4/mn.cidr
+ipv4/mo.cidr
+ipv4/mp.cidr
+ipv4/mq.cidr
+ipv4/mr.cidr
+ipv4/ms.cidr
+ipv4/mt.cidr
+ipv4/mu.cidr
+ipv4/mv.cidr
+ipv4/mw.cidr
+ipv4/mx.cidr
+ipv4/my.cidr
+ipv4/mz.cidr
+ipv4/na.cidr
+ipv4/nc.cidr
+ipv4/ne.cidr
+ipv4/nf.cidr
+ipv4/ng.cidr
+ipv4/ni.cidr
+ipv4/nl.cidr
+ipv4/no.cidr
+ipv4/np.cidr
+ipv4/nr.cidr
+ipv4/nu.cidr
+ipv4/nz.cidr
+ipv4/om.cidr
+ipv4/pa.cidr
+ipv4/pe.cidr
+ipv4/pf.cidr
+ipv4/pg.cidr
+ipv4/ph.cidr
+ipv4/pk.cidr
+ipv4/pl.cidr
+ipv4/pm.cidr
+ip2location/cidr/ipv4/pn.cidr
+ipv4/pr.cidr
+ipv4/ps.cidr
+ipv4/pt.cidr
+ipv4/pw.cidr
+ipv4/py.cidr
+ipv4/qa.cidr
+ipv4/re.cidr
+ipv4/ro.cidr
+ipv4/rs.cidr
+ipv4/ru.cidr
+ipv4/rw.cidr
+ipv4/sa.cidr
+ipv4/sb.cidr
+ipv4/sc.cidr
+ipv4/sd.cidr
+ipv4/se.cidr
+ipv4/sg.cidr
+ip2location/cidr/ipv4/sh.cidr
+ipv4/si.cidr
+ip2location/cidr/ipv4/sj.cidr
+ipv4/sk.cidr
+ipv4/sl.cidr
+ipv4/sm.cidr
+ipv4/sn.cidr
+ipv4/so.cidr
+ipv4/sr.cidr
+ipv4/ss.cidr
+ipv4/st.cidr
+ipv4/sv.cidr
+ipv4/sx.cidr
+ipv4/sy.cidr
+ipv4/sz.cidr
+ipv4/tc.cidr
+ipv4/td.cidr
+ip2location/cidr/ipv4/tf.cidr
+ipv4/tg.cidr
+ipv4/th.cidr
+ipv4/tj.cidr
+ipv4/tk.cidr
+ipv4/tl.cidr
+ipv4/tm.cidr
+ipv4/tn.cidr
+ipv4/to.cidr
+ipv4/tr.cidr
+ipv4/tt.cidr
+ipv4/tv.cidr
+ipv4/tw.cidr
+ipv4/tz.cidr
+ipv4/ua.cidr
+ipv4/ug.cidr
+ip2location/cidr/ipv4/um.cidr
+ipv4/us.cidr
+ipv4/uy.cidr
+ipv4/uz.cidr
+ipv4/va.cidr
+ipv4/vc.cidr
+ipv4/ve.cidr
+ipv4/vg.cidr
+ipv4/vi.cidr
+ipv4/vn.cidr
+ipv4/vu.cidr
+ipv4/wf.cidr
+ipv4/ws.cidr
+ipv4/ye.cidr
+ipv4/yt.cidr
+ipv4/za.cidr
+ipv4/zm.cidr
+ipv4/zw.cidr
+herrbischoff/cidr/ipv4/zz.cidr

my_route.py

@@ -14,7 +14,7 @@ import logging
 import urllib.request
 from argparse import ArgumentParser
 from datetime import datetime
-from os import path, sep, makedirs, remove, rmdir, walk
+from os import path, sep, makedirs, remove, rmdir, system, walk
 from shutil import copyfile
 from sys import platform
 from subprocess import Popen, PIPE
@@ -393,6 +393,8 @@ class Route(Connect):
         db_root_path: str,
         db_source_code: (str, type(None)) = None,
         download_token: (str, type(None)) = None,
+        download_user: (str, type(None)) = None,
+        download_pass: (str, type(None)) = None,
         force_download: bool = False,
         logger_alias: str = inspect.stack()[0].function
         ) -> bool:
@@ -444,6 +446,14 @@ class Route(Connect):
                     force_download=force_download,
                     logger_alias=logger_alias
                     )
+            elif name == 'wwwhmptoday':
+                return self.__update_source_wwwhmptoday(
+                    db_root_path=db_root_path,
+                    download_user=download_user,
+                    download_pass=download_pass,
+                    force_download=force_download,
+                    logger_alias=logger_alias
+                    )
 
     def __download_db(
         self,
@@ -836,6 +846,49 @@ class Route(Connect):
                 remove(db_source_file)
             return False
 
+    def __update_source_wwwhmptoday(
+        self,
+        db_root_path: str,
+        download_user: str,
+        download_pass: str,
+        force_download: bool = False,
+        logger_alias: str = inspect.stack()[0].function
+        ) -> bool:
+        local_logger = logging.getLogger(logger_alias)
+        if Do.args_valid(locals(), self.__update_source_wwwhmptoday.__annotations__):
+            if not path.exists(db_root_path + sep + 'sources'):
+                force_download = True
+            if force_download:
+                db_temp_path = path.dirname(db_root_path) + sep + 'tmp'
+                cmd_gitclone = (''
+                    + "git clone " + "https://" + download_user + ":" + download_pass
+                    + "@git.hmp.today/pavel.muhortov/my_route.db.git " + db_temp_path
+                )
+                ret_gitclone = system(cmd_gitclone)
+
+                if ret_gitclone == 0:
+                    try:
+                        makedirs(db_root_path, exist_ok=True)
+                        for root, dirs, files in walk(db_temp_path, topdown=False):
+                            for file in files:
+                                src_file = path.join(path.realpath(root), file)
+                                dst_file = src_file.replace(db_temp_path, db_root_path)
+                                makedirs(path.dirname(dst_file), exist_ok=True)
+                                copyfile(src=src_file, dst=dst_file)
+                                remove(path.join(path.realpath(root), file))
+                            for directory in dirs:
+                                rmdir(path.join(path.realpath(root), directory))
+                        rmdir(db_temp_path)
+                        return True
+                    except OSError as error:
+                        local_logger.debug(msg='error: ' + '\n' + str(error))
+                else:
+                    local_logger.warning(msg=''
+                        + 'git clone returned '+ str(ret_gitclone) + ' code. '
+                        + 'Restart by interactive and check stdout.'
+                        )
+            return False
+
 
 class Do():
     """Set of various methods (functions) for routine.
@@ -894,7 +947,8 @@ if __name__ == "__main__":
         description='Route management by CIDR lists.',
         epilog='Dependencies: '
                '- Python 3 (tested version 3.9.5), '
-               '- privileged rights '
+               '- privileged rights, '
+               '- git '
         )
     args.add_argument('--config', type=str,
         default=path.splitext(__file__)[0] + '.conf',
@@ -959,7 +1013,9 @@ if __name__ == "__main__":
                 enable_sources[key] = {
                     'enable': value,
                     'download_token': None,
-                    'db_source_code': None
+                    'db_source_code': None,
+                    'download_user': None,
+                    'download_pass': None
                     }
         if 'ip2location' in enable_sources:
             enable_sources['ip2location']['download_token'] = (
@@ -968,6 +1024,13 @@ if __name__ == "__main__":
             enable_sources['ip2location']['db_source_code'] = (
                 conf_sources.data['ip2l_database_code']
                 )
+        if 'wwwhmptoday' in enable_sources:
+            enable_sources['wwwhmptoday']['download_user'] = (
+                conf_sources.data['wwwhmptoday_user']
+                )
+            enable_sources['wwwhmptoday']['download_pass'] = (
+                conf_sources.data['wwwhmptoday_pass']
+                )
 
     logging.basicConfig(
         format='%(asctime)s %(levelname)s: %(name)s: %(message)s',
@@ -983,7 +1046,7 @@ if __name__ == "__main__":
         )
 
 
-    if Do.checkroot():
+    if not Do.checkroot():
         ro = Route(gateways=enable_gateway, db_root_path=db_root_path)
 
         if args['update']:
@@ -993,6 +1056,8 @@ if __name__ == "__main__":
                     db_root_path=db_root_path,
                     db_source_code=enable_sources[key]['db_source_code'],
                     download_token=enable_sources[key]['download_token'],
+                    download_user=enable_sources[key]['download_user'],
+                    download_pass=enable_sources[key]['download_pass'],
                     force_download=args['force'],
                     logger_alias='update sources ' + key
                     )