mssql-jobs-by-fsaver

README.md

@@ -1,37 +1,137 @@
-# template-bash
+# dbms
 
-Template repository for projects on bash
+Collection for database management systems
 
-* [`script.sh`](https://git.hmp.today/pavel.muhortov/template-bash#script-sh)
+* [`mssql-jobs-by-fsaver`](https://git.hmp.today/pavel.muhortov/dbms#mssql-jobs-by-fsaver)
+* [`mysqldump-wrapper.sh`](https://git.hmp.today/pavel.muhortov/dbms#mysqldump-wrapper-sh)
 
 ____
 
-## `script.sh`
+## `mssql-jobs-by-fsaver`
 
 **Description:**  
-> returning current username if privileged rights are exist  
+> running Microsoft SQL Server agent jobs by Effector Saver with handling of executing time and errors
-> or  
-> returning error, if privileged rights are not exist
 
 **Dependencies:**  
 >
-> * [bash](https://www.gnu.org/software/bash/) (tested version 5.1.4 on [Debian GNU/Linux 11](http://ftp.debian.org/debian/dists/bullseye/))
+> * [SQL Server](https://www.microsoft.com/en-us/sql-server) (tested version 2022 on [Windows Server 2022](https://learn.microsoft.com/en-us/windows-server/get-started/whats-new-in-windows-server-2022))
-> * [whoami](https://www.gnu.org/software/coreutils/whoami) (tested version 8.30 on [Debian GNU/Linux 11](http://ftp.debian.org/debian/dists/bullseye/))
+> * [SQL Server Management Studio](https://learn.microsoft.com/en-us/sql/ssms) (tested version 19.1 on [Windows Server 2022](https://learn.microsoft.com/en-us/windows-server/get-started/whats-new-in-windows-server-2022))
+> * [Effector Saver](https://mixbackup.com/) (tested version 4.11 on [Windows Server 2022](https://learn.microsoft.com/en-us/windows-server/get-started/whats-new-in-windows-server-2022))
+
+### Create maintenance plan with subplan name
+>
+> example maintanence plane name: `Daily-Database_Name`  
+> example subplan name: `Backup`  
+
+Microsoft SQL Server Management Studio -> Management -> Maintenance Plans -> New Maintenance Plan -> Name: `Daily-Database_Name` -> Subplan properties -> Name: `Backup`
+
+### Check auto made agent job and its name
+
+Microsoft SQL Server Management Studio -> SQL Server Agent -> Jobs -> `Daily-Database_Name.Backup`
+
+### Create Effector Saver task
+
+Effector Saver -> Tasks -> Add task -> Other tasks -> Commands -> Name: Daily-Database_Name -> Main preferences -> Add command
+
+Name:
+
+```text
+Daily-Database_Name.Backup
+```
+
+Command type:
+
+```text
+SQL script
+```
+
+Command text:
+
+```sql
+SET NOCOUNT ON
+
+DECLARE  @jobNM NVARCHAR(MAX) = 'Daily-Database_Name.Backup',
+         @jobID UNIQUEIDENTIFIER,
+         @maxID INT
+
+SELECT @jobID = job_id FROM msdb..sysjobs WHERE name = @jobNM
+SELECT @maxID = MAX(instance_id) FROM msdb..sysjobhistory WHERE job_id = @jobID AND step_id = 0
+SET @maxID = COALESCE(@maxID, -1)
+
+EXEC msdb..sp_start_job @job_id = @jobID
+WHILE (SELECT MAX(instance_id) FROM msdb..sysjobhistory WHERE job_id = @jobID AND step_id = 0) = @maxID
+    WAITFOR DELAY '00:00:01'
+
+SELECT @maxID = MAX(instance_id) FROM msdb..sysjobhistory WHERE job_id = @jobID AND step_id = 0
+IF (SELECT run_status FROM msdb..sysjobhistory WHERE instance_id = @maxID) = 0
+    RAISERROR (N'#ERROR# Microsoft SQL Server Job finished with error', 16, 1) WITH NOWAIT
+
+```
+
+____
+
+## `mysqldump-wrapper.sh`
+
+**Description:**  
+> creating database dump, copying to additional smb share, copies rotating
+> and
+> sending report to email
+
+**Dependencies:**  
+>
+> * privileged rights
+> * [mysqldump](https://dev.mysql.com/doc/refman/5.7/en/mysqldump.html) (tested version 5.6.35 on [CentOS 7](https://wiki.centos.org/Manuals/ReleaseNotes/CentOS7.2009))
+> * [gzip](https://www.gnu.org/software/gzip/manual/gzip.html) (tested version 1.5 on [CentOS 7](https://wiki.centos.org/Manuals/ReleaseNotes/CentOS7.2009))
+> * [cifs-utils](https://wiki.samba.org/index.php/LinuxCIFS_utils) (tested version 2.08 on [CentOS 7](https://wiki.centos.org/Manuals/ReleaseNotes/CentOS7.2009))
+> * [Python 3](https://www.python.org/downloads/) (tested version [3.9.5](https://git.hmp.today/pavel.muhortov/utils#build-python-sh) on [CentOS 7](https://wiki.centos.org/Manuals/ReleaseNotes/CentOS7.2009))
+> * [sendmail.py](https://git.hmp.today/pavel.muhortov/utils#sendmail-py)
 
 |  POSITION | PARAMETERS   |       DESCRIPTION      |    DEFAULT    |
 |-----------|--------------|------------------------|---------------|
-|     1     |**[qn]**|execution without pauses||
+|     1     |**[/path/to/file.conf]**|path to config file|**REQUIRED**|
-|     2     |**[/path/to/conf]**|path to config|`./script.conf`|
 
-Example usage in terminal with bash:
+### Installing mysqldump-wrapper.sh and setting up crontab
 
 ```bash
-bash ./script.sh qn ./script.conf
+# edit config
-```
+sudo tee /usr/local/bin/mysqldump-wrapper.conf > /dev/null <<'EOF'
+# mysql connection parameters
+db_host=db-server.domain.zone
+db_user=db-username
+db_pass=db-password
+db_name=database
 
-Example usage in terminal with make the script executable:
+# dump repository parameters
+dump_root=/home/user/backup
+dump_save=1
+
+# copy smb-repository parameters
+smb_host=smb-server.domain.zone
+smb_path=smb-share/backup
+smb_user=smb-username
+smb_domn=smb-domain
+smb_pass=smb-password
+copy_save=7
+
+# sendmail parameters
+smtp_pyth=/usr/local/opt/python-3.9/bin/python3.9
+smtp_send=/usr/local/bin/sendmail.py
+smtp_host=mail-server.domain.zone
+smtp_port=587
+smtp_from=mail-from@domain.zone
+smtp_pass=mail-password
+smtp_dest=mail-dest@domain.zone
+EOF
+```
 
 ```bash
-chmod u+x ./script.sh
+# download
-script.sh
+sudo wget https://git.hmp.today/pavel.muhortov/dbms/raw/branch/master/mysqldump-wrapper.sh -O /usr/local/bin/mysqldump-wrapper.sh
+sudo chmod +x /usr/local/bin/mysqldump-wrapper.sh
+```
+
+```bash
+# sudo sh -c "EDITOR=nano crontab -e"
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+00 20 * * * bash /usr/local/bin/mysqldump-wrapper.sh /usr/local/bin/mysqldump-wrapper.conf
 ```

mysqldump-wrapper.sh

@@ -0,0 +1,301 @@
+#!/usr/bin/env bash
+
+# DESCRIPTION:
+#   creating database dump, copying to additional smb share, copies rotating
+#   and
+#   sending report to email
+#
+# DEPENDENCIES:
+#   - privileged rights
+#   - mysqldump
+#   - gzip
+#   - cifs-utils
+#   - Python 3
+#   - sendmail.py
+#
+# PARAMETERS:
+#   1: "/path/to/file.conf" - path to config file
+#
+# FUNCTIONS:
+#
+
+#######################################
+# Print message and add to log.
+# Globals:
+#   logs: /path/to/file.log
+# Arguments:
+#   1: message to print and logging
+#######################################
+addtologs() {
+  printf "%s\n" "$(date +'%Y.%m.%d-%H:%M:%S') $1" | tee -a "${logs}"
+}
+
+#######################################
+# Exit procedure.
+# Globals:
+#   time: time of script start
+# Arguments:
+#   None
+#######################################
+execquite() {
+  addtologs "execution time is $(($(date +%s)-time)) seconds, exit."
+  exit "${1}"
+}
+
+#######################################
+# Error exit procedure.
+# Globals:
+#   None
+# Arguments:
+#   1: message to print and logging
+#######################################
+execerror() {
+  addtologs "error: $1"
+  execquite 1
+}
+
+#######################################
+# Parsing config file and creating global vars.
+# Globals:
+#   conf: /path/to/file.conf
+# Arguments:
+#   None
+#######################################
+getconfig() {
+  # mysql connection parameters
+  db_host=$(grep 'db_host=' "${conf}" | cut -d= -f2)
+  db_user=$(grep 'db_user=' "${conf}" | cut -d= -f2)
+  db_pass=$(grep 'db_pass=' "${conf}" | cut -d= -f2)
+  db_name=$(grep 'db_name=' "${conf}" | cut -d= -f2)
+
+  # dump repository parameters
+  dump_root=$(grep 'dump_root=' "${conf}" | cut -d= -f2)
+  dump_name="${db_name}_backup_$(date +%Y_%m_%d_%H%M%S).sql.gz"
+  dump_file="${dump_root}/${dump_name}"
+  dump_save=$(grep 'dump_save=' "${conf}" | cut -d= -f2)
+
+  # copy smb-repository parameters
+  smb_host=$(grep 'smb_host=' "${conf}" | cut -d= -f2)
+  smb_path=$(grep 'smb_path=' "${conf}" | cut -d= -f2)
+  smb_user=$(grep 'smb_user=' "${conf}" | cut -d= -f2)
+  smb_domn=$(grep 'smb_domn=' "${conf}" | cut -d= -f2)
+  smb_pass=$(grep 'smb_pass=' "${conf}" | cut -d= -f2)
+  copy_root="/mnt/${smb_host}/${smb_path}"
+  copy_file="${copy_root}/${dump_name}"
+  copy_save=$(grep 'copy_save=' "${conf}" | cut -d= -f2)
+
+  # sendmail parameters
+  smtp_pyth=$(grep 'smtp_pyth=' "${conf}" | cut -d= -f2)
+  smtp_send=$(grep 'smtp_send=' "${conf}" | cut -d= -f2)
+  smtp_host=$(grep 'smtp_host=' "${conf}" | cut -d= -f2)
+  smtp_port=$(grep 'smtp_port=' "${conf}" | cut -d= -f2)
+  smtp_from=$(grep 'smtp_from=' "${conf}" | cut -d= -f2)
+  smtp_pass=$(grep 'smtp_pass=' "${conf}" | cut -d= -f2)
+  smtp_dest=$(grep 'smtp_dest=' "${conf}" | cut -d= -f2)
+
+  # common parameters
+  logs="${dump_file}.log"
+}
+
+#######################################
+# Checking user rights.
+# Globals:
+#   None
+# Arguments:
+#   None
+# return:
+#   0 - if privileged rights, 1 - if not privileged rights
+#######################################
+checkroot() {
+  if [ "${EUID}" -ne 0 ]; then
+    return 1 # false
+  else
+    return 0 # true
+  fi
+}
+
+#######################################
+# Start mysqldump without database blocking (WARNING: InnoDB required).
+# Globals:
+#   db_user: db auth username
+#   db_pass: db auth password
+#   db_host: db host address
+#   db_name: db name
+#   dump_file: database_dump.sql.gz
+#   dump_root: /path/to/dump/directory
+#   addtolog: log-writer function
+# Arguments:
+#   None
+# return:
+#   0 - dump created, 1 - dump failed
+#######################################
+createdump() {
+  if mysqldump --single-transaction --skip-lock-tables \
+               --user="${db_user}" --password="${db_pass}" \
+               -h "${db_host}" "${db_name}" | \
+               gzip > "${dump_file}"; then
+    addtologs "created ${dump_file}"
+    return 0 # true
+
+  else
+    addtologs "failed ${dump_file}"
+    return 1 # false
+  fi
+}
+
+#######################################
+# Search and delete files, keep last file's amount.
+# Globals:
+#   addtolog: log-writer function
+# Arguments:
+#   1: path for search
+#   2: name for search (grep's regex)
+#   3: amount of last files to keep
+#######################################
+# shellcheck disable=SC2010,SC2207
+cleanolder(){
+  file_root=$1
+  file_name=$2
+  keep_last=$3
+  file_list=()
+
+  file_list=($(ls -tra "${file_root}" | grep "${file_name}"))
+  list_size=${#file_list[@]}
+  list_stop=$((list_size-keep_last))
+  if [ "${list_stop}" -gt 0 ]; then
+    for ((i=0; i<list_stop; i++)); do
+      rm -f "${file_root}/${file_list[i]}"
+      addtologs "deleted ${file_root}/${file_list[i]}"
+    done
+  fi
+}
+
+#######################################
+# Send email notification.
+# Globals:
+#   smtp_pyth: /path/to/bin/python
+#   smtp_send: /path/to/sendmail.py
+#   smtp_from: from@domain.zone
+#   smtp_pass: password for from@domain.zone
+#   smtp_dest: destination@domain.zone
+#   smtp_host: mail-server.domain.zone
+#   smtp_port: 25,465,587
+#   smtp_subj: subject of mail
+#   addtolog: log-writer function
+# Arguments:
+#   None
+#######################################
+startsendmail() {
+  (
+    "${smtp_pyth}" "${smtp_send}" \
+      -u "${smtp_from}" -p "${smtp_pass}" -d "${smtp_dest}" \
+      --smtp "${smtp_host}" --port "${smtp_port}" --stls "True" \
+      --subj "${smtp_subj}" --text "$(cat "${logs}")"
+  ) > /dev/null 2>&1
+  addtologs "sent mail with subject '${smtp_subj}' to '${smtp_dest}'"
+}
+
+#
+# VARIABLES:
+#
+
+conf=$1
+time=$(date +%s)
+newl=$'\n'
+
+#
+# MAIN:
+#
+
+if [ -e "${conf}" ]; then
+  getconfig
+
+else
+  logs=/dev/null
+  execerror "Config not found. Usage example: $0 '/path/to/dump.conf'"
+fi
+
+if ! command -v mysqldump &> /dev/null || \
+   ! command -v gzip &> /dev/null || \
+   ! command -v "${smtp_send}" &> /dev/null || \
+   ! command -v "${smtp_pyth}" &> /dev/null || \
+   ! command -v mount.cifs &> /dev/null; then
+  execerror "Not found dependencies."
+fi
+
+if ! checkroot; then
+  execerror "Restart this as root!"
+
+else
+  smtp_subj="[Failed] MySQL backup: '${db_name}' on ${db_host}"
+  db_orig_size_byte=$(mysql --user="${db_user}" --password="${db_pass}" -h "${db_host}" \
+    -e 'SELECT table_schema AS "Database", (SUM(data_length)+SUM(index_length)) AS "Size" FROM information_schema.TABLES GROUP BY table_schema;' | \
+    grep "${db_name}" | awk '{print $2}')
+  fs_root_free_byte=$(df -B1 "${dump_root}" | tail -1 | awk '{print $4}')
+  addtologs "${db_orig_size_byte} bytes are in '${db_name}' database."
+  addtologs "${fs_root_free_byte} bytes are free in ${dump_root}."
+
+  if [ "${db_orig_size_byte}" -gt "${fs_root_free_byte}" ]; then
+    addtologs "%s\n" "Not enough free space"
+
+  else
+    addtologs "Start backuping MySQL '${db_name}' database.${newl}"
+
+    if ! createdump; then
+      addtologs "Failed backuping MySQL '${db_name}' database."
+
+    else
+      smtp_subj="[Warning] MySQL backup: '${db_name}' on ${db_host}"
+      db_copy_size_unit=$(du -h "${dump_file}" | awk '{print $1}')
+      fs_root_free_unit=$(df -h "${dump_root}" | tail -1 | awk '{print $4}')
+      addtologs "created ${dump_file} of ${db_copy_size_unit}."
+      addtologs "${fs_root_free_unit} are free in ${dump_root}."
+
+      addtologs "Start cleaning. Keep last ${dump_save}.${newl}"
+      cleanolder "${dump_root}" "^${db_name}_backup_.*.sql.gz$" "${dump_save}"
+      cleanolder "${dump_root}" "^${db_name}_backup_.*.sql.gz.log$" "${dump_save}"
+      fs_root_free_unit=$(df -h "${dump_root}" | tail -1 | awk '{print $4}')
+      addtologs "${fs_root_free_unit} are free in ${dump_root}."
+
+      if mkdir -p "${copy_root}" && \
+        mount.cifs "//${smb_host}/${smb_path}" "${copy_root}" \
+        -o "user=${smb_user},domain=${smb_domn},password=${smb_pass},vers=2.0"; then
+        sleep 1
+
+        db_copy_size_byte=$(du -B1 "${dump_file}" | awk '{print $1}')
+        fs_copy_free_byte=$(df -B1 "${copy_root}" | tail -1 | awk '{print $4}')
+        addtologs "${db_copy_size_byte} bytes are in ${dump_file}."
+        addtologs "${fs_copy_free_byte} bytes are free in ${copy_root}."
+
+        if [ "${db_copy_size_byte}" -gt "${fs_copy_free_byte}" ]; then
+          addtologs "%s\n" "Not enough free space"
+
+        else
+          addtologs "Start copying backup to //${smb_host}/${smb_path}${newl}"
+
+          if ! cp "${dump_file}" "${copy_file}"; then
+            addtologs "failed ${db_name} dump copy in ${copy_root}"
+
+          else
+            smtp_subj="[Success] MySQL backup: '${db_name}' on ${db_host}"
+            addtologs "created ${copy_file}"
+            fs_copy_free_unit=$(df -h "${copy_root}" | tail -1 | awk '{print $4}')
+            addtologs "${fs_copy_free_unit} are free in ${copy_root}."
+            sleep 1
+
+            addtologs "Start cleaning. Keep last ${copy_save}.${newl}"
+            cleanolder "${copy_root}" "^${db_name}_backup_.*.sql.gz$" "${copy_save}"
+            fs_copy_free_unit=$(df -h "${copy_root}" | tail -1 | awk '{print $4}')
+            addtologs "${fs_copy_free_unit} are free in ${copy_root}."
+          fi
+        fi
+
+        umount "/mnt/${smb_host}/${smb_path}" && rm -rf "/mnt/${smb_host:?}"
+      fi
+    fi
+  fi
+
+  startsendmail
+fi
+
+execquite 0

script.sh

@@ -1,125 +0,0 @@
-#!/usr/bin/env bash
-
-# DESCRIPTION:
-#   returning current username if privileged rights are exist
-#   or
-#   returning error, if privileged rights are not exist
-#
-# DEPENDENCIES:
-#   - whoami
-#
-# PARAMETERS:
-#   1: "qn" - execution without pauses
-#   2: custom configuration file path
-#
-# FUNCTIONS:
-#
-
-#######################################
-# Print message and add to log.
-# Globals:
-#   logs
-# Arguments:
-#   1: message to print and logging
-#######################################
-addtologs() {
-  echo "$(date +'%Y.%m.%d-%H:%M:%S') $1" | tee -a "${logs}"
-}
-
-#######################################
-# Waiting for press [ENTER].
-# Globals:
-#   None
-# Arguments:
-#   None
-#######################################
-execpause() {
-  read -r -p "Press [ENTER] to continue... "
-}
-
-#######################################
-# Exit procedure.
-# Globals:
-#   show
-# Arguments:
-#   None
-#######################################
-execquite() {
-  addtologs "execution time is $(($(date +%s)-time)) seconds, exit"
-  if [ "${show}" != "qn" ]; then
-    execpause
-  fi
-  exit
-}
-
-#######################################
-# Error exit procedure
-# Globals:
-#   None
-# Arguments:
-#   1: message to print and logging
-#######################################
-execerror() {
-  addtologs "error: $1"
-  execquite
-}
-
-#######################################
-# Parsing config file and creating global vars.
-# Globals:
-#   None
-# Arguments:
-#   None
-#######################################
-getconfig() {
-  logs=$(grep "logs=" "${conf}" | cut -d= -f2)
-}
-
-#######################################
-# Checking user rights.
-# Globals:
-#   None
-# Arguments:
-#   None
-# return:
-#   0 - if privileged rights, 1 - if not privileged rights
-#######################################
-checkroot() {
-  if [ "${EUID}" -ne 0 ]; then
-    return 1 # false
-  else
-    return 0 # true
-  fi
-}
-
-#
-# VARIABLES:
-#
-
-show=$1
-conf=$2
-if [ -z "${conf}" ] || [ "${conf}" == "-" ]; then
-  conf="$(dirname "$(realpath "$0")")/$(basename -s .sh "$0").conf"
-fi
-
-time=$(date +%s)
-cd "$(dirname "$(realpath "$0")")" || execerror
-if [ ! -e "${conf}" ]; then
-  :
-else
-  getconfig
-fi
-if [ -z "${logs}" ]; then
-  logs=/dev/null
-elif [ ! -e "${logs}" ]; then
-  touch "${logs}"
-fi
-
-#
-# MAIN:
-#
-
-checkroot \
-&& echo "Running as $(whoami)" \
-&& execquite \
-|| execerror "Restart this as root!"