TISbackup/tisbackup.py

#!/usr/bin/python3
# -*- coding: utf-8 -*-
# -----------------------------------------------------------------------
#    This file is part of TISBackup
#
#    TISBackup is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    TISBackup is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with TISBackup.  If not, see <http://www.gnu.org/licenses/>.
#
# -----------------------------------------------------------------------
import datetime
import os
import sys
from os.path import isfile, join

tisbackup_root_dir = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, os.path.join(tisbackup_root_dir, "lib"))
sys.path.insert(0, os.path.join(tisbackup_root_dir, "libtisbackup"))

import errno
import logging
import os.path

from optparse import OptionParser

from iniparse import ConfigParser, ini

from libtisbackup.backup_mysql import backup_mysql

# from libtisbackup.backup_vmdk import backup_vmdk
# from libtisbackup.backup_switch import backup_switch
from libtisbackup.backup_null import backup_null
from libtisbackup.backup_pgsql import backup_pgsql
from libtisbackup.backup_rsync import backup_rsync, backup_rsync_ssh

# from libtisbackup.backup_oracle import backup_oracle
from libtisbackup.backup_rsync_btrfs import backup_rsync__btrfs_ssh, backup_rsync_btrfs

# from libtisbackup.backup_sqlserver import backup_sqlserver
from libtisbackup.backup_samba4 import backup_samba4
from libtisbackup.backup_xcp_metadata import backup_xcp_metadata
from libtisbackup.backup_xva import backup_xva
from libtisbackup.common import *
from libtisbackup.copy_vm_xcp import copy_vm_xcp

__version__ = "2.0"

usage = """\
%prog -c configfile action

TIS Files Backup system.

action is either :
 backup : launch all backups or a specific one if -s option is used
 cleanup : removed backups older than retension period
 checknagios : check all or a specific backup against max_backup_age parameter
 dumpstat : dump the content of database for the last 20 backups
 retryfailed :  try to relaunch the last failed backups
 listdrivers :  list available backup types and parameters for config inifile
 exportbackup :  copy lastest OK backups from local to location defned by --exportdir parameter
 register_existing : scan backup directories and add missing backups to database"""

version = "VERSION"

parser = OptionParser(usage=usage, version="%prog " + version)
parser.add_option(
    "-c", "--config", dest="config", default="/etc/tis/tisbackup-config.ini", help="Config file full path (default: %default)"
)
parser.add_option("-d", "--dry-run", dest="dry_run", default=False, action="store_true", help="Dry run (default: %default)")
parser.add_option("-v", "--verbose", dest="verbose", default=False, action="store_true", help="More information (default: %default)")
parser.add_option(
    "-s", "--sections", dest="sections", default="", help="Comma separated list of sections (backups) to process (default: All)"
)
parser.add_option(
    "-l",
    "--loglevel",
    dest="loglevel",
    default="info",
    type="choice",
    choices=["debug", "warning", "info", "error", "critical"],
    metavar="LOGLEVEL",
    help="Loglevel (default: %default)",
)
parser.add_option("-n", "--len", dest="statscount", default=30, type="int", help="Number of lines to list for dumpstat (default: %default)")
parser.add_option(
    "-b",
    "--backupdir",
    dest="backup_base_dir",
    default="",
    help="Base directory for all backups (default: [global] backup_base_dir in config file)",
)
parser.add_option(
    "-x", "--exportdir", dest="exportdir", default="", help="Directory where to export latest backups with exportbackup (nodefault)"
)


class tis_backup:
    logger = logging.getLogger("tisbackup")

    def __init__(self, dry_run=False, verbose=False, backup_base_dir=""):
        self.dry_run = dry_run
        self.verbose = verbose
        self.backup_base_dir = backup_base_dir
        self.backup_base_dir = ""
        self.backup_list = []
        self.dry_run = dry_run
        self.verbose = False

    def read_ini_file(self, filename):
        ini.change_comment_syntax()
        cp = ConfigParser()
        cp.read(filename)

        if not self.backup_base_dir:
            self.backup_base_dir = cp.get("global", "backup_base_dir")
        if not os.path.isdir(self.backup_base_dir):
            self.logger.info("Creating backup directory %s" % self.backup_base_dir)
            os.makedirs(self.backup_base_dir)

        self.logger.debug("backup directory : " + self.backup_base_dir)
        self.dbstat = BackupStat(os.path.join(self.backup_base_dir, "log", "tisbackup.sqlite"))

        for section in cp.sections():
            if section != "global":
                self.logger.debug("reading backup config " + section)
                backup_item = None
                type = cp.get(section, "type")

                backup_item = backup_drivers[type](
                    backup_name=section, backup_dir=os.path.join(self.backup_base_dir, section), dbstat=self.dbstat, dry_run=self.dry_run
                )
                backup_item.read_config(cp)
                backup_item.verbose = self.verbose

                self.backup_list.append(backup_item)

        # TODO check hostname socket.gethostbyname_ex('cnn.com')
        # TODO socket.gethostbyaddr('64.236.16.20')
        # TODO limit backup to one backup on the command line

    def checknagios(self, sections=[]):
        try:
            if not sections:
                sections = [backup_item.backup_name for backup_item in self.backup_list]

            self.logger.debug("Start of check nagios for %s" % (",".join(sections),))
            try:
                worst_nagiosstatus = None
                ok = []
                warning = []
                critical = []
                unknown = []
                nagiosoutput = ""
                for backup_item in self.backup_list:
                    if not sections or backup_item.backup_name in sections:
                        (nagiosstatus, log) = backup_item.checknagios()
                        if nagiosstatus == nagiosStateCritical:
                            critical.append((backup_item.backup_name, log))
                        elif nagiosstatus == nagiosStateWarning:
                            warning.append((backup_item.backup_name, log))
                        elif nagiosstatus == nagiosStateOk:
                            ok.append((backup_item.backup_name, log))
                        else:
                            unknown.append((backup_item.backup_name, log))
                        self.logger.debug('[%s] nagios:"%i" log: %s', backup_item.backup_name, nagiosstatus, log)

                if not ok and not critical and not unknown and not warning:
                    self.logger.debug("Nothing processed")
                    worst_nagiosstatus = nagiosStateUnknown
                    nagiosoutput = 'UNKNOWN : Unknown backup sections "%s"' % sections

                globallog = []

                if unknown:
                    if not worst_nagiosstatus:
                        worst_nagiosstatus = nagiosStateUnknown
                        nagiosoutput = "UNKNOWN status backups %s" % (",".join([b[0] for b in unknown]))
                    globallog.extend(unknown)

                if critical:
                    if not worst_nagiosstatus:
                        worst_nagiosstatus = nagiosStateCritical
                        nagiosoutput = "CRITICAL backups %s" % (",".join([b[0] for b in critical]))
                    globallog.extend(critical)

                if warning:
                    if not worst_nagiosstatus:
                        worst_nagiosstatus = nagiosStateWarning
                        nagiosoutput = "WARNING backups %s" % (",".join([b[0] for b in warning]))
                    globallog.extend(warning)

                if ok:
                    if not worst_nagiosstatus:
                        worst_nagiosstatus = nagiosStateOk
                        nagiosoutput = "OK backups %s" % (",".join([b[0] for b in ok]))
                    globallog.extend(ok)

                if worst_nagiosstatus == nagiosStateOk:
                    nagiosoutput = "ALL backups OK %s" % (",".join(sections))

            except BaseException as e:
                worst_nagiosstatus = nagiosStateCritical
                nagiosoutput = "EXCEPTION", "Critical : %s" % str(e)
                raise

        finally:
            self.logger.debug('worst nagios status :"%i"', worst_nagiosstatus)
            print("%s (tisbackup V%s)" % (nagiosoutput, version))
            print("\n".join(["[%s]:%s" % (log_elem[0], log_elem[1]) for log_elem in globallog]))
            sys.exit(worst_nagiosstatus)

    def process_backup(self, sections=[]):
        processed = []
        errors = []
        if not sections:
            sections = [backup_item.backup_name for backup_item in self.backup_list]

        self.logger.info("Processing backup for %s" % (",".join(sections)))
        for backup_item in self.backup_list:
            if not sections or backup_item.backup_name in sections:
                try:
                    assert isinstance(backup_item, backup_generic)
                    self.logger.info("Processing [%s]", (backup_item.backup_name))
                    stats = backup_item.process_backup()
                    processed.append((backup_item.backup_name, stats))
                except BaseException as e:
                    self.logger.critical("Backup [%s] processed with error : %s", backup_item.backup_name, e)
                    errors.append((backup_item.backup_name, str(e)))
        if not processed and not errors:
            self.logger.critical("No backup properly finished or processed")
        else:
            if processed:
                self.logger.info("Backup processed : %s", ",".join([b[0] for b in processed]))
            if errors:
                self.logger.error("Backup processed with errors: %s", ",".join([b[0] for b in errors]))

    def export_backups(self, sections=[], exportdir=""):
        processed = []
        errors = []
        if not sections:
            sections = [backup_item.backup_name for backup_item in self.backup_list]

        self.logger.info("Exporting OK backups for %s to %s" % (",".join(sections), exportdir))

        for backup_item in self.backup_list:
            if backup_item.backup_name in sections:
                try:
                    assert isinstance(backup_item, backup_generic)
                    self.logger.info("Processing [%s]", (backup_item.backup_name))
                    stats = backup_item.export_latestbackup(destdir=exportdir)
                    processed.append((backup_item.backup_name, stats))
                except BaseException as e:
                    self.logger.critical("Export Backup [%s] processed with error : %s", backup_item.backup_name, e)
                    errors.append((backup_item.backup_name, str(e)))
        if not processed and not errors:
            self.logger.critical("No export backup properly finished or processed")
        else:
            if processed:
                self.logger.info("Export Backups processed : %s", ",".join([b[0] for b in processed]))
            if errors:
                self.logger.error("Export Backups processed with errors: %s", ",".join([b[0] for b in errors]))

    def retry_failed_backups(self, maxage_hours=30):
        processed = []
        errors = []

        # before mindate, backup is too old
        mindate = datetime2isodate((datetime.datetime.now() - datetime.timedelta(hours=maxage_hours)))
        failed_backups = self.dbstat.query(
            """\
        select  distinct backup_name as bname
        from stats
        where  status="OK"  and  backup_start>=?""",
            (mindate,),
        )

        defined_backups = list(map(lambda f: f.backup_name, [x for x in self.backup_list if not isinstance(x, backup_null)]))
        failed_backups_names = set(defined_backups) - set([b["bname"] for b in failed_backups if b["bname"] in defined_backups])

        if failed_backups_names:
            self.logger.info("Processing backup for %s", ",".join(failed_backups_names))
            for backup_item in self.backup_list:
                if backup_item.backup_name in failed_backups_names:
                    try:
                        assert isinstance(backup_item, backup_generic)
                        self.logger.info("Processing [%s]", (backup_item.backup_name))
                        stats = backup_item.process_backup()
                        processed.append((backup_item.backup_name, stats))
                    except BaseException as e:
                        self.logger.critical("Backup [%s] not processed, error : %s", backup_item.backup_name, e)
                        errors.append((backup_item.backup_name, str(e)))
            if not processed and not errors:
                self.logger.critical("No backup properly finished or processed")
            else:
                if processed:
                    self.logger.info("Backup processed : %s", ",".join([b[0] for b in errors]))
                if errors:
                    self.logger.error("Backup processed with errors: %s", ",".join([b[0] for b in errors]))
        else:
            self.logger.info("No recent failed backups found in database")

    def cleanup_backup_section(self, sections=[]):
        processed = False
        if not sections:
            sections = [backup_item.backup_name for backup_item in self.backup_list]

        self.logger.info("Processing cleanup for %s" % (",".join(sections)))
        for backup_item in self.backup_list:
            if backup_item.backup_name in sections:
                try:
                    assert isinstance(backup_item, backup_generic)
                    self.logger.info("Processing cleanup of [%s]", (backup_item.backup_name))
                    backup_item.cleanup_backup()
                    processed = True
                except BaseException as e:
                    self.logger.critical("Cleanup of [%s] not processed, error : %s", backup_item.backup_name, e)
        if not processed:
            self.logger.critical("No cleanup properly finished or processed")

    def register_existingbackups(self, sections=[]):
        if not sections:
            sections = [backup_item.backup_name for backup_item in self.backup_list]

        self.logger.info("Append existing backups to database...")
        for backup_item in self.backup_list:
            if backup_item.backup_name in sections:
                backup_item.register_existingbackups()

    def html_report(self):
        for backup_item in self.backup_list:
            if not section or section == backup_item.backup_name:
                assert isinstance(backup_item, backup_generic)
                if not maxage_hours:
                    maxage_hours = backup_item.maximum_backup_age
                (nagiosstatus, log) = backup_item.checknagios(maxage_hours=maxage_hours)
                globallog.append("[%s] %s" % (backup_item.backup_name, log))
                self.logger.debug('[%s] nagios:"%i" log: %s', backup_item.backup_name, nagiosstatus, log)
                # processed = True
                # if nagiosstatus >= worst_nagiosstatus:
                #     worst_nagiosstatus = nagiosstatus


def main():
    (options, args) = parser.parse_args()

    if len(args) != 1:
        print("ERROR : You must provide one action to perform")
        parser.print_usage()
        sys.exit(2)

    backup_start_date = datetime.datetime.now().strftime("%Y%m%d-%Hh%Mm%S")

    # options
    action = args[0]
    if action == "listdrivers":
        for t in backup_drivers:
            print(backup_drivers[t].get_help())
        sys.exit(0)

    config_file = options.config
    dry_run = options.dry_run
    verbose = options.verbose

    loglevel = options.loglevel

    # setup Logger
    logger = logging.getLogger("tisbackup")
    hdlr = logging.StreamHandler()
    hdlr.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
    logger.addHandler(hdlr)

    # set loglevel
    if loglevel in ("debug", "warning", "info", "error", "critical"):
        numeric_level = getattr(logging, loglevel.upper(), None)
        if not isinstance(numeric_level, int):
            raise ValueError("Invalid log level: %s" % loglevel)
        logger.setLevel(numeric_level)

    # Config file
    if not os.path.isfile(config_file):
        logger.error("Error : could not find file : " + config_file + ", please check the path")
    logger.info("Using " + config_file + " config file")

    cp = ConfigParser()
    cp.read(config_file)

    backup_base_dir = options.backup_base_dir or cp.get("global", "backup_base_dir")
    log_dir = os.path.join(backup_base_dir, "log")
    if not os.path.exists(log_dir):
        os.makedirs(log_dir)

    # if we run the nagios check, we don't create log file, everything is piped to stdout
    if action != "checknagios":
        try:
            hdlr = logging.FileHandler(os.path.join(log_dir, "tisbackup_%s.log" % (backup_start_date)))
            hdlr.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
            logger.addHandler(hdlr)
        except IOError as e:
            if action == "cleanup" and e.errno == errno.ENOSPC:
                logger.warning("No space left on device, disabling file logging.")
            else:
                raise e

    # Main
    backup = tis_backup(dry_run=dry_run, verbose=verbose, backup_base_dir=backup_base_dir)
    backup.read_ini_file(config_file)

    backup_sections = options.sections.split(",") if options.sections else []

    all_sections = [backup_item.backup_name for backup_item in backup.backup_list]
    if not backup_sections:
        backup_sections = all_sections
    else:
        for b in backup_sections:
            if b not in all_sections:
                raise Exception("Section %s is not defined in config file" % b)

    if dry_run:
        logger.warning("WARNING : DRY RUN, nothing will be done, just printing on screen...")

    if action == "backup":
        backup.process_backup(backup_sections)
    elif action == "exportbackup":
        if not options.exportdir:
            raise Exception("No export directory supplied dor exportbackup action")
        backup.export_backups(backup_sections, options.exportdir)
    elif action == "cleanup":
        backup.cleanup_backup_section(backup_sections)
    elif action == "checknagios":
        backup.checknagios(backup_sections)
    elif action == "dumpstat":
        for s in backup_sections:
            backup.dbstat.last_backups(s, count=options.statscount)
    elif action == "retryfailed":
        backup.retry_failed_backups()
    elif action == "register_existing":
        backup.register_existingbackups(backup_sections)

    else:
        logger.error('Unhandled action "%s", quitting...', action)
        sys.exit(1)


if __name__ == "__main__":
    main()