refactor(drivers): organize backup modules into drivers subfolder

- Move all backup_*.py files to libtisbackup/drivers/ subdirectory
- Move XenAPI.py and copy_vm_xcp.py to drivers/ (driver-specific)
- Create drivers/__init__.py with automatic driver imports
- Update tisbackup.py imports to use new structure
- Add pyvmomi>=8.0.0 as mandatory dependency
- Sync requirements.txt with pyproject.toml dependencies
- Add pylint>=3.0.0 and pytest-cov>=6.0.0 to dev dependencies
- Configure pylint and coverage tools in pyproject.toml
- Add conventional commits guidelines to CLAUDE.md
- Enhance .gitignore with comprehensive patterns for Python, IDEs, testing, and secrets
- Update CLAUDE.md documentation with new structure and tooling

Breaking Changes:
- Drivers must now be imported from libtisbackup.drivers instead of libtisbackup
- All backup driver files relocated to drivers/ subdirectory

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

libtisbackup/drivers/XenAPI.py

@@ -0,0 +1,264 @@
+# Copyright (c) Citrix Systems, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#  1) Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#
+#  2) Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# --------------------------------------------------------------------
+# Parts of this file are based upon xmlrpclib.py, the XML-RPC client
+# interface included in the Python distribution.
+#
+# Copyright (c) 1999-2002 by Secret Labs AB
+# Copyright (c) 1999-2002 by Fredrik Lundh
+#
+# By obtaining, using, and/or copying this software and/or its
+# associated documentation, you agree that you have read, understood,
+# and will comply with the following terms and conditions:
+#
+# Permission to use, copy, modify, and distribute this software and
+# its associated documentation for any purpose and without fee is
+# hereby granted, provided that the above copyright notice appears in
+# all copies, and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of
+# Secret Labs AB or the author not be used in advertising or publicity
+# pertaining to distribution of the software without specific, written
+# prior permission.
+#
+# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
+# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT-
+# ABILITY AND FITNESS.  IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR
+# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
+# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+# OF THIS SOFTWARE.
+# --------------------------------------------------------------------
+
+import gettext
+import socket
+import sys
+
+import six.moves.http_client as httplib
+import six.moves.xmlrpc_client as xmlrpclib
+
+translation = gettext.translation("xen-xm", fallback=True)
+
+API_VERSION_1_1 = "1.1"
+API_VERSION_1_2 = "1.2"
+
+
+class Failure(Exception):
+    def __init__(self, details):
+        self.details = details
+
+    def __str__(self):
+        try:
+            return str(self.details)
+        except Exception as exn:
+            msg = "Xen-API failure: %s" % exn
+            sys.stderr.write(msg)
+            return msg
+
+    def _details_map(self):
+        return dict([(str(i), self.details[i]) for i in range(len(self.details))])
+
+
+# Just a "constant" that we use to decide whether to retry the RPC
+_RECONNECT_AND_RETRY = object()
+
+
+class UDSHTTPConnection(httplib.HTTPConnection):
+    """HTTPConnection subclass to allow HTTP over Unix domain sockets."""
+
+    def connect(self):
+        path = self.host.replace("_", "/")
+        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        self.sock.connect(path)
+
+
+class UDSHTTP(httplib.HTTPConnection):
+    _connection_class = UDSHTTPConnection
+
+
+class UDSTransport(xmlrpclib.Transport):
+    def __init__(self, use_datetime=0):
+        self._use_datetime = use_datetime
+        self._extra_headers = []
+        self._connection = (None, None)
+
+    def add_extra_header(self, key, value):
+        self._extra_headers += [(key, value)]
+
+    def make_connection(self, host):
+        # Python 2.4 compatibility
+        if sys.version_info[0] <= 2 and sys.version_info[1] < 7:
+            return UDSHTTP(host)
+        else:
+            return UDSHTTPConnection(host)
+
+    def send_request(self, connection, handler, request_body):
+        connection.putrequest("POST", handler)
+        for key, value in self._extra_headers:
+            connection.putheader(key, value)
+
+
+class Session(xmlrpclib.ServerProxy):
+    """A server proxy and session manager for communicating with xapi using
+    the Xen-API.
+
+    Example:
+
+    session = Session('http://localhost/')
+    session.login_with_password('me', 'mypassword', '1.0', 'xen-api-scripts-xenapi.py')
+    session.xenapi.VM.start(vm_uuid)
+    session.xenapi.session.logout()
+    """
+
+    def __init__(self, uri, transport=None, encoding=None, verbose=0, allow_none=1, ignore_ssl=False):
+        # Fix for CA-172901 (+ Python 2.4 compatibility)
+        # Fix for context=ctx ( < Python 2.7.9 compatibility)
+        if not (sys.version_info[0] <= 2 and sys.version_info[1] <= 7 and sys.version_info[2] <= 9) and ignore_ssl:
+            import ssl
+
+            ctx = ssl._create_unverified_context()
+            xmlrpclib.ServerProxy.__init__(self, uri, transport, encoding, verbose, allow_none, context=ctx)
+        else:
+            xmlrpclib.ServerProxy.__init__(self, uri, transport, encoding, verbose, allow_none)
+        self.transport = transport
+        self._session = None
+        self.last_login_method = None
+        self.last_login_params = None
+        self.API_version = API_VERSION_1_1
+
+    def xenapi_request(self, methodname, params):
+        if methodname.startswith("login"):
+            self._login(methodname, params)
+            return None
+        elif methodname == "logout" or methodname == "session.logout":
+            self._logout()
+            return None
+        else:
+            retry_count = 0
+            while retry_count < 3:
+                full_params = (self._session,) + params
+                result = _parse_result(getattr(self, methodname)(*full_params))
+                if result is _RECONNECT_AND_RETRY:
+                    retry_count += 1
+                    if self.last_login_method:
+                        self._login(self.last_login_method, self.last_login_params)
+                    else:
+                        raise xmlrpclib.Fault(401, "You must log in")
+                else:
+                    return result
+            raise xmlrpclib.Fault(500, "Tried 3 times to get a valid session, but failed")
+
+    def _login(self, method, params):
+        try:
+            result = _parse_result(getattr(self, "session.%s" % method)(*params))
+            if result is _RECONNECT_AND_RETRY:
+                raise xmlrpclib.Fault(500, "Received SESSION_INVALID when logging in")
+            self._session = result
+            self.last_login_method = method
+            self.last_login_params = params
+            self.API_version = self._get_api_version()
+        except socket.error as e:
+            if e.errno == socket.errno.ETIMEDOUT:
+                raise xmlrpclib.Fault(504, "The connection timed out")
+            else:
+                raise e
+
+    def _logout(self):
+        try:
+            if self.last_login_method.startswith("slave_local"):
+                return _parse_result(self.session.local_logout(self._session))
+            else:
+                return _parse_result(self.session.logout(self._session))
+        finally:
+            self._session = None
+            self.last_login_method = None
+            self.last_login_params = None
+            self.API_version = API_VERSION_1_1
+
+    def _get_api_version(self):
+        pool = self.xenapi.pool.get_all()[0]
+        host = self.xenapi.pool.get_master(pool)
+        major = self.xenapi.host.get_API_version_major(host)
+        minor = self.xenapi.host.get_API_version_minor(host)
+        return "%s.%s" % (major, minor)
+
+    def __getattr__(self, name):
+        if name == "handle":
+            return self._session
+        elif name == "xenapi":
+            return _Dispatcher(self.API_version, self.xenapi_request, None)
+        elif name.startswith("login") or name.startswith("slave_local"):
+            return lambda *params: self._login(name, params)
+        elif name == "logout":
+            return _Dispatcher(self.API_version, self.xenapi_request, "logout")
+        else:
+            return xmlrpclib.ServerProxy.__getattr__(self, name)
+
+
+def xapi_local():
+    return Session("http://_var_lib_xcp_xapi/", transport=UDSTransport())
+
+
+def _parse_result(result):
+    if not isinstance(type(result), dict) or "Status" not in result:
+        raise xmlrpclib.Fault(500, "Missing Status in response from server" + result)
+    if result["Status"] == "Success":
+        if "Value" in result:
+            return result["Value"]
+        else:
+            raise xmlrpclib.Fault(500, "Missing Value in response from server")
+    else:
+        if "ErrorDescription" in result:
+            if result["ErrorDescription"][0] == "SESSION_INVALID":
+                return _RECONNECT_AND_RETRY
+            else:
+                raise Failure(result["ErrorDescription"])
+        else:
+            raise xmlrpclib.Fault(500, "Missing ErrorDescription in response from server")
+
+
+# Based upon _Method from xmlrpclib.
+class _Dispatcher:
+    def __init__(self, API_version, send, name):
+        self.__API_version = API_version
+        self.__send = send
+        self.__name = name
+
+    def __repr__(self):
+        if self.__name:
+            return "<XenAPI._Dispatcher for %s>" % self.__name
+        else:
+            return "<XenAPI._Dispatcher>"
+
+    def __getattr__(self, name):
+        if self.__name is None:
+            return _Dispatcher(self.__API_version, self.__send, name)
+        else:
+            return _Dispatcher(self.__API_version, self.__send, "%s.%s" % (self.__name, name))
+
+    def __call__(self, *args):
+        return self.__send(self.__name, args)

libtisbackup/drivers/__init__.py

@@ -0,0 +1,60 @@
+# -----------------------------------------------------------------------
+#    This file is part of TISBackup
+#
+#    TISBackup is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    TISBackup is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with TISBackup.  If not, see <http://www.gnu.org/licenses/>.
+#
+# -----------------------------------------------------------------------
+
+"""
+TISBackup drivers - Pluggable backup driver implementations.
+
+This package contains all backup driver implementations:
+- Database drivers (MySQL, PostgreSQL, Oracle, SQL Server)
+- File sync drivers (rsync, rsync+btrfs)
+- VM backup drivers (XenServer XVA, VMware VMDK)
+- Other drivers (Samba4, network switches, etc.)
+"""
+
+# Import all drivers to ensure they register themselves
+from .backup_mysql import backup_mysql
+from .backup_null import backup_null
+from .backup_oracle import backup_oracle
+from .backup_pgsql import backup_pgsql
+from .backup_rsync import backup_rsync, backup_rsync_ssh
+from .backup_rsync_btrfs import backup_rsync_btrfs, backup_rsync__btrfs_ssh
+from .backup_samba4 import backup_samba4
+from .backup_sqlserver import backup_sqlserver
+from .backup_switch import backup_switch
+from .backup_vmdk import backup_vmdk
+from .backup_xcp_metadata import backup_xcp_metadata
+from .backup_xva import backup_xva
+from .copy_vm_xcp import copy_vm_xcp
+
+__all__ = [
+    "backup_mysql",
+    "backup_null",
+    "backup_oracle",
+    "backup_pgsql",
+    "backup_rsync",
+    "backup_rsync_ssh",
+    "backup_rsync_btrfs",
+    "backup_rsync__btrfs_ssh",
+    "backup_samba4",
+    "backup_sqlserver",
+    "backup_switch",
+    "backup_vmdk",
+    "backup_xcp_metadata",
+    "backup_xva",
+    "copy_vm_xcp",
+]

libtisbackup/drivers/backup_mysql.py

@@ -0,0 +1,195 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------
+#    This file is part of TISBackup
+#
+#    TISBackup is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    TISBackup is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with TISBackup.  If not, see <http://www.gnu.org/licenses/>.
+#
+# -----------------------------------------------------------------------
+
+
+import sys
+
+try:
+    sys.stderr = open("/dev/null")  # Silence silly warnings from paramiko
+    import paramiko
+except ImportError as e:
+    print(("Error : can not load paramiko library %s" % e))
+    raise
+
+sys.stderr = sys.__stderr__
+
+from libtisbackup import *
+
+
+class backup_mysql(backup_generic):
+    """Backup a mysql database as gzipped sql file through ssh"""
+
+    type = "mysql+ssh"
+    required_params = backup_generic.required_params + ["db_user", "db_passwd", "private_key"]
+    optional_params = backup_generic.optional_params + ["db_name"]
+
+    db_name = ""
+    db_user = ""
+    db_passwd = ""
+
+    dest_dir = ""
+
+    def do_backup(self, stats):
+        self.dest_dir = os.path.join(self.backup_dir, self.backup_start_date)
+
+        if not os.path.isdir(self.dest_dir):
+            if not self.dry_run:
+                os.makedirs(self.dest_dir)
+            else:
+                print(('mkdir "%s"' % self.dest_dir))
+        else:
+            raise Exception("backup destination directory already exists : %s" % self.dest_dir)
+
+        self.logger.debug("[%s] Connecting to %s with user root and key %s", self.backup_name, self.server_name, self.private_key)
+        mykey = load_ssh_private_key(self.private_key)
+
+        self.ssh = paramiko.SSHClient()
+        self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        self.ssh.connect(self.server_name, username="root", pkey=mykey, port=self.ssh_port)
+
+        self.db_passwd = self.db_passwd.replace("$", r"\$")
+        if not self.db_name:
+            stats["log"] = "Successfully backuping processed to the following databases :"
+            stats["status"] = "List"
+            cmd = 'mysql -N  -B -p  -e "SHOW DATABASES;" -u ' + self.db_user + "  -p" + self.db_passwd + " 2> /dev/null"
+            self.logger.debug("[%s] List databases: %s", self.backup_name, cmd)
+            (error_code, output) = ssh_exec(cmd, ssh=self.ssh)
+            self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
+            if error_code:
+                raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
+            databases = output.split("\n")
+            for database in databases:
+                if database != "":
+                    self.db_name = database.rstrip()
+                    self.do_mysqldump(stats)
+
+        else:
+            stats["log"] = "Successfully backup processed to the following database :"
+            self.do_mysqldump(stats)
+
+    def do_mysqldump(self, stats):
+        t = datetime.datetime.now()
+        backup_start_date = t.strftime("%Y%m%d-%Hh%Mm%S")
+
+        # dump db
+        stats["status"] = "Dumping"
+        cmd = (
+            "mysqldump --single-transaction -u"
+            + self.db_user
+            + "  -p"
+            + self.db_passwd
+            + " "
+            + self.db_name
+            + " > /tmp/"
+            + self.db_name
+            + "-"
+            + backup_start_date
+            + ".sql"
+        )
+        self.logger.debug("[%s] Dump DB : %s", self.backup_name, cmd)
+        if not self.dry_run:
+            (error_code, output) = ssh_exec(cmd, ssh=self.ssh)
+            print(output)
+            self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
+            if error_code:
+                raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
+
+        # zip the file
+        stats["status"] = "Zipping"
+        cmd = "gzip /tmp/" + self.db_name + "-" + backup_start_date + ".sql"
+        self.logger.debug("[%s] Compress backup : %s", self.backup_name, cmd)
+        if not self.dry_run:
+            (error_code, output) = ssh_exec(cmd, ssh=self.ssh)
+            self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
+            if error_code:
+                raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
+
+        # get the file
+        stats["status"] = "SFTP"
+        filepath = "/tmp/" + self.db_name + "-" + backup_start_date + ".sql.gz"
+        localpath = os.path.join(self.dest_dir, self.db_name + ".sql.gz")
+        self.logger.debug("[%s] Get gz backup with sftp on %s from %s to %s", self.backup_name, self.server_name, filepath, localpath)
+        if not self.dry_run:
+            transport = self.ssh.get_transport()
+            sftp = paramiko.SFTPClient.from_transport(transport)
+            sftp.get(filepath, localpath)
+            sftp.close()
+
+        if not self.dry_run:
+            stats["total_files_count"] = 1 + stats.get("total_files_count", 0)
+            stats["written_files_count"] = 1 + stats.get("written_files_count", 0)
+            stats["total_bytes"] = os.stat(localpath).st_size + stats.get("total_bytes", 0)
+            stats["written_bytes"] = os.stat(localpath).st_size + stats.get("written_bytes", 0)
+        stats["log"] = '%s "%s"' % (stats["log"], self.db_name)
+        stats["backup_location"] = self.dest_dir
+
+        stats["status"] = "RMTemp"
+        cmd = "rm -f  /tmp/" + self.db_name + "-" + backup_start_date + ".sql.gz"
+        self.logger.debug("[%s] Remove temp gzip : %s", self.backup_name, cmd)
+        if not self.dry_run:
+            (error_code, output) = ssh_exec(cmd, ssh=self.ssh)
+            self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
+            if error_code:
+                raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
+        stats["status"] = "OK"
+
+    def register_existingbackups(self):
+        """scan backup dir and insert stats in database"""
+
+        registered = [
+            b["backup_location"]
+            for b in self.dbstat.query("select distinct backup_location from stats where backup_name=?", (self.backup_name,))
+        ]
+
+        filelist = os.listdir(self.backup_dir)
+        filelist.sort()
+        p = re.compile(r"^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}$")
+        for item in filelist:
+            if p.match(item):
+                dir_name = os.path.join(self.backup_dir, item)
+                if dir_name not in registered:
+                    start = datetime.datetime.strptime(item, "%Y%m%d-%Hh%Mm%S").isoformat()
+                    if fileisodate(dir_name) > start:
+                        stop = fileisodate(dir_name)
+                    else:
+                        stop = start
+                    self.logger.info("Registering %s started on %s", dir_name, start)
+                    self.logger.debug("  Disk usage %s", 'du -sb "%s"' % dir_name)
+                    if not self.dry_run:
+                        size_bytes = int(os.popen('du -sb "%s"' % dir_name).read().split("\t")[0])
+                    else:
+                        size_bytes = 0
+                    self.logger.debug("  Size in bytes : %i", size_bytes)
+                    if not self.dry_run:
+                        self.dbstat.add(
+                            self.backup_name,
+                            self.server_name,
+                            "",
+                            backup_start=start,
+                            backup_end=stop,
+                            status="OK",
+                            total_bytes=size_bytes,
+                            backup_location=dir_name,
+                        )
+                else:
+                    self.logger.info("Skipping %s, already registered", dir_name)
+
+
+register_driver(backup_mysql)

libtisbackup/drivers/backup_null.py

@@ -0,0 +1,58 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------
+#    This file is part of TISBackup
+#
+#    TISBackup is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    TISBackup is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with TISBackup.  If not, see <http://www.gnu.org/licenses/>.
+#
+# -----------------------------------------------------------------------
+
+import datetime
+import os
+
+from libtisbackup import *
+
+
+class backup_null(backup_generic):
+    """Null backup to register servers which don't need any backups
+    but we still want to know they are taken in account"""
+
+    type = "null"
+
+    required_params = ["type", "server_name", "backup_name"]
+    optional_params = []
+
+    def do_backup(self, stats):
+        pass
+
+    def process_backup(self):
+        pass
+
+    def cleanup_backup(self):
+        pass
+
+    def register_existingbackups(self):
+        pass
+
+    def export_latestbackup(self, destdir):
+        return {}
+
+    def checknagios(self, maxage_hours=30):
+        return (nagiosStateOk, "No backups needs to be performed")
+
+
+register_driver(backup_null)
+
+if __name__ == "__main__":
+    pass

libtisbackup/drivers/backup_oracle.py

@@ -0,0 +1,191 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------
+#    This file is part of TISBackup
+#
+#    TISBackup is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    TISBackup is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with TISBackup.  If not, see <http://www.gnu.org/licenses/>.
+#
+# -----------------------------------------------------------------------
+import sys
+
+try:
+    sys.stderr = open("/dev/null")  # Silence silly warnings from paramiko
+    import paramiko
+except ImportError as e:
+    print(("Error : can not load paramiko library %s" % e))
+    raise
+
+sys.stderr = sys.__stderr__
+
+import base64
+import datetime
+import os
+import re
+
+from libtisbackup import *
+
+
+class backup_oracle(backup_generic):
+    """Backup a oracle database as zipped file through ssh"""
+
+    type = "oracle+ssh"
+    required_params = backup_generic.required_params + ["db_name", "private_key", "userid"]
+    optional_params = ["username", "remote_backup_dir", "ignore_error_oracle_code"]
+    db_name = ""
+    username = "oracle"
+    remote_backup_dir = r"/home/oracle/backup"
+    ignore_error_oracle_code = []
+
+    def do_backup(self, stats):
+        self.logger.debug(
+            "[%s] Connecting to %s with user %s and key %s", self.backup_name, self.server_name, self.username, self.private_key
+        )
+        mykey = load_ssh_private_key(self.private_key)
+
+        self.ssh = paramiko.SSHClient()
+        self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        self.ssh.connect(self.server_name, username=self.username, pkey=mykey, port=self.ssh_port)
+
+        t = datetime.datetime.now()
+        self.backup_start_date = t.strftime("%Y%m%d-%Hh%Mm%S")
+        dumpfile = self.remote_backup_dir + "/" + self.db_name + "_" + self.backup_start_date + ".dmp"
+        dumplog = self.remote_backup_dir + "/" + self.db_name + "_" + self.backup_start_date + ".log"
+
+        self.dest_dir = os.path.join(self.backup_dir, self.backup_start_date)
+        if not os.path.isdir(self.dest_dir):
+            if not self.dry_run:
+                os.makedirs(self.dest_dir)
+            else:
+                print(('mkdir "%s"' % self.dest_dir))
+        else:
+            raise Exception("backup destination directory already exists : %s" % self.dest_dir)
+        # dump db
+        stats["status"] = "Dumping"
+        cmd = "exp '%s'  file='%s' grants=y log='%s'" % (self.userid, dumpfile, dumplog)
+        self.logger.debug("[%s] Dump DB : %s", self.backup_name, cmd)
+        if not self.dry_run:
+            (error_code, output) = ssh_exec(cmd, ssh=self.ssh)
+            self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
+            if error_code:
+                localpath = os.path.join(self.dest_dir, self.db_name + ".log")
+                self.logger.debug("[%s] Get log file with sftp on %s from %s to %s", self.backup_name, self.server_name, dumplog, localpath)
+                transport = self.ssh.get_transport()
+                sftp = paramiko.SFTPClient.from_transport(transport)
+                sftp.get(dumplog, localpath)
+                sftp.close()
+
+                file = open(localpath)
+                for line in file:
+                    if (
+                        re.search("EXP-[0-9]+:", line)
+                        and re.match("EXP-[0-9]+:", line).group(0).replace(":", "") not in self.ignore_error_oracle_code
+                    ):
+                        stats["status"] = "RMTemp"
+                        self.clean_dumpfiles(dumpfile, dumplog)
+                        raise Exception(
+                            'Aborting, Not null exit code (%s) for "%s"' % (re.match("EXP-[0-9]+:", line).group(0).replace(":", ""), cmd)
+                        )
+                file.close()
+
+        # zip the file
+        stats["status"] = "Zipping"
+        cmd = "gzip  %s" % dumpfile
+        self.logger.debug("[%s] Compress backup : %s", self.backup_name, cmd)
+        if not self.dry_run:
+            (error_code, output) = ssh_exec(cmd, ssh=self.ssh)
+            self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
+        if error_code:
+            raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
+
+        # get the file
+        stats["status"] = "SFTP"
+        filepath = dumpfile + ".gz"
+        localpath = os.path.join(self.dest_dir, self.db_name + ".dmp.gz")
+        self.logger.debug("[%s] Get gz backup with sftp on %s from %s to %s", self.backup_name, self.server_name, filepath, localpath)
+        if not self.dry_run:
+            transport = self.ssh.get_transport()
+            sftp = paramiko.SFTPClient.from_transport(transport)
+            sftp.get(filepath, localpath)
+            sftp.close()
+
+        if not self.dry_run:
+            stats["total_files_count"] = 1
+            stats["written_files_count"] = 1
+            stats["total_bytes"] = os.stat(localpath).st_size
+            stats["written_bytes"] = os.stat(localpath).st_size
+        stats["log"] = "gzip dump of DB %s:%s (%d bytes) to %s" % (self.server_name, self.db_name, stats["written_bytes"], localpath)
+        stats["backup_location"] = self.dest_dir
+        stats["status"] = "RMTemp"
+        self.clean_dumpfiles(dumpfile, dumplog)
+        stats["status"] = "OK"
+
+    def register_existingbackups(self):
+        """scan backup dir and insert stats in database"""
+
+        registered = [
+            b["backup_location"]
+            for b in self.dbstat.query("select distinct backup_location from stats where backup_name=?", (self.backup_name,))
+        ]
+
+        filelist = os.listdir(self.backup_dir)
+        filelist.sort()
+        p = re.compile(r"^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}$")
+        for item in filelist:
+            if p.match(item):
+                dir_name = os.path.join(self.backup_dir, item)
+                if dir_name not in registered:
+                    start = datetime.datetime.strptime(item, "%Y%m%d-%Hh%Mm%S").isoformat()
+                    if fileisodate(dir_name) > start:
+                        stop = fileisodate(dir_name)
+                    else:
+                        stop = start
+                    self.logger.info("Registering %s started on %s", dir_name, start)
+                    self.logger.debug("  Disk usage %s", 'du -sb "%s"' % dir_name)
+                    if not self.dry_run:
+                        size_bytes = int(os.popen('du -sb "%s"' % dir_name).read().split("\t")[0])
+                    else:
+                        size_bytes = 0
+                    self.logger.debug("  Size in bytes : %i", size_bytes)
+                    if not self.dry_run:
+                        self.dbstat.add(
+                            self.backup_name,
+                            self.server_name,
+                            "",
+                            backup_start=start,
+                            backup_end=stop,
+                            status="OK",
+                            total_bytes=size_bytes,
+                            backup_location=dir_name,
+                        )
+                else:
+                    self.logger.info("Skipping %s, already registered", dir_name)
+
+    def clean_dumpfiles(self, dumpfile, dumplog):
+        cmd = 'rm -f "%s.gz" "%s"' % (dumpfile, dumplog)
+        self.logger.debug("[%s] Remove temp gzip : %s", self.backup_name, cmd)
+        if not self.dry_run:
+            (error_code, output) = ssh_exec(cmd, ssh=self.ssh)
+            self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
+            if error_code:
+                raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
+        cmd = "rm -f " + self.remote_backup_dir + "/" + self.db_name + "_" + self.backup_start_date + ".dmp"
+        self.logger.debug("[%s] Remove temp dump : %s", self.backup_name, cmd)
+        if not self.dry_run:
+            (error_code, output) = ssh_exec(cmd, ssh=self.ssh)
+            self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
+            if error_code:
+                raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
+
+
+register_driver(backup_oracle)

libtisbackup/drivers/backup_pgsql.py

@@ -0,0 +1,173 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------
+#    This file is part of TISBackup
+#
+#    TISBackup is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    TISBackup is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with TISBackup.  If not, see <http://www.gnu.org/licenses/>.
+#
+# -----------------------------------------------------------------------
+import sys
+
+try:
+    sys.stderr = open("/dev/null")  # Silence silly warnings from paramiko
+    import paramiko
+except ImportError as e:
+    print(("Error : can not load paramiko library %s" % e))
+    raise
+
+sys.stderr = sys.__stderr__
+
+from libtisbackup import *
+
+
+class backup_pgsql(backup_generic):
+    """Backup a postgresql database as gzipped sql file through ssh"""
+
+    type = "pgsql+ssh"
+    required_params = backup_generic.required_params + ["private_key"]
+    optional_params = backup_generic.optional_params + ["db_name", "tmp_dir", "encoding"]
+
+    db_name = ""
+    tmp_dir = "/tmp"
+    encoding = "UTF8"
+
+    def do_backup(self, stats):
+        self.dest_dir = os.path.join(self.backup_dir, self.backup_start_date)
+
+        if not os.path.isdir(self.dest_dir):
+            if not self.dry_run:
+                os.makedirs(self.dest_dir)
+            else:
+                print(('mkdir "%s"' % self.dest_dir))
+        else:
+            raise Exception("backup destination directory already exists : %s" % self.dest_dir)
+
+        mykey = load_ssh_private_key(self.private_key)
+
+        self.logger.debug(
+            '[%s] Trying to connect to "%s" with username root and key "%s"', self.backup_name, self.server_name, self.private_key
+        )
+        self.ssh = paramiko.SSHClient()
+        self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        self.ssh.connect(self.server_name, username="root", pkey=mykey, port=self.ssh_port)
+
+        if self.db_name:
+            stats["log"] = "Successfully backup processed to the following database :"
+            self.do_pgsqldump(stats)
+        else:
+            stats["log"] = "Successfully backuping processed to the following databases :"
+            stats["status"] = "List"
+            cmd = """su -  postgres -c 'psql -A -t  -c "SELECT datname FROM pg_database WHERE datistemplate = false;"' 2> /dev/null"""
+            self.logger.debug("[%s] List databases: %s", self.backup_name, cmd)
+            (error_code, output) = ssh_exec(cmd, ssh=self.ssh)
+            self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
+            if error_code:
+                raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
+            databases = output.split("\n")
+            for database in databases:
+                if database.strip() not in ("", "template0", "template1"):
+                    self.db_name = database.strip()
+                    self.do_pgsqldump(stats)
+
+        stats["status"] = "OK"
+
+    def do_pgsqldump(self, stats):
+        t = datetime.datetime.now()
+        backup_start_date = t.strftime("%Y%m%d-%Hh%Mm%S")
+        params = {
+            "encoding": self.encoding,
+            "db_name": self.db_name,
+            "tmp_dir": self.tmp_dir,
+            "dest_dir": self.dest_dir,
+            "backup_start_date": backup_start_date,
+        }
+        # dump db
+        filepath = "%(tmp_dir)s/%(db_name)s-%(backup_start_date)s.sql.gz" % params
+        cmd = "su -  postgres -c 'pg_dump -E %(encoding)s -Z9 %(db_name)s'" % params
+        cmd += " > " + filepath
+        self.logger.debug("[%s] %s ", self.backup_name, cmd)
+        if not self.dry_run:
+            (error_code, output) = ssh_exec(cmd, ssh=self.ssh)
+            self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
+            if error_code:
+                raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
+
+        # get the file
+        localpath = "%(dest_dir)s/%(db_name)s-%(backup_start_date)s.sql.gz" % params
+        self.logger.debug('[%s] get the file using sftp from "%s" to "%s" ', self.backup_name, filepath, localpath)
+        if not self.dry_run:
+            transport = self.ssh.get_transport()
+            sftp = paramiko.SFTPClient.from_transport(transport)
+            sftp.get(filepath, localpath)
+            sftp.close()
+
+        if not self.dry_run:
+            stats["total_files_count"] = 1 + stats.get("total_files_count", 0)
+            stats["written_files_count"] = 1 + stats.get("written_files_count", 0)
+            stats["total_bytes"] = os.stat(localpath).st_size + stats.get("total_bytes", 0)
+            stats["written_bytes"] = os.stat(localpath).st_size + stats.get("written_bytes", 0)
+        stats["log"] = '%s "%s"' % (stats["log"], self.db_name)
+        stats["backup_location"] = self.dest_dir
+
+        cmd = "rm -f %(tmp_dir)s/%(db_name)s-%(backup_start_date)s.sql.gz" % params
+        self.logger.debug("[%s] %s ", self.backup_name, cmd)
+        if not self.dry_run:
+            (error_code, output) = ssh_exec(cmd, ssh=self.ssh)
+            self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
+            if error_code:
+                raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
+
+    def register_existingbackups(self):
+        """scan backup dir and insert stats in database"""
+
+        registered = [
+            b["backup_location"]
+            for b in self.dbstat.query("select distinct backup_location from stats where backup_name=?", (self.backup_name,))
+        ]
+
+        filelist = os.listdir(self.backup_dir)
+        filelist.sort()
+        p = re.compile(r"^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}$")
+        for item in filelist:
+            if p.match(item):
+                dir_name = os.path.join(self.backup_dir, item)
+                if dir_name not in registered:
+                    start = datetime.datetime.strptime(item, "%Y%m%d-%Hh%Mm%S").isoformat()
+                    if fileisodate(dir_name) > start:
+                        stop = fileisodate(dir_name)
+                    else:
+                        stop = start
+                    self.logger.info("Registering %s started on %s", dir_name, start)
+                    self.logger.debug("  Disk usage %s", 'du -sb "%s"' % dir_name)
+                    if not self.dry_run:
+                        size_bytes = int(os.popen('du -sb "%s"' % dir_name).read().split("\t")[0])
+                    else:
+                        size_bytes = 0
+                    self.logger.debug("  Size in bytes : %i", size_bytes)
+                    if not self.dry_run:
+                        self.dbstat.add(
+                            self.backup_name,
+                            self.server_name,
+                            "",
+                            backup_start=start,
+                            backup_end=stop,
+                            status="OK",
+                            total_bytes=size_bytes,
+                            backup_location=dir_name,
+                        )
+                else:
+                    self.logger.info("Skipping %s, already registered", dir_name)
+
+
+register_driver(backup_pgsql)

libtisbackup/drivers/backup_rsync.py

@@ -0,0 +1,378 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------
+#    This file is part of TISBackup
+#
+#    TISBackup is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    TISBackup is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with TISBackup.  If not, see <http://www.gnu.org/licenses/>.
+#
+# -----------------------------------------------------------------------
+
+import datetime
+import logging
+import os
+import os.path
+import re
+import time
+
+from libtisbackup import *
+
+
+class backup_rsync(backup_generic):
+    """Backup a directory on remote server with rsync and rsync protocol (requires running remote rsync daemon)"""
+
+    type = "rsync"
+    required_params = backup_generic.required_params + ["remote_user", "remote_dir", "rsync_module", "password_file"]
+    optional_params = backup_generic.optional_params + [
+        "compressionlevel",
+        "compression",
+        "bwlimit",
+        "exclude_list",
+        "protect_args",
+        "overload_args",
+    ]
+
+    remote_user = "root"
+    remote_dir = ""
+
+    exclude_list = ""
+    rsync_module = ""
+    password_file = ""
+    compression = ""
+    bwlimit = 0
+    protect_args = "1"
+    overload_args = None
+    compressionlevel = 0
+
+    def read_config(self, iniconf):
+        assert isinstance(iniconf, ConfigParser)
+        backup_generic.read_config(self, iniconf)
+        if not self.bwlimit and iniconf.has_option("global", "bw_limit"):
+            self.bwlimit = iniconf.getint("global", "bw_limit")
+        if not self.compressionlevel and iniconf.has_option("global", "compression_level"):
+            self.compressionlevel = iniconf.getint("global", "compression_level")
+
+    def do_backup(self, stats):
+        if not self.set_lock():
+            self.logger.error("[%s] a lock file is set, a backup maybe already running!!", self.backup_name)
+            return False
+
+        try:
+            try:
+                backup_source = "undefined"
+                dest_dir = os.path.join(self.backup_dir, self.backup_start_date + ".rsync/")
+                if not os.path.isdir(dest_dir):
+                    if not self.dry_run:
+                        os.makedirs(dest_dir)
+                    else:
+                        print(('mkdir "%s"' % dest_dir))
+                else:
+                    raise Exception("backup destination directory already exists : %s" % dest_dir)
+
+                options = ["-rt", "--stats", "--delete-excluded", "--numeric-ids", "--delete-after"]
+                if self.logger.level:
+                    options.append("-P")
+
+                if self.dry_run:
+                    options.append("-d")
+
+                if self.overload_args is not None:
+                    options.append(self.overload_args)
+                elif "cygdrive" not in self.remote_dir:
+                    # we don't preserve owner, group, links, hardlinks, perms for windows/cygwin as it is not reliable nor useful
+                    options.append("-lpgoD")
+
+                # the protect-args option is not available in all rsync version
+                if self.protect_args.lower() not in ("false", "no", "0"):
+                    options.append("--protect-args")
+
+                if self.compression.lower() in ("true", "yes", "1"):
+                    options.append("-z")
+
+                if self.compressionlevel:
+                    options.append("--compress-level=%s" % self.compressionlevel)
+
+                if self.bwlimit:
+                    options.append("--bwlimit %s" % self.bwlimit)
+
+                latest = self.get_latest_backup(self.backup_start_date)
+                if latest:
+                    options.extend(['--link-dest="%s"' % os.path.join("..", b, "") for b in latest])
+
+                def strip_quotes(s):
+                    if s[0] == '"':
+                        s = s[1:]
+                    if s[-1] == '"':
+                        s = s[:-1]
+                    return s
+
+                # Add excludes
+                if "--exclude" in self.exclude_list:
+                    # old settings with exclude_list=--exclude toto --exclude=titi
+                    excludes = [
+                        strip_quotes(s).strip() for s in self.exclude_list.replace("--exclude=", "").replace("--exclude ", "").split()
+                    ]
+                else:
+                    try:
+                        # newsettings with exclude_list='too','titi', parsed as a str python list content
+                        excludes = eval("[%s]" % self.exclude_list)
+                    except Exception as e:
+                        raise Exception(
+                            "Error reading exclude list : value %s, eval error %s (don't forget quotes and comma...)"
+                            % (self.exclude_list, e)
+                        )
+                options.extend(['--exclude="%s"' % x for x in excludes])
+
+                if self.rsync_module and not self.password_file:
+                    raise Exception("You must specify a password file if you specify a rsync module")
+
+                if not self.rsync_module and not self.private_key:
+                    raise Exception("If you don" "t use SSH, you must specify a rsync module")
+
+                # rsync_re = re.compile(r'(?P<server>[^:]*)::(?P<export>[^/]*)/(?P<path>.*)')
+                # ssh_re = re.compile(r'((?P<user>.*)@)?(?P<server>[^:]*):(?P<path>/.*)')
+
+                # Add ssh connection params
+                if self.rsync_module:
+                    # Case of rsync exports
+                    if self.password_file:
+                        options.append('--password-file="%s"' % self.password_file)
+                    backup_source = "%s@%s::%s%s" % (self.remote_user, self.server_name, self.rsync_module, self.remote_dir)
+                else:
+                    # case of rsync + ssh
+                    ssh_params = ["-o StrictHostKeyChecking=no"]
+                    ssh_params.append("-o BatchMode=yes")
+
+                    if self.private_key:
+                        ssh_params.append("-i %s" % self.private_key)
+                    if self.cipher_spec:
+                        ssh_params.append("-c %s" % self.cipher_spec)
+                    if self.ssh_port != 22:
+                        ssh_params.append("-p %i" % self.ssh_port)
+                    options.append('-e "/usr/bin/ssh %s"' % (" ".join(ssh_params)))
+                    backup_source = "%s@%s:%s" % (self.remote_user, self.server_name, self.remote_dir)
+
+                # ensure there is a slash at end
+                if backup_source[-1] != "/":
+                    backup_source += "/"
+
+                options_params = " ".join(options)
+
+                cmd = "/usr/bin/rsync %s %s %s 2>&1" % (options_params, backup_source, dest_dir)
+                self.logger.debug("[%s] rsync : %s", self.backup_name, cmd)
+
+                if not self.dry_run:
+                    self.line = ""
+                    process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
+
+                    def ondata(data, context):
+                        if context.verbose:
+                            print(data)
+                        context.logger.debug(data)
+
+                    log = monitor_stdout(process, ondata, self)
+
+                    reg_total_files = re.compile(r"Number of files: (?P<file>\d+)")
+                    reg_transferred_files = re.compile(r"Number of .*files transferred: (?P<file>\d+)")
+                    for l in log.splitlines():
+                        line = l.replace(",", "")
+                        m = reg_total_files.match(line)
+                        if m:
+                            stats["total_files_count"] += int(m.groupdict()["file"])
+                        m = reg_transferred_files.match(line)
+                        if m:
+                            stats["written_files_count"] += int(m.groupdict()["file"])
+                        if line.startswith("Total file size:"):
+                            stats["total_bytes"] += int(line.split(":")[1].split()[0])
+                        if line.startswith("Total transferred file size:"):
+                            stats["written_bytes"] += int(line.split(":")[1].split()[0])
+
+                    returncode = process.returncode
+                    ## deal with exit code 24 (file vanished)
+                    if returncode == 24:
+                        self.logger.warning("[" + self.backup_name + "] Note: some files vanished before transfer")
+                    elif returncode == 23:
+                        self.logger.warning("[" + self.backup_name + "] unable so set uid on some files")
+                    elif returncode != 0:
+                        self.logger.error("[" + self.backup_name + "] shell program exited with error code " + str(returncode))
+                        raise Exception(
+                            "[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd, log[-512:]
+                        )
+                else:
+                    print(cmd)
+
+                # we suppress the .rsync suffix if everything went well
+                finaldest = os.path.join(self.backup_dir, self.backup_start_date)
+                self.logger.debug("[%s] renaming target directory from %s to %s", self.backup_name, dest_dir, finaldest)
+                if not self.dry_run:
+                    os.rename(dest_dir, finaldest)
+                    self.logger.debug("[%s] touching datetime of target directory %s", self.backup_name, finaldest)
+                    print((os.popen('touch "%s"' % finaldest).read()))
+                else:
+                    print(("mv", dest_dir, finaldest))
+                stats["backup_location"] = finaldest
+                stats["status"] = "OK"
+                stats["log"] = "ssh+rsync backup from %s OK, %d bytes written for %d changed files" % (
+                    backup_source,
+                    stats["written_bytes"],
+                    stats["written_files_count"],
+                )
+
+            except BaseException as e:
+                stats["status"] = "ERROR"
+                stats["log"] = str(e)
+                raise
+
+        finally:
+            self.remove_lock()
+
+    def get_latest_backup(self, current):
+        result = []
+        filelist = os.listdir(self.backup_dir)
+        filelist.sort()
+        filelist.reverse()
+        # full = ''
+        r_full = re.compile(r"^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}$")
+        r_partial = re.compile(r"^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}.rsync$")
+        # we take all latest partials younger than the latest full and the latest full
+        for item in filelist:
+            if r_partial.match(item) and item < current:
+                result.append(item)
+            elif r_full.match(item) and item < current:
+                result.append(item)
+                break
+
+        return result
+
+    def register_existingbackups(self):
+        """scan backup dir and insert stats in database"""
+
+        registered = [
+            b["backup_location"]
+            for b in self.dbstat.query("select distinct backup_location from stats where backup_name=?", (self.backup_name,))
+        ]
+
+        filelist = os.listdir(self.backup_dir)
+        filelist.sort()
+        p = re.compile(r"^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}$")
+        for item in filelist:
+            if p.match(item):
+                dir_name = os.path.join(self.backup_dir, item)
+                if dir_name not in registered:
+                    start = datetime.datetime.strptime(item, "%Y%m%d-%Hh%Mm%S").isoformat()
+                    if fileisodate(dir_name) > start:
+                        stop = fileisodate(dir_name)
+                    else:
+                        stop = start
+                    self.logger.info("Registering %s started on %s", dir_name, start)
+                    self.logger.debug("  Disk usage %s", 'du -sb "%s"' % dir_name)
+                    if not self.dry_run:
+                        size_bytes = int(os.popen('du -sb "%s"' % dir_name).read().split("\t")[0])
+                    else:
+                        size_bytes = 0
+                    self.logger.debug("  Size in bytes : %i", size_bytes)
+                    if not self.dry_run:
+                        self.dbstat.add(
+                            self.backup_name,
+                            self.server_name,
+                            "",
+                            backup_start=start,
+                            backup_end=stop,
+                            status="OK",
+                            total_bytes=size_bytes,
+                            backup_location=dir_name,
+                        )
+                else:
+                    self.logger.info("Skipping %s, already registered", dir_name)
+
+    def is_pid_still_running(self, lockfile):
+        f = open(lockfile)
+        lines = f.readlines()
+        f.close()
+        if len(lines) == 0:
+            self.logger.info("[" + self.backup_name + "] empty lock file, removing...")
+            return False
+
+        for line in lines:
+            if line.startswith("pid="):
+                pid = line.split("=")[1].strip()
+                if os.path.exists("/proc/" + pid):
+                    self.logger.info("[" + self.backup_name + "] process still there")
+                    return True
+                else:
+                    self.logger.info("[" + self.backup_name + "] process not there anymore remove lock")
+                    return False
+            else:
+                self.logger.info("[" + self.backup_name + "] incorrrect lock file : no pid line")
+                return False
+
+    def set_lock(self):
+        self.logger.debug("[" + self.backup_name + "] setting lock")
+
+        # TODO: improve for race condition
+        # TODO: also check if process is really there
+        if os.path.isfile(self.backup_dir + "/lock"):
+            self.logger.debug("[" + self.backup_name + "] File " + self.backup_dir + "/lock already exist")
+            if not self.is_pid_still_running(self.backup_dir + "/lock"):
+                self.logger.info("[" + self.backup_name + "] removing lock file " + self.backup_dir + "/lock")
+                os.unlink(self.backup_dir + "/lock")
+            else:
+                return False
+
+        lockfile = open(self.backup_dir + "/lock", "w")
+        # Write all the lines at once:
+        lockfile.write("pid=" + str(os.getpid()))
+        lockfile.write("\nbackup_time=" + self.backup_start_date)
+        lockfile.close()
+        return True
+
+    def remove_lock(self):
+        self.logger.debug("[%s] removing lock", self.backup_name)
+        os.unlink(self.backup_dir + "/lock")
+
+
+class backup_rsync_ssh(backup_rsync):
+    """Backup a directory on remote server with rsync and ssh protocol (requires rsync software on remote host)"""
+
+    type = "rsync+ssh"
+    required_params = backup_generic.required_params + ["remote_user", "remote_dir", "private_key"]
+    optional_params = backup_generic.optional_params + [
+        "compression",
+        "bwlimit",
+        "ssh_port",
+        "exclude_list",
+        "protect_args",
+        "overload_args",
+        "cipher_spec",
+    ]
+    cipher_spec = ""
+
+
+register_driver(backup_rsync)
+register_driver(backup_rsync_ssh)
+
+if __name__ == "__main__":
+    logger = logging.getLogger("tisbackup")
+    logger.setLevel(logging.DEBUG)
+    formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+    cp = ConfigParser()
+    cp.read("/opt/tisbackup/configtest.ini")
+    dbstat = BackupStat("/backup/data/log/tisbackup.sqlite")
+    b = backup_rsync("htouvet", "/backup/data/htouvet", dbstat)
+    b.read_config(cp)
+    b.process_backup()
+    print((b.checknagios()))

libtisbackup/drivers/backup_rsync_btrfs.py

@@ -0,0 +1,395 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------
+#    This file is part of TISBackup
+#
+#    TISBackup is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    TISBackup is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with TISBackup.  If not, see <http://www.gnu.org/licenses/>.
+#
+# -----------------------------------------------------------------------
+
+import datetime
+import logging
+import os
+import os.path
+import re
+import time
+
+from libtisbackup import *
+
+
+class backup_rsync_btrfs(backup_generic):
+    """Backup a directory on remote server with rsync and btrfs protocol (requires running remote rsync daemon)"""
+
+    type = "rsync+btrfs"
+    required_params = backup_generic.required_params + ["remote_user", "remote_dir", "rsync_module", "password_file"]
+    optional_params = backup_generic.optional_params + [
+        "compressionlevel",
+        "compression",
+        "bwlimit",
+        "exclude_list",
+        "protect_args",
+        "overload_args",
+    ]
+
+    remote_user = "root"
+    remote_dir = ""
+
+    exclude_list = ""
+    rsync_module = ""
+    password_file = ""
+    compression = ""
+    bwlimit = 0
+    protect_args = "1"
+    overload_args = None
+    compressionlevel = 0
+
+    def read_config(self, iniconf):
+        assert isinstance(iniconf, ConfigParser)
+        backup_generic.read_config(self, iniconf)
+        if not self.bwlimit and iniconf.has_option("global", "bw_limit"):
+            self.bwlimit = iniconf.getint("global", "bw_limit")
+        if not self.compressionlevel and iniconf.has_option("global", "compression_level"):
+            self.compressionlevel = iniconf.getint("global", "compression_level")
+
+    def do_backup(self, stats):
+        if not self.set_lock():
+            self.logger.error("[%s] a lock file is set, a backup maybe already running!!", self.backup_name)
+            return False
+
+        try:
+            try:
+                backup_source = "undefined"
+                dest_dir = os.path.join(self.backup_dir, "last_backup")
+                if not os.path.isdir(dest_dir):
+                    if not self.dry_run:
+                        cmd = "/bin/btrfs subvolume create %s" % dest_dir
+                        process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
+                        log = monitor_stdout(process, "", self)
+                        returncode = process.returncode
+                        if returncode != 0:
+                            self.logger.error("[" + self.backup_name + "] shell program exited with error code: %s" % log)
+                            raise Exception("[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd)
+                        else:
+                            self.logger.info("[" + self.backup_name + "] create btrs volume: %s" % dest_dir)
+                    else:
+                        print(('btrfs subvolume create "%s"' % dest_dir))
+
+                options = ["-rt", "--stats", "--delete-excluded", "--numeric-ids", "--delete-after"]
+                if self.logger.level:
+                    options.append("-P")
+
+                if self.dry_run:
+                    options.append("-d")
+
+                if self.overload_args is not None:
+                    options.append(self.overload_args)
+                elif "cygdrive" not in self.remote_dir:
+                    # we don't preserve owner, group, links, hardlinks, perms for windows/cygwin as it is not reliable nor useful
+                    options.append("-lpgoD")
+
+                # the protect-args option is not available in all rsync version
+                if self.protect_args.lower() not in ("false", "no", "0"):
+                    options.append("--protect-args")
+
+                if self.compression.lower() in ("true", "yes", "1"):
+                    options.append("-z")
+
+                if self.compressionlevel:
+                    options.append("--compress-level=%s" % self.compressionlevel)
+
+                if self.bwlimit:
+                    options.append("--bwlimit %s" % self.bwlimit)
+
+                # latest = self.get_latest_backup(self.backup_start_date)
+                # remove link-dest replace by btrfs
+                # if latest:
+                #    options.extend(['--link-dest="%s"' % os.path.join('..',b,'') for b in latest])
+
+                def strip_quotes(s):
+                    if s[0] == '"':
+                        s = s[1:]
+                    if s[-1] == '"':
+                        s = s[:-1]
+                    return s
+
+                # Add excludes
+                if "--exclude" in self.exclude_list:
+                    # old settings with exclude_list=--exclude toto --exclude=titi
+                    excludes = [
+                        strip_quotes(s).strip() for s in self.exclude_list.replace("--exclude=", "").replace("--exclude ", "").split()
+                    ]
+                else:
+                    try:
+                        # newsettings with exclude_list='too','titi', parsed as a str python list content
+                        excludes = eval("[%s]" % self.exclude_list)
+                    except Exception as e:
+                        raise Exception(
+                            "Error reading exclude list : value %s, eval error %s (don't forget quotes and comma...)"
+                            % (self.exclude_list, e)
+                        )
+                options.extend(['--exclude="%s"' % x for x in excludes])
+
+                if self.rsync_module and not self.password_file:
+                    raise Exception("You must specify a password file if you specify a rsync module")
+
+                if not self.rsync_module and not self.private_key:
+                    raise Exception("If you don" "t use SSH, you must specify a rsync module")
+
+                # rsync_re = re.compile(r'(?P<server>[^:]*)::(?P<export>[^/]*)/(?P<path>.*)')
+                # ssh_re = re.compile(r'((?P<user>.*)@)?(?P<server>[^:]*):(?P<path>/.*)')
+
+                # Add ssh connection params
+                if self.rsync_module:
+                    # Case of rsync exports
+                    if self.password_file:
+                        options.append('--password-file="%s"' % self.password_file)
+                    backup_source = "%s@%s::%s%s" % (self.remote_user, self.server_name, self.rsync_module, self.remote_dir)
+                else:
+                    # case of rsync + ssh
+                    ssh_params = ["-o StrictHostKeyChecking=no"]
+                    if self.private_key:
+                        ssh_params.append("-i %s" % self.private_key)
+                    if self.cipher_spec:
+                        ssh_params.append("-c %s" % self.cipher_spec)
+                    if self.ssh_port != 22:
+                        ssh_params.append("-p %i" % self.ssh_port)
+                    options.append('-e "/usr/bin/ssh %s"' % (" ".join(ssh_params)))
+                    backup_source = "%s@%s:%s" % (self.remote_user, self.server_name, self.remote_dir)
+
+                # ensure there is a slash at end
+                if backup_source[-1] != "/":
+                    backup_source += "/"
+
+                options_params = " ".join(options)
+
+                cmd = "/usr/bin/rsync %s %s %s 2>&1" % (options_params, backup_source, dest_dir)
+                self.logger.debug("[%s] rsync : %s", self.backup_name, cmd)
+
+                if not self.dry_run:
+                    self.line = ""
+                    process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
+
+                    def ondata(data, context):
+                        if context.verbose:
+                            print(data)
+                        context.logger.debug(data)
+
+                    log = monitor_stdout(process, ondata, self)
+
+                    reg_total_files = re.compile(r"Number of files: (?P<file>\d+)")
+                    reg_transferred_files = re.compile(r"Number of .*files transferred: (?P<file>\d+)")
+                    for l in log.splitlines():
+                        line = l.replace(",", "")
+                        m = reg_total_files.match(line)
+                        if m:
+                            stats["total_files_count"] += int(m.groupdict()["file"])
+                        m = reg_transferred_files.match(line)
+                        if m:
+                            stats["written_files_count"] += int(m.groupdict()["file"])
+                        if line.startswith("Total file size:"):
+                            stats["total_bytes"] += int(line.split(":")[1].split()[0])
+                        if line.startswith("Total transferred file size:"):
+                            stats["written_bytes"] += int(line.split(":")[1].split()[0])
+
+                    returncode = process.returncode
+                    ## deal with exit code 24 (file vanished)
+                    if returncode == 24:
+                        self.logger.warning("[" + self.backup_name + "] Note: some files vanished before transfer")
+                    elif returncode == 23:
+                        self.logger.warning("[" + self.backup_name + "] unable so set uid on some files")
+                    elif returncode != 0:
+                        self.logger.error("[" + self.backup_name + "] shell program exited with error code ", str(returncode))
+                        raise Exception(
+                            "[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd, log[-512:]
+                        )
+                else:
+                    print(cmd)
+
+                # we take a snapshot of last_backup if everything went well
+                finaldest = os.path.join(self.backup_dir, self.backup_start_date)
+                self.logger.debug("[%s] snapshoting last_backup directory from %s to %s", self.backup_name, dest_dir, finaldest)
+                if not os.path.isdir(finaldest):
+                    if not self.dry_run:
+                        cmd = "/bin/btrfs subvolume snapshot %s %s" % (dest_dir, finaldest)
+                        process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
+                        log = monitor_stdout(process, "", self)
+                        returncode = process.returncode
+                        if returncode != 0:
+                            self.logger.error("[" + self.backup_name + "] shell program exited with error code " + str(returncode))
+                            raise Exception(
+                                "[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd, log[-512:]
+                            )
+                        else:
+                            self.logger.info("[" + self.backup_name + "] snapshot directory created %s" % finaldest)
+                    else:
+                        print(("btrfs snapshot of %s to %s" % (dest_dir, finaldest)))
+                else:
+                    raise Exception("snapshot directory already exists : %s" % finaldest)
+                self.logger.debug("[%s] touching datetime of target directory %s", self.backup_name, finaldest)
+                print((os.popen('touch "%s"' % finaldest).read()))
+                stats["backup_location"] = finaldest
+                stats["status"] = "OK"
+                stats["log"] = "ssh+rsync+btrfs backup from %s OK, %d bytes written for %d changed files" % (
+                    backup_source,
+                    stats["written_bytes"],
+                    stats["written_files_count"],
+                )
+
+            except BaseException as e:
+                stats["status"] = "ERROR"
+                stats["log"] = str(e)
+                raise
+
+        finally:
+            self.remove_lock()
+
+    def get_latest_backup(self, current):
+        result = []
+        filelist = os.listdir(self.backup_dir)
+        filelist.sort()
+        filelist.reverse()
+        # full = ''
+        r_full = re.compile(r"^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}$")
+        r_partial = re.compile(r"^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}.rsync$")
+        # we take all latest partials younger than the latest full and the latest full
+        for item in filelist:
+            if r_partial.match(item) and item < current:
+                result.append(item)
+            elif r_full.match(item) and item < current:
+                result.append(item)
+                break
+        return result
+
+    def register_existingbackups(self):
+        """scan backup dir and insert stats in database"""
+
+        registered = [
+            b["backup_location"]
+            for b in self.dbstat.query("select distinct backup_location from stats where backup_name=?", (self.backup_name,))
+        ]
+
+        filelist = os.listdir(self.backup_dir)
+        filelist.sort()
+        p = re.compile(r"^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}$")
+        for item in filelist:
+            if p.match(item):
+                dir_name = os.path.join(self.backup_dir, item)
+                if dir_name not in registered:
+                    start = datetime.datetime.strptime(item, "%Y%m%d-%Hh%Mm%S").isoformat()
+                    if fileisodate(dir_name) > start:
+                        stop = fileisodate(dir_name)
+                    else:
+                        stop = start
+                    self.logger.info("Registering %s started on %s", dir_name, start)
+                    self.logger.debug("  Disk usage %s", 'du -sb "%s"' % dir_name)
+                    if not self.dry_run:
+                        size_bytes = int(os.popen('du -sb "%s"' % dir_name).read().split("\t")[0])
+                    else:
+                        size_bytes = 0
+                    self.logger.debug("  Size in bytes : %i", size_bytes)
+                    if not self.dry_run:
+                        self.dbstat.add(
+                            self.backup_name,
+                            self.server_name,
+                            "",
+                            backup_start=start,
+                            backup_end=stop,
+                            status="OK",
+                            total_bytes=size_bytes,
+                            backup_location=dir_name,
+                        )
+                else:
+                    self.logger.info("Skipping %s, already registered", dir_name)
+
+    def is_pid_still_running(self, lockfile):
+        f = open(lockfile)
+        lines = f.readlines()
+        f.close()
+        if len(lines) == 0:
+            self.logger.info("[" + self.backup_name + "] empty lock file, removing...")
+            return False
+
+        for line in lines:
+            if line.startswith("pid="):
+                pid = line.split("=")[1].strip()
+                if os.path.exists("/proc/" + pid):
+                    self.logger.info("[" + self.backup_name + "] process still there")
+                    return True
+                else:
+                    self.logger.info("[" + self.backup_name + "] process not there anymore remove lock")
+                    return False
+            else:
+                self.logger.info("[" + self.backup_name + "] incorrrect lock file : no pid line")
+                return False
+
+    def set_lock(self):
+        self.logger.debug("[" + self.backup_name + "] setting lock")
+
+        # TODO: improve for race condition
+        # TODO: also check if process is really there
+        if os.path.isfile(self.backup_dir + "/lock"):
+            self.logger.debug("[" + self.backup_name + "] File " + self.backup_dir + "/lock already exist")
+            if not self.is_pid_still_running(self.backup_dir + "/lock"):
+                self.logger.info("[" + self.backup_name + "] removing lock file " + self.backup_dir + "/lock")
+                os.unlink(self.backup_dir + "/lock")
+            else:
+                return False
+
+        lockfile = open(self.backup_dir + "/lock", "w")
+        # Write all the lines at once:
+        lockfile.write("pid=" + str(os.getpid()))
+        lockfile.write("\nbackup_time=" + self.backup_start_date)
+        lockfile.close()
+        return True
+
+    def remove_lock(self):
+        self.logger.debug("[%s] removing lock", self.backup_name)
+        os.unlink(self.backup_dir + "/lock")
+
+
+class backup_rsync__btrfs_ssh(backup_rsync_btrfs):
+    """Backup a directory on remote server with rsync,ssh and btrfs protocol (requires rsync software on remote host)"""
+
+    type = "rsync+btrfs+ssh"
+    required_params = backup_generic.required_params + ["remote_user", "remote_dir", "private_key"]
+    optional_params = backup_generic.optional_params + [
+        "compression",
+        "bwlimit",
+        "ssh_port",
+        "exclude_list",
+        "protect_args",
+        "overload_args",
+        "cipher_spec",
+    ]
+    cipher_spec = ""
+
+
+register_driver(backup_rsync_btrfs)
+register_driver(backup_rsync__btrfs_ssh)
+
+if __name__ == "__main__":
+    logger = logging.getLogger("tisbackup")
+    logger.setLevel(logging.DEBUG)
+    formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+    cp = ConfigParser()
+    cp.read("/opt/tisbackup/configtest.ini")
+    dbstat = BackupStat("/backup/data/log/tisbackup.sqlite")
+    b = backup_rsync("htouvet", "/backup/data/htouvet", dbstat)
+    b.read_config(cp)
+    b.process_backup()
+    print((b.checknagios()))

libtisbackup/drivers/backup_samba4.py

@@ -0,0 +1,174 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------
+#    This file is part of TISBackup
+#
+#    TISBackup is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    TISBackup is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with TISBackup.  If not, see <http://www.gnu.org/licenses/>.
+#
+# -----------------------------------------------------------------------
+
+
+import sys
+
+try:
+    sys.stderr = open("/dev/null")  # Silence silly warnings from paramiko
+    import paramiko
+except ImportError as e:
+    print("Error : can not load paramiko library %s" % e)
+    raise
+
+sys.stderr = sys.__stderr__
+
+from libtisbackup import *
+
+
+class backup_samba4(backup_generic):
+    """Backup a samba4 databases as gzipped tdbs file through ssh"""
+
+    type = "samba4"
+    required_params = backup_generic.required_params + ["private_key"]
+    optional_params = backup_generic.optional_params + ["root_dir_samba"]
+
+    root_dir_samba = "/var/lib/samba/"
+
+    def do_backup(self, stats):
+        self.dest_dir = os.path.join(self.backup_dir, self.backup_start_date)
+
+        if not os.path.isdir(self.dest_dir):
+            if not self.dry_run:
+                os.makedirs(self.dest_dir)
+            else:
+                print('mkdir "%s"' % self.dest_dir)
+        else:
+            raise Exception("backup destination directory already exists : %s" % self.dest_dir)
+
+        self.logger.debug("[%s] Connecting to %s with user root and key %s", self.backup_name, self.server_name, self.private_key)
+        mykey = load_ssh_private_key(self.private_key)
+
+        self.ssh = paramiko.SSHClient()
+        self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        self.ssh.connect(self.server_name, username="root", pkey=mykey, port=self.ssh_port)
+
+        stats["log"] = "Successfully backuping processed to the following databases :"
+        stats["status"] = "List"
+        dir_ldbs = os.path.join(self.root_dir_samba + "/private/sam.ldb.d/")
+        cmd = "ls %s/*.ldb 2> /dev/null" % dir_ldbs
+        self.logger.debug("[%s] List databases: %s", self.backup_name, cmd)
+        (error_code, output) = ssh_exec(cmd, ssh=self.ssh)
+        self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
+        if error_code:
+            raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
+        databases = output.split("\n")
+        for database in databases:
+            if database != "":
+                self.db_name = database.rstrip()
+                self.do_mysqldump(stats)
+
+    def do_mysqldump(self, stats):
+        # t = datetime.datetime.now()
+        # backup_start_date =  t.strftime('%Y%m%d-%Hh%Mm%S')
+
+        # dump db
+        stats["status"] = "Dumping"
+        cmd = "tdbbackup -s .tisbackup " + self.db_name
+        self.logger.debug("[%s] Dump DB : %s", self.backup_name, cmd)
+        if not self.dry_run:
+            (error_code, output) = ssh_exec(cmd, ssh=self.ssh)
+            print(output)
+            self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
+            if error_code:
+                raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
+
+        # zip the file
+        stats["status"] = "Zipping"
+        cmd = 'gzip -f "%s.tisbackup"' % self.db_name
+        self.logger.debug("[%s] Compress backup : %s", self.backup_name, cmd)
+        if not self.dry_run:
+            (error_code, output) = ssh_exec(cmd, ssh=self.ssh)
+            self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
+            if error_code:
+                raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
+
+        # get the file
+        stats["status"] = "SFTP"
+        filepath = self.db_name + ".tisbackup.gz"
+        localpath = os.path.join(self.dest_dir, os.path.basename(self.db_name) + ".tisbackup.gz")
+        self.logger.debug("[%s] Get gz backup with sftp on %s from %s to %s", self.backup_name, self.server_name, filepath, localpath)
+        if not self.dry_run:
+            transport = self.ssh.get_transport()
+            sftp = paramiko.SFTPClient.from_transport(transport)
+            sftp.get(filepath, localpath)
+            sftp.close()
+
+        if not self.dry_run:
+            stats["total_files_count"] = 1 + stats.get("total_files_count", 0)
+            stats["written_files_count"] = 1 + stats.get("written_files_count", 0)
+            stats["total_bytes"] = os.stat(localpath).st_size + stats.get("total_bytes", 0)
+            stats["written_bytes"] = os.stat(localpath).st_size + stats.get("written_bytes", 0)
+        stats["log"] = '%s "%s"' % (stats["log"], self.db_name)
+        stats["backup_location"] = self.dest_dir
+
+        stats["status"] = "RMTemp"
+        cmd = 'rm -f  "%s"' % filepath
+        self.logger.debug("[%s] Remove temp gzip : %s", self.backup_name, cmd)
+        if not self.dry_run:
+            (error_code, output) = ssh_exec(cmd, ssh=self.ssh)
+            self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
+            if error_code:
+                raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
+        stats["status"] = "OK"
+
+    def register_existingbackups(self):
+        """scan backup dir and insert stats in database"""
+
+        registered = [
+            b["backup_location"]
+            for b in self.dbstat.query("select distinct backup_location from stats where backup_name=?", (self.backup_name,))
+        ]
+
+        filelist = os.listdir(self.backup_dir)
+        filelist.sort()
+        p = re.compile(r"^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}$")
+        for item in filelist:
+            if p.match(item):
+                dir_name = os.path.join(self.backup_dir, item)
+                if dir_name not in registered:
+                    start = datetime.datetime.strptime(item, "%Y%m%d-%Hh%Mm%S").isoformat()
+                    if fileisodate(dir_name) > start:
+                        stop = fileisodate(dir_name)
+                    else:
+                        stop = start
+                    self.logger.info("Registering %s started on %s", dir_name, start)
+                    self.logger.debug("  Disk usage %s", 'du -sb "%s"' % dir_name)
+                    if not self.dry_run:
+                        size_bytes = int(os.popen('du -sb "%s"' % dir_name).read().split("\t")[0])
+                    else:
+                        size_bytes = 0
+                    self.logger.debug("  Size in bytes : %i", size_bytes)
+                    if not self.dry_run:
+                        self.dbstat.add(
+                            self.backup_name,
+                            self.server_name,
+                            "",
+                            backup_start=start,
+                            backup_end=stop,
+                            status="OK",
+                            total_bytes=size_bytes,
+                            backup_location=dir_name,
+                        )
+                else:
+                    self.logger.info("Skipping %s, already registered", dir_name)
+
+
+register_driver(backup_samba4)

libtisbackup/drivers/backup_sqlserver.py

@@ -0,0 +1,171 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------
+#    This file is part of TISBackup
+#
+#    TISBackup is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    TISBackup is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with TISBackup.  If not, see <http://www.gnu.org/licenses/>.
+#
+# -----------------------------------------------------------------------
+
+
+import sys
+
+try:
+    sys.stderr = open("/dev/null")  # Silence silly warnings from paramiko
+    import paramiko
+except ImportError as e:
+    print("Error : can not load paramiko library %s" % e)
+    raise
+
+sys.stderr = sys.__stderr__
+
+import base64
+import datetime
+import os
+
+from libtisbackup import *
+
+
+class backup_sqlserver(backup_generic):
+    """Backup a SQLSERVER database as gzipped sql file through ssh"""
+
+    type = "sqlserver+ssh"
+    required_params = backup_generic.required_params + ["db_name", "private_key"]
+    optional_params = ["username", "remote_backup_dir", "sqlserver_before_2005", "db_server_name", "db_user", "db_password"]
+    db_name = ""
+    db_user = ""
+    db_password = ""
+    userdb = "-E"
+    username = "Administrateur"
+    remote_backup_dir = r"c:/WINDOWS/Temp/"
+    sqlserver_before_2005 = False
+    db_server_name = "localhost"
+
+    def do_backup(self, stats):
+        mykey = load_ssh_private_key(self.private_key)
+
+        self.logger.debug("[%s] Connecting to %s with user root and key %s", self.backup_name, self.server_name, self.private_key)
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(self.server_name, username=self.username, pkey=mykey, port=self.ssh_port)
+
+        t = datetime.datetime.now()
+        backup_start_date = t.strftime("%Y%m%d-%Hh%Mm%S")
+
+        backup_file = self.remote_backup_dir + "/" + self.db_name + "-" + backup_start_date + ".bak"
+        if not self.db_user == "":
+            self.userdb = "-U %s -P %s" % (self.db_user, self.db_password)
+
+        # dump db
+        stats["status"] = "Dumping"
+        if self.sqlserver_before_2005:
+            cmd = """osql -E -Q "BACKUP DATABASE [%s]
+				      TO DISK='%s'
+				      WITH FORMAT" """ % (self.db_name, backup_file)
+        else:
+            cmd = """sqlcmd %s -S "%s" -d master -Q "BACKUP DATABASE [%s]
+                	                                       TO DISK = N'%s'
+                        	                               WITH INIT, NOUNLOAD ,
+                                	                       NAME = N'Backup %s', NOSKIP ,STATS = 10, NOFORMAT" """ % (
+                self.userdb,
+                self.db_server_name,
+                self.db_name,
+                backup_file,
+                self.db_name,
+            )
+        self.logger.debug("[%s] Dump DB : %s", self.backup_name, cmd)
+        try:
+            if not self.dry_run:
+                (error_code, output) = ssh_exec(cmd, ssh=ssh)
+                self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
+                if error_code:
+                    raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
+
+            # zip the file
+            stats["status"] = "Zipping"
+            cmd = 'gzip "%s"' % backup_file
+            self.logger.debug("[%s] Compress backup : %s", self.backup_name, cmd)
+            if not self.dry_run:
+                (error_code, output) = ssh_exec(cmd, ssh=ssh)
+                self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
+                if error_code:
+                    raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
+
+            # get the file
+            stats["status"] = "SFTP"
+            filepath = backup_file + ".gz"
+            localpath = os.path.join(self.backup_dir, self.db_name + "-" + backup_start_date + ".bak.gz")
+            self.logger.debug("[%s] Get gz backup with sftp on %s from %s to %s", self.backup_name, self.server_name, filepath, localpath)
+            if not self.dry_run:
+                transport = ssh.get_transport()
+                sftp = paramiko.SFTPClient.from_transport(transport)
+                sftp.get(filepath, localpath)
+                sftp.close()
+
+            if not self.dry_run:
+                stats["total_files_count"] = 1
+                stats["written_files_count"] = 1
+                stats["total_bytes"] = os.stat(localpath).st_size
+                stats["written_bytes"] = os.stat(localpath).st_size
+            stats["log"] = "gzip dump of DB %s:%s (%d bytes) to %s" % (self.server_name, self.db_name, stats["written_bytes"], localpath)
+            stats["backup_location"] = localpath
+
+        finally:
+            stats["status"] = "RMTemp"
+            cmd = 'rm -f "%s" "%s"' % (backup_file + ".gz", backup_file)
+            self.logger.debug("[%s] Remove temp gzip : %s", self.backup_name, cmd)
+            if not self.dry_run:
+                (error_code, output) = ssh_exec(cmd, ssh=ssh)
+                self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
+                if error_code:
+                    raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
+
+        stats["status"] = "OK"
+
+    def register_existingbackups(self):
+        """scan backup dir and insert stats in database"""
+
+        registered = [
+            b["backup_location"]
+            for b in self.dbstat.query("select distinct backup_location from stats where backup_name=?", (self.backup_name,))
+        ]
+
+        filelist = os.listdir(self.backup_dir)
+        filelist.sort()
+        p = re.compile(r"^%s-(?P<date>\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}).bak.gz$" % self.db_name)
+        for item in filelist:
+            sr = p.match(item)
+            if sr:
+                file_name = os.path.join(self.backup_dir, item)
+                start = datetime.datetime.strptime(sr.groups()[0], "%Y%m%d-%Hh%Mm%S").isoformat()
+                if file_name not in registered:
+                    self.logger.info("Registering %s from %s", file_name, fileisodate(file_name))
+                    size_bytes = int(os.popen('du -sb "%s"' % file_name).read().split("\t")[0])
+                    self.logger.debug("  Size in bytes : %i", size_bytes)
+                    if not self.dry_run:
+                        self.dbstat.add(
+                            self.backup_name,
+                            self.server_name,
+                            "",
+                            backup_start=start,
+                            backup_end=fileisodate(file_name),
+                            status="OK",
+                            total_bytes=size_bytes,
+                            backup_location=file_name,
+                        )
+                else:
+                    self.logger.info("Skipping %s from %s, already registered", file_name, fileisodate(file_name))
+
+
+register_driver(backup_sqlserver)

libtisbackup/drivers/backup_switch.py

@@ -0,0 +1,267 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------
+#    This file is part of TISBackup
+#
+#    TISBackup is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    TISBackup is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with TISBackup.  If not, see <http://www.gnu.org/licenses/>.
+#
+# -----------------------------------------------------------------------
+
+import base64
+import datetime
+import logging
+import os
+import os.path
+import re
+import select
+import socket
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from stat import *
+
+import pexpect
+import requests
+
+from . import XenAPI
+from libtisbackup import *
+
+
+class backup_switch(backup_generic):
+    """Backup a startup-config on a switch"""
+
+    type = "switch"
+
+    required_params = backup_generic.required_params + ["switch_ip", "switch_type"]
+    optional_params = backup_generic.optional_params + ["switch_user", "switch_password"]
+
+    switch_user = ""
+    switch_password = ""
+
+    def switch_hp(self, filename):
+        s = socket.socket()
+        try:
+            s.connect((self.switch_ip, 23))
+            s.close()
+        except:
+            raise
+
+        child = pexpect.spawn("telnet " + self.switch_ip)
+        time.sleep(1)
+        if self.switch_user != "":
+            child.sendline(self.switch_user)
+            child.sendline(self.switch_password + "\r")
+        else:
+            child.sendline(self.switch_password + "\r")
+        try:
+            child.expect("#")
+        except:
+            raise Exception("Bad Credentials")
+        child.sendline("terminal length 1000\r")
+        child.expect("#")
+        child.sendline("show config\r")
+        child.maxread = 100000000
+        child.expect("Startup.+$")
+        lines = child.after
+        if "-- MORE --" in lines:
+            raise Exception("Terminal lenght is not sufficient")
+        child.expect("#")
+        lines += child.before
+        child.sendline("logout\r")
+        child.send("y\r")
+        for line in lines.split("\n")[1:-1]:
+            open(filename, "a").write(line.strip() + "\n")
+
+    def switch_cisco(self, filename):
+        s = socket.socket()
+        try:
+            s.connect((self.switch_ip, 23))
+            s.close()
+        except:
+            raise
+
+        child = pexpect.spawn("telnet " + self.switch_ip)
+        time.sleep(1)
+        if self.switch_user:
+            child.sendline(self.switch_user)
+        child.expect("Password: ")
+        child.sendline(self.switch_password + "\r")
+        try:
+            child.expect(">")
+        except:
+            raise Exception("Bad Credentials")
+        child.sendline("enable\r")
+        child.expect("Password: ")
+        child.sendline(self.switch_password + "\r")
+        try:
+            child.expect("#")
+        except:
+            raise Exception("Bad Credentials")
+        child.sendline("terminal length 0\r")
+        child.expect("#")
+        child.sendline("show run\r")
+        child.expect("Building configuration...")
+        child.expect("#")
+        running_config = child.before
+        child.sendline("show vlan\r")
+        child.expect("VLAN")
+        child.expect("#")
+        vlan = "VLAN" + child.before
+        open(filename, "a").write(running_config + "\n" + vlan)
+        child.send("exit\r")
+        child.close()
+
+    def switch_linksys_SRW2024(self, filename):
+        s = socket.socket()
+        try:
+            s.connect((self.switch_ip, 23))
+            s.close()
+        except:
+            raise
+
+        child = pexpect.spawn("telnet " + self.switch_ip)
+        time.sleep(1)
+        if hasattr(self, "switch_password"):
+            child.sendline(self.switch_user + "\t")
+            child.sendline(self.switch_password + "\r")
+        else:
+            child.sendline(self.switch_user + "\r")
+        try:
+            child.expect("Menu")
+        except:
+            raise Exception("Bad Credentials")
+        child.sendline("\032")
+        child.expect(">")
+        child.sendline("lcli")
+        child.expect("Name:")
+        if hasattr(self, "switch_password"):
+            child.send(self.switch_user + "\r" + self.switch_password + "\r")
+        else:
+            child.sendline(self.switch_user)
+        child.expect(".*#")
+
+        child.sendline("terminal datadump")
+        child.expect("#")
+        child.sendline("show startup-config")
+        child.expect("#")
+        lines = child.before
+        if "Unrecognized command" in lines:
+            raise Exception("Bad Credentials")
+        child.sendline("exit")
+        # child.expect( ">")
+        # child.sendline("logout")
+        for line in lines.split("\n")[1:-1]:
+            open(filename, "a").write(line.strip() + "\n")
+
+    def switch_dlink_DGS1210(self, filename):
+        login_data = {"Login": self.switch_user, "Password": self.switch_password, "sellanId": 0, "sellan": 0, "lang_seqid": 1}
+        resp = requests.post(
+            "http://%s/form/formLoginApply" % self.switch_ip,
+            data=login_data,
+            headers={"Referer": "http://%s/www/login.html" % self.switch_ip},
+        )
+        if "Wrong password" in resp.text:
+            raise Exception("Wrong password")
+        resp = requests.post(
+            "http://%s/BinFile/config.bin" % self.switch_ip, headers={"Referer": "http://%s/www/iss/013_download_cfg.html" % self.switch_ip}
+        )
+        with open(filename, "w") as f:
+            f.write(resp.content)
+
+    def switch_dlink_DGS1510(self, filename):
+        s = socket.socket()
+        try:
+            s.connect((self.switch_ip, 23))
+            s.close()
+        except:
+            raise
+
+        child = pexpect.spawn("telnet " + self.switch_ip)
+        time.sleep(1)
+        if self.switch_user:
+            child.sendline(self.switch_user)
+        child.expect("Password:")
+        child.sendline(self.switch_password + "\r")
+        try:
+            child.expect("#")
+        except:
+            raise Exception("Bad Credentials")
+        child.sendline("terminal length 0\r")
+        child.expect("#")
+        child.sendline("show run\r")
+        child.logfile_read = open(filename, "a")
+        child.expect("End of configuration file")
+        child.expect("#--")
+        child.expect("#")
+        child.close()
+        myre = re.compile(r"#--+")
+        config = myre.split(open(filename).read())[2]
+        with open(filename, "w") as f:
+            f.write(config)
+
+    def do_backup(self, stats):
+        try:
+            dest_filename = os.path.join(self.backup_dir, "%s-%s" % (self.backup_name, self.backup_start_date))
+
+            # options = []
+            # options_params = " ".join(options)
+            if "LINKSYS-SRW2024" == self.switch_type:
+                dest_filename += ".txt"
+                self.switch_linksys_SRW2024(dest_filename)
+            elif self.switch_type in [
+                "CISCO",
+            ]:
+                dest_filename += ".txt"
+                self.switch_cisco(dest_filename)
+            elif self.switch_type in ["HP-PROCURVE-4104GL", "HP-PROCURVE-2524"]:
+                dest_filename += ".txt"
+                self.switch_hp(dest_filename)
+            elif "DLINK-DGS1210" == self.switch_type:
+                dest_filename += ".bin"
+                self.switch_dlink_DGS1210(dest_filename)
+            elif "DLINK-DGS1510" == self.switch_type:
+                dest_filename += ".cfg"
+                self.switch_dlink_DGS1510(dest_filename)
+            else:
+                raise Exception("Unknown Switch type")
+
+            stats["total_files_count"] = 1
+            stats["written_files_count"] = 1
+            stats["total_bytes"] = os.stat(dest_filename).st_size
+            stats["written_bytes"] = stats["total_bytes"]
+            stats["backup_location"] = dest_filename
+            stats["status"] = "OK"
+            stats["log"] = "Switch backup from %s OK, %d bytes written" % (self.server_name, stats["written_bytes"])
+
+        except BaseException as e:
+            stats["status"] = "ERROR"
+            stats["log"] = str(e)
+            raise
+
+
+register_driver(backup_switch)
+
+if __name__ == "__main__":
+    logger = logging.getLogger("tisbackup")
+    logger.setLevel(logging.DEBUG)
+    formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+    cp = ConfigParser()
+    cp.read("/opt/tisbackup/configtest.ini")
+    b = backup_xva()
+    b.read_config(cp)

libtisbackup/drivers/backup_vmdk.py

@@ -0,0 +1,270 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------
+#    This file is part of TISBackup
+#
+#    TISBackup is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    TISBackup is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with TISBackup.  If not, see <http://www.gnu.org/licenses/>.
+#
+# -----------------------------------------------------------------------
+
+import atexit
+import getpass
+from datetime import date, datetime, timedelta
+
+import pyVmomi
+import requests
+from pyVim.connect import Disconnect, SmartConnect
+from pyVmomi import vim, vmodl
+
+# Disable HTTPS verification warnings.
+from requests.packages import urllib3
+
+from libtisbackup import *
+
+urllib3.disable_warnings()
+import os
+import re
+import tarfile
+import time
+import xml.etree.ElementTree as ET
+from stat import *
+
+
+class backup_vmdk(backup_generic):
+    type = "esx-vmdk"
+
+    required_params = backup_generic.required_params + ["esxhost", "password_file", "server_name"]
+    optional_params = backup_generic.optional_params + ["esx_port", "prefix_clone", "create_ovafile", "halt_vm"]
+
+    esx_port = 443
+    prefix_clone = "clone-"
+    create_ovafile = "no"
+    halt_vm = "no"
+
+    def make_compatible_cookie(self, client_cookie):
+        cookie_name = client_cookie.split("=", 1)[0]
+        cookie_value = client_cookie.split("=", 1)[1].split(";", 1)[0]
+        cookie_path = client_cookie.split("=", 1)[1].split(";", 1)[1].split(";", 1)[0].lstrip()
+        cookie_text = " " + cookie_value + "; $" + cookie_path
+        # Make a cookie
+        cookie = dict()
+        cookie[cookie_name] = cookie_text
+        return cookie
+
+    def download_file(self, url, local_filename, cookie, headers):
+        r = requests.get(url, stream=True, headers=headers, cookies=cookie, verify=False)
+        with open(local_filename, "wb") as f:
+            for chunk in r.iter_content(chunk_size=1024 * 1024 * 64):
+                if chunk:
+                    f.write(chunk)
+                    f.flush()
+        return local_filename
+
+    def export_vmdks(self, vm):
+        HttpNfcLease = vm.ExportVm()
+        try:
+            infos = HttpNfcLease.info
+            device_urls = infos.deviceUrl
+            vmdks = []
+            for device_url in device_urls:
+                deviceId = device_url.key
+                deviceUrlStr = device_url.url
+                diskFileName = vm.name.replace(self.prefix_clone, "") + "-" + device_url.targetId
+                diskUrlStr = deviceUrlStr.replace("*", self.esxhost)
+                # diskLocalPath = './' + diskFileName
+
+                cookie = self.make_compatible_cookie(si._stub.cookie)
+                headers = {"Content-Type": "application/octet-stream"}
+                self.logger.debug("[%s] exporting disk: %s" % (self.server_name, diskFileName))
+
+                self.download_file(diskUrlStr, diskFileName, cookie, headers)
+                vmdks.append({"filename": diskFileName, "id": deviceId})
+        finally:
+            HttpNfcLease.Complete()
+        return vmdks
+
+    def create_ovf(self, vm, vmdks):
+        ovfDescParams = vim.OvfManager.CreateDescriptorParams()
+        ovf = si.content.ovfManager.CreateDescriptor(vm, ovfDescParams)
+        root = ET.fromstring(ovf.ovfDescriptor)
+        new_id = list(root[0][1].attrib.values())[0][1:3]
+        ovfFiles = []
+        for vmdk in vmdks:
+            old_id = vmdk["id"][1:3]
+            id = vmdk["id"].replace(old_id, new_id)
+            ovfFiles.append(vim.OvfManager.OvfFile(size=os.path.getsize(vmdk["filename"]), path=vmdk["filename"], deviceId=id))
+
+        ovfDescParams = vim.OvfManager.CreateDescriptorParams()
+        ovfDescParams.ovfFiles = ovfFiles
+
+        ovf = si.content.ovfManager.CreateDescriptor(vm, ovfDescParams)
+        ovf_filename = vm.name + ".ovf"
+        self.logger.debug("[%s] creating ovf file: %s" % (self.server_name, ovf_filename))
+
+        with open(ovf_filename, "w") as f:
+            f.write(ovf.ovfDescriptor)
+        return ovf_filename
+
+    def create_ova(self, vm, vmdks, ovf_filename):
+        ova_filename = vm.name + ".ova"
+        vmdks.insert(0, {"filename": ovf_filename, "id": "false"})
+        self.logger.debug("[%s] creating ova file: %s" % (self.server_name, ova_filename))
+        with tarfile.open(ova_filename, "w") as tar:
+            for vmdk in vmdks:
+                tar.add(vmdk["filename"])
+                os.unlink(vmdk["filename"])
+        return ova_filename
+
+    def clone_vm(self, vm):
+        task = self.wait_task(
+            vm.CreateSnapshot_Task(
+                name="backup", description="Automatic backup " + datetime.now().strftime("%Y-%m-%d %H:%M:%s"), memory=False, quiesce=True
+            )
+        )
+        snapshot = task.info.result
+        prefix_vmclone = self.prefix_clone
+        clone_name = prefix_vmclone + vm.name
+        datastore = "[%s]" % vm.datastore[0].name
+
+        vmx_file = vim.vm.FileInfo(logDirectory=None, snapshotDirectory=None, suspendDirectory=None, vmPathName=datastore)
+
+        config = vim.vm.ConfigSpec(
+            name=clone_name, memoryMB=vm.summary.config.memorySizeMB, numCPUs=vm.summary.config.numCpu, files=vmx_file
+        )
+
+        hosts = datacenter.hostFolder.childEntity
+        resource_pool = hosts[0].resourcePool
+
+        self.wait_task(vmFolder.CreateVM_Task(config=config, pool=resource_pool))
+
+        new_vm = [x for x in vmFolder.childEntity if x.name == clone_name][0]
+
+        controller = vim.vm.device.VirtualDeviceSpec()
+        controller.operation = vim.vm.device.VirtualDeviceSpec.Operation.add
+        controller.device = vim.vm.device.VirtualLsiLogicController(busNumber=0, sharedBus="noSharing")
+        controller.device.key = 0
+        i = 0
+
+        vm_devices = []
+        clone_folder = "%s/" % "/".join(new_vm.summary.config.vmPathName.split("/")[:-1])
+        for device in vm.config.hardware.device:
+            if device.__class__.__name__ == "vim.vm.device.VirtualDisk":
+                cur_vers = int(re.findall(r"\d{3,6}", device.backing.fileName)[0])
+
+                if cur_vers == 1:
+                    source = device.backing.fileName.replace("-000001", "")
+                else:
+                    source = device.backing.fileName.replace("%d." % cur_vers, "%d." % (cur_vers - 1))
+
+                dest = clone_folder + source.split("/")[-1]
+                disk_spec = vim.VirtualDiskManager.VirtualDiskSpec(diskType="sparseMonolithic", adapterType="ide")
+                self.wait_task(si.content.virtualDiskManager.CopyVirtualDisk_Task(sourceName=source, destName=dest, destSpec=disk_spec))
+                # self.wait_task(si.content.virtualDiskManager.ShrinkVirtualDisk_Task(dest))
+
+                diskfileBacking = vim.vm.device.VirtualDisk.FlatVer2BackingInfo()
+                diskfileBacking.fileName = dest
+                diskfileBacking.diskMode = "persistent"
+                vdisk_spec = vim.vm.device.VirtualDeviceSpec()
+                vdisk_spec.operation = vim.vm.device.VirtualDeviceSpec.Operation.add
+                vdisk_spec.device = vim.vm.device.VirtualDisk(capacityInKB=10000, controllerKey=controller.device.key)
+                vdisk_spec.device.key = 0
+                vdisk_spec.device.backing = diskfileBacking
+                vdisk_spec.device.unitNumber = i
+                vm_devices.append(vdisk_spec)
+                i += 1
+
+        vm_devices.append(controller)
+
+        config.deviceChange = vm_devices
+        self.wait_task(new_vm.ReconfigVM_Task(config))
+        self.wait_task(snapshot.RemoveSnapshot_Task(removeChildren=True))
+        return new_vm
+
+    def wait_task(self, task):
+        while task.info.state in ["queued", "running"]:
+            time.sleep(2)
+        self.logger.debug("[%s] %s", self.server_name, task.info.descriptionId)
+        return task
+
+    def do_backup(self, stats):
+        try:
+            dest_dir = os.path.join(self.backup_dir, "%s" % self.backup_start_date)
+            if not os.path.isdir(dest_dir):
+                if not self.dry_run:
+                    os.makedirs(dest_dir)
+                else:
+                    print('mkdir "%s"' % dest_dir)
+            else:
+                raise Exception("backup destination directory already exists : %s" % dest_dir)
+            os.chdir(dest_dir)
+            user_esx, password_esx, null = open(self.password_file).read().split("\n")
+
+            global si
+            si = SmartConnect(host=self.esxhost, user=user_esx, pwd=password_esx, port=self.esx_port)
+
+            if not si:
+                raise Exception("Could not connect to the specified host using specified " "username and password")
+
+            atexit.register(Disconnect, si)
+
+            content = si.RetrieveContent()
+            for child in content.rootFolder.childEntity:
+                if hasattr(child, "vmFolder"):
+                    global vmFolder, datacenter
+                    datacenter = child
+                    vmFolder = datacenter.vmFolder
+                    vmList = vmFolder.childEntity
+                    for vm in vmList:
+                        if vm.name == self.server_name:
+                            vm_is_off = vm.summary.runtime.powerState == "poweredOff"
+                            if str2bool(self.halt_vm):
+                                vm.ShutdownGuest()
+                                vm_is_off = True
+
+                            if vm_is_off:
+                                vmdks = self.export_vmdks(vm)
+                                ovf_filename = self.create_ovf(vm, vmdks)
+                            else:
+                                new_vm = self.clone_vm(vm)
+                                vmdks = self.export_vmdks(new_vm)
+                                ovf_filename = self.create_ovf(vm, vmdks)
+                                self.wait_task(new_vm.Destroy_Task())
+
+                            if str2bool(self.create_ovafile):
+                                ova_filename = self.create_ova(vm, vmdks, ovf_filename)  # noqa : F841
+
+                            if str2bool(self.halt_vm):
+                                vm.PowerOnVM()
+
+            if os.path.exists(dest_dir):
+                for file in os.listdir(dest_dir):
+                    stats["written_bytes"] += os.stat(file)[ST_SIZE]
+                    stats["total_files_count"] += 1
+                    stats["written_files_count"] += 1
+                stats["total_bytes"] = stats["written_bytes"]
+            else:
+                stats["written_bytes"] = 0
+
+            stats["backup_location"] = dest_dir
+
+            stats["log"] = "XVA backup from %s OK, %d bytes written" % (self.server_name, stats["written_bytes"])
+            stats["status"] = "OK"
+
+        except BaseException as e:
+            stats["status"] = "ERROR"
+            stats["log"] = str(e)
+            raise
+
+
+register_driver(backup_vmdk)

libtisbackup/drivers/backup_xcp_metadata.py

@@ -0,0 +1,101 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------
+#    This file is part of TISBackup
+#
+#    TISBackup is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    TISBackup is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with TISBackup.  If not, see <http://www.gnu.org/licenses/>.
+#
+# -----------------------------------------------------------------------
+
+
+import paramiko
+
+from libtisbackup import *
+
+
+class backup_xcp_metadata(backup_generic):
+    """Backup metatdata of a xcp pool using xe pool-dump-database"""
+
+    type = "xcp-dump-metadata"
+    required_params = ["type", "server_name", "private_key", "backup_name"]
+
+    def do_backup(self, stats):
+        self.logger.debug("[%s] Connecting to %s with user root and key %s", self.backup_name, self.server_name, self.private_key)
+
+        t = datetime.datetime.now()
+        backup_start_date = t.strftime("%Y%m%d-%Hh%Mm%S")
+
+        # dump pool medatadata
+        localpath = os.path.join(self.backup_dir, "xcp_metadata-" + backup_start_date + ".dump")
+        stats["status"] = "Dumping"
+        if not self.dry_run:
+            cmd = "/opt/xensource/bin/xe pool-dump-database file-name="
+            self.logger.debug("[%s] Dump XCP Metadata : %s", self.backup_name, cmd)
+            (error_code, output) = ssh_exec(cmd, server_name=self.server_name, private_key=self.private_key, remote_user="root")
+
+            with open(localpath, "w") as f:
+                f.write(output)
+
+        # zip the file
+        stats["status"] = "Zipping"
+        cmd = "gzip %s " % localpath
+        self.logger.debug("[%s] Compress backup : %s", self.backup_name, cmd)
+        if not self.dry_run:
+            call_external_process(cmd)
+        localpath += ".gz"
+        if not self.dry_run:
+            stats["total_files_count"] = 1
+            stats["written_files_count"] = 1
+            stats["total_bytes"] = os.stat(localpath).st_size
+            stats["written_bytes"] = os.stat(localpath).st_size
+        stats["log"] = "gzip dump of DB %s:%s (%d bytes) to %s" % (self.server_name, "xcp metadata dump", stats["written_bytes"], localpath)
+        stats["backup_location"] = localpath
+        stats["status"] = "OK"
+
+    def register_existingbackups(self):
+        """scan metatdata backup files and insert stats in database"""
+
+        registered = [
+            b["backup_location"]
+            for b in self.dbstat.query("select distinct backup_location from stats where backup_name=?", (self.backup_name,))
+        ]
+
+        filelist = os.listdir(self.backup_dir)
+        filelist.sort()
+        p = re.compile(r"^%s-(?P<date>\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}).dump.gz$" % self.server_name)
+        for item in filelist:
+            sr = p.match(item)
+            if sr:
+                file_name = os.path.join(self.backup_dir, item)
+                start = datetime.datetime.strptime(sr.groups()[0], "%Y%m%d-%Hh%Mm%S").isoformat()
+                if file_name not in registered:
+                    self.logger.info("Registering %s from %s", file_name, fileisodate(file_name))
+                    size_bytes = int(os.popen('du -sb "%s"' % file_name).read().split("\t")[0])
+                    self.logger.debug("  Size in bytes : %i", size_bytes)
+                    if not self.dry_run:
+                        self.dbstat.add(
+                            self.backup_name,
+                            self.server_name,
+                            "",
+                            backup_start=start,
+                            backup_end=fileisodate(file_name),
+                            status="OK",
+                            total_bytes=size_bytes,
+                            backup_location=file_name,
+                        )
+                else:
+                    self.logger.info("Skipping %s from %s, already registered", file_name, fileisodate(file_name))
+
+
+register_driver(backup_xcp_metadata)

libtisbackup/drivers/backup_xva.py

@@ -0,0 +1,309 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------
+#    This file is part of TISBackup
+#
+#    TISBackup is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    TISBackup is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with TISBackup.  If not, see <http://www.gnu.org/licenses/>.
+#
+# -----------------------------------------------------------------------
+
+import datetime
+import hashlib
+import logging
+import os
+import re
+import socket
+import ssl
+import subprocess
+import tarfile
+import urllib.error
+import urllib.parse
+import urllib.request
+from stat import *
+
+import requests
+
+from . import XenAPI
+from libtisbackup import *
+
+if hasattr(ssl, "_create_unverified_context"):
+    ssl._create_default_https_context = ssl._create_unverified_context
+
+
+class backup_xva(backup_generic):
+    """Backup a VM running on a XCP server as a XVA file (requires xe tools and XenAPI)"""
+
+    type = "xen-xva"
+
+    required_params = backup_generic.required_params + ["xcphost", "password_file", "server_name"]
+    optional_params = backup_generic.optional_params + [
+        "enable_https",
+        "halt_vm",
+        "verify_export",
+        "reuse_snapshot",
+        "ignore_proxies",
+        "use_compression",
+    ]
+
+    enable_https = "no"
+    halt_vm = "no"
+    verify_export = "no"
+    reuse_snapshot = "no"
+    ignore_proxies = "yes"
+    use_compression = "true"
+
+    if str2bool(ignore_proxies):
+        os.environ["http_proxy"] = ""
+        os.environ["https_proxy"] = ""
+
+    def verify_export_xva(self, filename):
+        self.logger.debug("[%s] Verify xva export integrity", self.server_name)
+        tar = tarfile.open(filename)
+        members = tar.getmembers()
+        for tarinfo in members:
+            if re.search("^[0-9]*$", os.path.basename(tarinfo.name)):
+                sha1sum = hashlib.sha1(tar.extractfile(tarinfo).read()).hexdigest()
+                sha1sum2 = tar.extractfile(tarinfo.name + ".checksum").read()
+                if not sha1sum == sha1sum2:
+                    raise Exception("File corrupt")
+        tar.close()
+
+    def export_xva(self, vdi_name, filename, halt_vm, dry_run, enable_https=True, reuse_snapshot="no"):
+        user_xen, password_xen, null = open(self.password_file).read().split("\n")
+        session = XenAPI.Session("https://" + self.xcphost)
+        try:
+            session.login_with_password(user_xen, password_xen)
+        except XenAPI.Failure as error:
+            msg, ip = error.details
+
+            if msg == "HOST_IS_SLAVE":
+                xcphost = ip
+                session = XenAPI.Session("https://" + xcphost)
+                session.login_with_password(user_xen, password_xen)
+
+        if not session.xenapi.VM.get_by_name_label(vdi_name):
+            return "bad VM name: %s" % vdi_name
+
+        vm = session.xenapi.VM.get_by_name_label(vdi_name)[0]
+        status_vm = session.xenapi.VM.get_power_state(vm)
+
+        self.logger.debug("[%s] Check if previous fail backups exist", vdi_name)
+        backups_fail = [f for f in os.listdir(self.backup_dir) if f.startswith(vdi_name) and f.endswith(".tmp")]
+        for backup_fail in backups_fail:
+            self.logger.debug('[%s] Delete backup "%s"', vdi_name, backup_fail)
+            os.unlink(os.path.join(self.backup_dir, backup_fail))
+
+        # add snapshot option
+        if not str2bool(halt_vm):
+            self.logger.debug("[%s] Check if previous tisbackups snapshots exist", vdi_name)
+            old_snapshots = session.xenapi.VM.get_by_name_label("tisbackup-%s" % (vdi_name))
+            self.logger.debug("[%s] Old snaps count %s", vdi_name, len(old_snapshots))
+
+            if len(old_snapshots) == 1 and str2bool(reuse_snapshot):
+                snapshot = old_snapshots[0]
+                self.logger.debug('[%s] Reusing snap "%s"', vdi_name, session.xenapi.VM.get_name_description(snapshot))
+                vm = snapshot  # vm = session.xenapi.VM.get_by_name_label("tisbackup-%s"%(vdi_name))[0]
+            else:
+                self.logger.debug("[%s] Deleting %s old snaps", vdi_name, len(old_snapshots))
+                for old_snapshot in old_snapshots:
+                    self.logger.debug("[%s] Destroy snapshot %s", vdi_name, session.xenapi.VM.get_name_description(old_snapshot))
+                    try:
+                        for vbd in session.xenapi.VM.get_VBDs(old_snapshot):
+                            if session.xenapi.VBD.get_type(vbd) == "CD" and not session.xenapi.VBD.get_record(vbd)["empty"]:
+                                session.xenapi.VBD.eject(vbd)
+                            else:
+                                vdi = session.xenapi.VBD.get_VDI(vbd)
+                                if "NULL" not in vdi:
+                                    session.xenapi.VDI.destroy(vdi)
+                        session.xenapi.VM.destroy(old_snapshot)
+                    except XenAPI.Failure as error:
+                        return "error when destroy snapshot %s" % (error)
+
+                now = datetime.datetime.now()
+                self.logger.debug("[%s] Snapshot in progress", vdi_name)
+                try:
+                    snapshot = session.xenapi.VM.snapshot(vm, "tisbackup-%s" % (vdi_name))
+                    self.logger.debug("[%s] got snapshot %s", vdi_name, snapshot)
+                except XenAPI.Failure as error:
+                    return "error when snapshot %s" % (error)
+                # get snapshot opaqueRef
+                vm = session.xenapi.VM.get_by_name_label("tisbackup-%s" % (vdi_name))[0]
+                session.xenapi.VM.set_name_description(snapshot, "snapshot created by tisbackup on: %s" % (now.strftime("%Y-%m-%d %H:%M")))
+        else:
+            self.logger.debug("[%s] Status of VM: %s", self.backup_name, status_vm)
+            if status_vm == "Running":
+                self.logger.debug("[%s] Shudown in progress", self.backup_name)
+                if dry_run:
+                    print("session.xenapi.VM.clean_shutdown(vm)")
+                else:
+                    session.xenapi.VM.clean_shutdown(vm)
+        try:
+            try:
+                filename_temp = filename + ".tmp"
+                self.logger.debug("[%s] Copy in progress", self.backup_name)
+                if not str2bool(self.use_compression):
+                    socket.setdefaulttimeout(120)
+
+                scheme = "http://"
+                if str2bool(enable_https):
+                    scheme = "https://"
+
+                # url = scheme+user_xen+":"+password_xen+"@"+self.xcphost+"/export?use_compression="+self.use_compression+"&uuid="+session.xenapi.VM.get_uuid(vm)
+                top_level_url = (
+                    scheme + self.xcphost + "/export?use_compression=" + self.use_compression + "&uuid=" + session.xenapi.VM.get_uuid(vm)
+                )
+                r = requests.get(top_level_url, auth=(user_xen, password_xen))
+                open(filename_temp, "wb").write(r.content)
+
+            except Exception as e:
+                self.logger.error("[%s] error when fetching snap: %s", "tisbackup-%s" % (vdi_name), e)
+                if os.path.exists(filename_temp):
+                    os.unlink(filename_temp)
+                raise
+
+        finally:
+            if not str2bool(halt_vm):
+                self.logger.debug("[%s] Destroy snapshot", "tisbackup-%s" % (vdi_name))
+                try:
+                    for vbd in session.xenapi.VM.get_VBDs(snapshot):
+                        if session.xenapi.VBD.get_type(vbd) == "CD" and not session.xenapi.VBD.get_record(vbd)["empty"]:
+                            session.xenapi.VBD.eject(vbd)
+                        else:
+                            vdi = session.xenapi.VBD.get_VDI(vbd)
+                            if "NULL" not in vdi:
+                                session.xenapi.VDI.destroy(vdi)
+                    session.xenapi.VM.destroy(snapshot)
+                except XenAPI.Failure as error:
+                    return "error when destroy snapshot %s" % (error)
+
+            elif status_vm == "Running":
+                self.logger.debug("[%s] Starting in progress", self.backup_name)
+                if dry_run:
+                    print("session.xenapi.Async.VM.start(vm,False,True)")
+                else:
+                    session.xenapi.Async.VM.start(vm, False, True)
+
+            session.logout()
+
+        if os.path.exists(filename_temp):
+            # Verify tar file integrity using subprocess instead of os.system
+            try:
+                subprocess.run(
+                    ["tar", "tf", filename_temp],
+                    capture_output=True,
+                    check=True,
+                    timeout=300
+                )
+            except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
+                os.unlink(filename_temp)
+                return "Tar error"
+
+            if str2bool(self.verify_export):
+                self.verify_export_xva(filename_temp)
+            os.rename(filename_temp, filename)
+
+        return 0
+
+    def do_backup(self, stats):
+        try:
+            dest_filename = os.path.join(self.backup_dir, "%s-%s.%s" % (self.backup_name, self.backup_start_date, "xva"))
+
+            # options = []
+            # options_params = " ".join(options)
+            cmd = self.export_xva(
+                vdi_name=self.server_name,
+                filename=dest_filename,
+                halt_vm=self.halt_vm,
+                enable_https=self.enable_https,
+                dry_run=self.dry_run,
+                reuse_snapshot=self.reuse_snapshot,
+            )
+            if os.path.exists(dest_filename):
+                stats["written_bytes"] = os.stat(dest_filename)[ST_SIZE]
+                stats["total_files_count"] = 1
+                stats["written_files_count"] = 1
+                stats["total_bytes"] = stats["written_bytes"]
+            else:
+                stats["written_bytes"] = 0
+
+            stats["backup_location"] = dest_filename
+            if cmd == 0:
+                stats["log"] = "XVA backup from %s OK, %d bytes written" % (self.server_name, stats["written_bytes"])
+                stats["status"] = "OK"
+            else:
+                raise Exception(cmd)
+
+        except BaseException as e:
+            stats["status"] = "ERROR"
+            stats["log"] = str(e)
+            raise
+
+    def register_existingbackups(self):
+        """scan backup dir and insert stats in database"""
+
+        registered = [
+            b["backup_location"]
+            for b in self.dbstat.query("select distinct backup_location from stats where backup_name=?", (self.backup_name,))
+        ]
+
+        filelist = os.listdir(self.backup_dir)
+        filelist.sort()
+        for item in filelist:
+            if item.endswith(".xva"):
+                dir_name = os.path.join(self.backup_dir, item)
+                if dir_name not in registered:
+                    start = (
+                        datetime.datetime.strptime(item, self.backup_name + "-%Y%m%d-%Hh%Mm%S.xva") + datetime.timedelta(0, 30 * 60)
+                    ).isoformat()
+                    if fileisodate(dir_name) > start:
+                        stop = fileisodate(dir_name)
+                    else:
+                        stop = start
+                    self.logger.info("Registering %s started on %s", dir_name, start)
+                    self.logger.debug("  Disk usage %s", 'du -sb "%s"' % dir_name)
+                    if not self.dry_run:
+                        size_bytes = int(os.popen('du -sb "%s"' % dir_name).read().split("\t")[0])
+                    else:
+                        size_bytes = 0
+                    self.logger.debug("  Size in bytes : %i", size_bytes)
+                    if not self.dry_run:
+                        self.dbstat.add(
+                            self.backup_name,
+                            self.server_name,
+                            "",
+                            backup_start=start,
+                            backup_end=stop,
+                            status="OK",
+                            total_bytes=size_bytes,
+                            backup_location=dir_name,
+                            TYPE="BACKUP",
+                        )
+                else:
+                    self.logger.info("Skipping %s, already registered", dir_name)
+
+
+register_driver(backup_xva)
+
+if __name__ == "__main__":
+    logger = logging.getLogger("tisbackup")
+    logger.setLevel(logging.DEBUG)
+    formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+    cp = ConfigParser()
+    cp.read("/opt/tisbackup/configtest.ini")
+    b = backup_xva()
+    b.read_config(cp)

libtisbackup/drivers/copy_vm_xcp.py

@@ -0,0 +1,287 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------
+#    This file is part of TISBackup
+#
+#    TISBackup is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    TISBackup is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with TISBackup.  If not, see <http://www.gnu.org/licenses/>.
+#
+# -----------------------------------------------------------------------
+
+import base64
+import datetime
+import logging
+import os
+import os.path
+import re
+import select
+import socket
+import ssl
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from stat import *
+
+from . import XenAPI
+from libtisbackup import *
+
+if hasattr(ssl, "_create_unverified_context"):
+    ssl._create_default_https_context = ssl._create_unverified_context
+
+
+class copy_vm_xcp(backup_generic):
+    """Backup a VM running on a XCP server on a second SR (requires xe tools and XenAPI)"""
+
+    type = "copy-vm-xcp"
+
+    required_params = backup_generic.required_params + ["server_name", "storage_name", "password_file", "vm_name", "network_name"]
+    optional_params = backup_generic.optional_params + ["start_vm", "max_copies", "delete_snapshot", "halt_vm"]
+
+    start_vm = "no"
+    max_copies = 1
+    halt_vm = "no"
+    delete_snapshot = "yes"
+
+    def read_config(self, iniconf):
+        assert isinstance(iniconf, ConfigParser)
+        backup_generic.read_config(self, iniconf)
+        if self.start_vm in "no" and iniconf.has_option("global", "start_vm"):
+            self.start_vm = iniconf.get("global", "start_vm")
+        if self.max_copies == 1 and iniconf.has_option("global", "max_copies"):
+            self.max_copies = iniconf.getint("global", "max_copies")
+        if self.delete_snapshot == "yes" and iniconf.has_option("global", "delete_snapshot"):
+            self.delete_snapshot = iniconf.get("global", "delete_snapshot")
+
+    def copy_vm_to_sr(self, vm_name, storage_name, dry_run, delete_snapshot="yes"):
+        user_xen, password_xen, null = open(self.password_file).read().split("\n")
+        session = XenAPI.Session("https://" + self.server_name)
+        try:
+            session.login_with_password(user_xen, password_xen)
+        except XenAPI.Failure as error:
+            msg, ip = error.details
+
+            if msg == "HOST_IS_SLAVE":
+                server_name = ip
+                session = XenAPI.Session("https://" + server_name)
+                session.login_with_password(user_xen, password_xen)
+
+        self.logger.debug("[%s] VM (%s) to backup in storage: %s", self.backup_name, vm_name, storage_name)
+        now = datetime.datetime.now()
+
+        # get storage opaqueRef
+        try:
+            storage = session.xenapi.SR.get_by_name_label(storage_name)[0]
+        except IndexError as error:
+            result = (1, "error get SR opaqueref %s" % (error))
+            return result
+
+        # get vm to copy opaqueRef
+        try:
+            vm = session.xenapi.VM.get_by_name_label(vm_name)[0]
+        except IndexError as error:
+            result = (1, "error get VM opaqueref %s" % (error))
+            return result
+
+        # get vm backup network opaqueRef
+        try:
+            networkRef = session.xenapi.network.get_by_name_label(self.network_name)[0]
+        except IndexError as error:
+            result = (1, "error get VM network opaqueref %s" % (error))
+            return result
+
+        if str2bool(self.halt_vm):
+            status_vm = session.xenapi.VM.get_power_state(vm)
+            self.logger.debug("[%s] Status of VM: %s", self.backup_name, status_vm)
+            if status_vm == "Running":
+                self.logger.debug("[%s] Shutdown in progress", self.backup_name)
+                if dry_run:
+                    print("session.xenapi.VM.clean_shutdown(vm)")
+                else:
+                    session.xenapi.VM.clean_shutdown(vm)
+            snapshot = vm
+        else:
+            # do the snapshot
+            self.logger.debug("[%s] Snapshot in progress", self.backup_name)
+            try:
+                snapshot = session.xenapi.VM.snapshot(vm, "tisbackup-%s" % (vm_name))
+            except XenAPI.Failure as error:
+                result = (1, "error when snapshot %s" % (error))
+                return result
+
+            # get snapshot opaqueRef
+            snapshot = session.xenapi.VM.get_by_name_label("tisbackup-%s" % (vm_name))[0]
+            session.xenapi.VM.set_name_description(snapshot, "snapshot created by tisbackup on : %s" % (now.strftime("%Y-%m-%d %H:%M")))
+
+        vm_backup_name = "zzz-%s-" % (vm_name)
+
+        # Check if old backup exit
+        list_backups = []
+        for vm_ref in session.xenapi.VM.get_all():
+            name_lablel = session.xenapi.VM.get_name_label(vm_ref)
+            if vm_backup_name in name_lablel:
+                list_backups.append(name_lablel)
+
+        list_backups.sort()
+
+        if len(list_backups) >= 1:
+            # Shutting last backup if started
+            last_backup_vm = session.xenapi.VM.get_by_name_label(list_backups[-1])[0]
+            if "Halted" not in session.xenapi.VM.get_power_state(last_backup_vm):
+                self.logger.debug("[%s] Shutting down last backup vm : %s", self.backup_name, list_backups[-1])
+                session.xenapi.VM.hard_shutdown(last_backup_vm)
+
+            # Delete oldest backup if exist
+            if len(list_backups) >= int(self.max_copies):
+                for i in range(len(list_backups) - int(self.max_copies) + 1):
+                    oldest_backup_vm = session.xenapi.VM.get_by_name_label(list_backups[i])[0]
+                    if "Halted" not in session.xenapi.VM.get_power_state(oldest_backup_vm):
+                        self.logger.debug("[%s] Shutting down old vm : %s", self.backup_name, list_backups[i])
+                        session.xenapi.VM.hard_shutdown(oldest_backup_vm)
+
+                    try:
+                        self.logger.debug("[%s] Deleting old vm : %s", self.backup_name, list_backups[i])
+                        for vbd in session.xenapi.VM.get_VBDs(oldest_backup_vm):
+                            if session.xenapi.VBD.get_type(vbd) == "CD" and not session.xenapi.VBD.get_record(vbd)["empty"]:
+                                session.xenapi.VBD.eject(vbd)
+                            else:
+                                vdi = session.xenapi.VBD.get_VDI(vbd)
+                                if "NULL" not in vdi:
+                                    session.xenapi.VDI.destroy(vdi)
+
+                        session.xenapi.VM.destroy(oldest_backup_vm)
+                    except XenAPI.Failure as error:
+                        result = (1, "error when destroy old backup vm %s" % (error))
+                        return result
+
+        self.logger.debug("[%s] Copy %s in progress on %s", self.backup_name, vm_name, storage_name)
+        try:
+            backup_vm = session.xenapi.VM.copy(snapshot, vm_backup_name + now.strftime("%Y-%m-%d %H:%M"), storage)
+        except XenAPI.Failure as error:
+            result = (1, "error when copy %s" % (error))
+            return result
+
+        # define VM as a template
+        session.xenapi.VM.set_is_a_template(backup_vm, False)
+
+        # change the network of the new VM
+        try:
+            vifDestroy = session.xenapi.VM.get_VIFs(backup_vm)
+        except IndexError as error:
+            result = (1, "error get VIF opaqueref %s" % (error))
+            return result
+
+        for i in vifDestroy:
+            vifRecord = session.xenapi.VIF.get_record(i)
+            session.xenapi.VIF.destroy(i)
+            data = {
+                "MAC": vifRecord["MAC"],
+                "MAC_autogenerated": False,
+                "MTU": vifRecord["MTU"],
+                "VM": backup_vm,
+                "current_operations": vifRecord["current_operations"],
+                "currently_attached": vifRecord["currently_attached"],
+                "device": vifRecord["device"],
+                "ipv4_allowed": vifRecord["ipv4_allowed"],
+                "ipv6_allowed": vifRecord["ipv6_allowed"],
+                "locking_mode": vifRecord["locking_mode"],
+                "network": networkRef,
+                "other_config": vifRecord["other_config"],
+                "qos_algorithm_params": vifRecord["qos_algorithm_params"],
+                "qos_algorithm_type": vifRecord["qos_algorithm_type"],
+                "qos_supported_algorithms": vifRecord["qos_supported_algorithms"],
+                "runtime_properties": vifRecord["runtime_properties"],
+                "status_code": vifRecord["status_code"],
+                "status_detail": vifRecord["status_detail"],
+            }
+            try:
+                session.xenapi.VIF.create(data)
+            except Exception as error:
+                result = (1, error)
+                return result
+
+        if self.start_vm in ["true", "1", "t", "y", "yes", "oui"]:
+            session.xenapi.VM.start(backup_vm, False, True)
+
+        session.xenapi.VM.set_name_description(backup_vm, "snapshot created by tisbackup on : %s" % (now.strftime("%Y-%m-%d %H:%M")))
+
+        size_backup = 0
+        for vbd in session.xenapi.VM.get_VBDs(backup_vm):
+            if session.xenapi.VBD.get_type(vbd) == "CD" and not session.xenapi.VBD.get_record(vbd)["empty"]:
+                session.xenapi.VBD.eject(vbd)
+            else:
+                vdi = session.xenapi.VBD.get_VDI(vbd)
+                if "NULL" not in vdi:
+                    size_backup = size_backup + int(session.xenapi.VDI.get_record(vdi)["physical_utilisation"])
+
+        result = (0, size_backup)
+        if self.delete_snapshot == "no":
+            return result
+
+        # Disable automatic boot
+        if "auto_poweron" in session.xenapi.VM.get_other_config(backup_vm):
+            session.xenapi.VM.remove_from_other_config(backup_vm, "auto_poweron")
+
+        if not str2bool(self.halt_vm):
+            # delete the snapshot
+            try:
+                for vbd in session.xenapi.VM.get_VBDs(snapshot):
+                    if session.xenapi.VBD.get_type(vbd) == "CD" and not session.xenapi.VBD.get_record(vbd)["empty"]:
+                        session.xenapi.VBD.eject(vbd)
+                    else:
+                        vdi = session.xenapi.VBD.get_VDI(vbd)
+                        if "NULL" not in vdi:
+                            session.xenapi.VDI.destroy(vdi)
+                session.xenapi.VM.destroy(snapshot)
+            except XenAPI.Failure as error:
+                result = (1, "error when destroy snapshot %s" % (error))
+                return result
+        else:
+            if status_vm == "Running":
+                self.logger.debug("[%s] Starting in progress", self.backup_name)
+                if dry_run:
+                    print("session.xenapi.VM.start(vm,False,True)")
+                else:
+                    session.xenapi.VM.start(vm, False, True)
+
+        return result
+
+    def do_backup(self, stats):
+        try:
+            # timestamp = int(time.time())
+            cmd = self.copy_vm_to_sr(self.vm_name, self.storage_name, self.dry_run, delete_snapshot=self.delete_snapshot)
+
+            if cmd[0] == 0:
+                # timeExec = int(time.time()) - timestamp
+                stats["log"] = "copy of %s to an other storage OK" % (self.backup_name)
+                stats["status"] = "OK"
+                stats["total_files_count"] = 1
+                stats["total_bytes"] = cmd[1]
+
+                stats["backup_location"] = self.storage_name
+            else:
+                stats["status"] = "ERROR"
+                stats["log"] = cmd[1]
+
+        except BaseException as e:
+            stats["status"] = "ERROR"
+            stats["log"] = str(e)
+            raise
+
+    def register_existingbackups(self):
+        """scan backup dir and insert stats in database"""
+        # This backup is on target server, no data available on this server
+        pass
+
+
+register_driver(copy_vm_xcp)