migrate to Python3 (from alejeune)

libtisbackup/XenAPI.py

@@ -1,20 +1,30 @@
-#============================================================================
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of version 2.1 of the GNU Lesser General Public
-# License as published by the Free Software Foundation.
+# Copyright (c) Citrix Systems, Inc.
+# All rights reserved.
 #
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
 #
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-#============================================================================
-# Copyright (C) 2006-2007 XenSource Inc.
-#============================================================================
+#  1) Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
 #
+#  2) Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# --------------------------------------------------------------------
 # Parts of this file are based upon xmlrpclib.py, the XML-RPC client
 # interface included in the Python distribution.
 #
@@ -45,27 +55,16 @@
 # --------------------------------------------------------------------
 
 import gettext
-import xmlrpclib
-import httplib
+import six.moves.xmlrpc_client as xmlrpclib
+import six.moves.http_client as httplib
 import socket
+import sys
 
 translation = gettext.translation('xen-xm', fallback = True)
 
 API_VERSION_1_1 = '1.1'
 API_VERSION_1_2 = '1.2'
 
-#
-# Methods that have different parameters between API versions 1.1 and 1.2, and
-# the number of parameters in 1.1.
-#
-COMPATIBILITY_METHODS_1_1 = [
-    ('SR.create'     , 8),
-    ('SR.introduce'  , 6),
-    ('SR.make'       , 7),
-    ('VDI.snapshot'  , 1),
-    ('VDI.clone'     , 1),
-    ]
-
 class Failure(Exception):
     def __init__(self, details):
         self.details = details
@@ -73,17 +72,18 @@ class Failure(Exception):
     def __str__(self):
         try:
             return str(self.details)
-        except Exception, exn:
-            import sys
-            print >>sys.stderr, exn
-            return "Xen-API failure: %s" % str(self.details)
+        except Exception as exn:
+            msg = "Xen-API failure: %s" % exn
+            sys.stderr.write(msg)
+            return msg
 
     def _details_map(self):
         return dict([(str(i), self.details[i])
                      for i in range(len(self.details))])
 
 
-_RECONNECT_AND_RETRY = (lambda _ : ())
+# Just a "constant" that we use to decide whether to retry the RPC
+_RECONNECT_AND_RETRY = object()
 
 class UDSHTTPConnection(httplib.HTTPConnection):
     """HTTPConnection subclass to allow HTTP over Unix domain sockets. """
@@ -92,12 +92,26 @@ class UDSHTTPConnection(httplib.HTTPConnection):
         self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
         self.sock.connect(path)
 
-class UDSHTTP(httplib.HTTP):
+class UDSHTTP(httplib.HTTPConnection):
     _connection_class = UDSHTTPConnection
 
 class UDSTransport(xmlrpclib.Transport):
+    def __init__(self, use_datetime=0):
+        self._use_datetime = use_datetime
+        self._extra_headers=[]
+        self._connection = (None, None)
+    def add_extra_header(self, key, value):
+        self._extra_headers += [ (key,value) ]
     def make_connection(self, host):
-        return UDSHTTP(host)
+        # Python 2.4 compatibility
+        if sys.version_info[0] <= 2 and sys.version_info[1] < 7:
+            return UDSHTTP(host)
+        else:
+            return UDSHTTPConnection(host)
+    def send_request(self, connection, handler, request_body):
+        connection.putrequest("POST", handler)
+        for key, value in self._extra_headers:
+            connection.putheader(key, value)
 
 class Session(xmlrpclib.ServerProxy):
     """A server proxy and session manager for communicating with xapi using
@@ -106,15 +120,26 @@ class Session(xmlrpclib.ServerProxy):
     Example:
 
     session = Session('http://localhost/')
-    session.login_with_password('me', 'mypassword')
+    session.login_with_password('me', 'mypassword', '1.0', 'xen-api-scripts-xenapi.py')
     session.xenapi.VM.start(vm_uuid)
     session.xenapi.session.logout()
     """
 
     def __init__(self, uri, transport=None, encoding=None, verbose=0,
-                 allow_none=1):
-        xmlrpclib.ServerProxy.__init__(self, uri, transport, encoding,
-                                       verbose, allow_none)
+                 allow_none=1, ignore_ssl=False):
+
+        # Fix for CA-172901 (+ Python 2.4 compatibility)
+        # Fix for context=ctx ( < Python 2.7.9 compatibility)
+        if not (sys.version_info[0] <= 2 and sys.version_info[1] <= 7 and sys.version_info[2] <= 9 ) \
+                and ignore_ssl:
+            import ssl
+            ctx = ssl._create_unverified_context()
+            xmlrpclib.ServerProxy.__init__(self, uri, transport, encoding,
+                                           verbose, allow_none, context=ctx)
+        else:
+            xmlrpclib.ServerProxy.__init__(self, uri, transport, encoding,
+                                           verbose, allow_none)
+        self.transport = transport
         self._session = None
         self.last_login_method = None
         self.last_login_params = None
@@ -125,7 +150,7 @@ class Session(xmlrpclib.ServerProxy):
         if methodname.startswith('login'):
             self._login(methodname, params)
             return None
-        elif methodname == 'logout':
+        elif methodname == 'logout' or methodname == 'session.logout':
             self._logout()
             return None
         else:
@@ -133,7 +158,7 @@ class Session(xmlrpclib.ServerProxy):
             while retry_count < 3:
                 full_params = (self._session,) + params
                 result = _parse_result(getattr(self, methodname)(*full_params))
-                if result == _RECONNECT_AND_RETRY:
+                if result is _RECONNECT_AND_RETRY:
                     retry_count += 1
                     if self.last_login_method:
                         self._login(self.last_login_method,
@@ -145,21 +170,24 @@ class Session(xmlrpclib.ServerProxy):
             raise xmlrpclib.Fault(
                 500, 'Tried 3 times to get a valid session, but failed')
 
-
     def _login(self, method, params):
-        result = _parse_result(getattr(self, 'session.%s' % method)(*params))
-        if result == _RECONNECT_AND_RETRY:
-            raise xmlrpclib.Fault(
-                500, 'Received SESSION_INVALID when logging in')
-        self._session = result
-        self.last_login_method = method
-        self.last_login_params = params
-        if method.startswith("slave_local"):
-            self.API_version = API_VERSION_1_2
-        else:
+        try:
+            result = _parse_result(
+                getattr(self, 'session.%s' % method)(*params))
+            if result is _RECONNECT_AND_RETRY:
+                raise xmlrpclib.Fault(
+                    500, 'Received SESSION_INVALID when logging in')
+            self._session = result
+            self.last_login_method = method
+            self.last_login_params = params
             self.API_version = self._get_api_version()
+        except socket.error as e:
+            if e.errno == socket.errno.ETIMEDOUT:
+                raise xmlrpclib.Fault(504, 'The connection timed out')
+            else:
+                raise e
 
-    def logout(self):
+    def _logout(self):
         try:
             if self.last_login_method.startswith("slave_local"):
                 return _parse_result(self.session.local_logout(self._session))
@@ -174,11 +202,9 @@ class Session(xmlrpclib.ServerProxy):
     def _get_api_version(self):
         pool = self.xenapi.pool.get_all()[0]
         host = self.xenapi.pool.get_master(pool)
-        if (self.xenapi.host.get_API_version_major(host) == "1" and
-            self.xenapi.host.get_API_version_minor(host) == "2"):
-            return API_VERSION_1_2
-        else:
-            return API_VERSION_1_1
+        major = self.xenapi.host.get_API_version_major(host)
+        minor = self.xenapi.host.get_API_version_minor(host)
+        return "%s.%s"%(major,minor)
 
     def __getattr__(self, name):
         if name == 'handle':
@@ -187,11 +213,13 @@ class Session(xmlrpclib.ServerProxy):
             return _Dispatcher(self.API_version, self.xenapi_request, None)
         elif name.startswith('login') or name.startswith('slave_local'):
             return lambda *params: self._login(name, params)
+        elif name == 'logout':
+            return _Dispatcher(self.API_version, self.xenapi_request, "logout")
         else:
             return xmlrpclib.ServerProxy.__getattr__(self, name)
 
 def xapi_local():
-    return Session("http://_var_xapi_xapi/", transport=UDSTransport())
+    return Session("http://_var_lib_xcp_xapi/", transport=UDSTransport())
 
 def _parse_result(result):
     if type(result) != dict or 'Status' not in result:
@@ -233,10 +261,4 @@ class _Dispatcher:
             return _Dispatcher(self.__API_version, self.__send, "%s.%s" % (self.__name, name))
 
     def __call__(self, *args):
-        if self.__API_version == API_VERSION_1_1:
-            for m in COMPATIBILITY_METHODS_1_1:
-                if self.__name == m[0]:
-                    return self.__send(self.__name, args[0:m[1]])
-
         return self.__send(self.__name, args)
-

libtisbackup/backup_mysql.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 # -----------------------------------------------------------------------
 #    This file is part of TISBackup
@@ -24,13 +24,13 @@ import sys
 try:
     sys.stderr = open('/dev/null')       # Silence silly warnings from paramiko
     import paramiko
-except ImportError,e:
-    print "Error : can not load paramiko library %s" % e
+except ImportError as e:
+    print(("Error : can not load paramiko library %s" % e))
     raise
 
 sys.stderr = sys.__stderr__
 
-from common import *
+from libtisbackup.common import *
 
 class backup_mysql(backup_generic):
     """Backup a mysql database as gzipped sql file through ssh"""
@@ -52,7 +52,7 @@ class backup_mysql(backup_generic):
             if not self.dry_run:
                 os.makedirs(self.dest_dir)
             else:
-                print 'mkdir "%s"' % self.dest_dir
+                print(('mkdir "%s"' % self.dest_dir))
         else:
             raise Exception('backup destination directory already exists : %s' % self.dest_dir)
 
@@ -101,7 +101,7 @@ class backup_mysql(backup_generic):
         self.logger.debug('[%s] Dump DB : %s',self.backup_name,cmd)
         if not self.dry_run:
             (error_code,output) = ssh_exec(cmd,ssh=self.ssh)
-            print output
+            print(output)
             self.logger.debug("[%s] Output of %s :\n%s",self.backup_name,cmd,output)
             if error_code:
                 raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd))
@@ -176,4 +176,4 @@ class backup_mysql(backup_generic):
                     self.logger.info('Skipping %s, already registered',dir_name)
 
 
-register_driver(backup_mysql)
+register_driver(backup_mysql)

libtisbackup/backup_null.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 # -----------------------------------------------------------------------
 #    This file is part of TISBackup
@@ -20,7 +20,7 @@
 
 import os
 import datetime
-from common import *
+from .common import *
 
 
 class backup_null(backup_generic):

libtisbackup/backup_oracle.py

@@ -21,8 +21,8 @@ import sys
 try:
     sys.stderr = open('/dev/null')       # Silence silly warnings from paramiko
     import paramiko
-except ImportError,e:
-    print "Error : can not load paramiko library %s" % e
+except ImportError as e:
+    print(("Error : can not load paramiko library %s" % e))
     raise
 
 sys.stderr = sys.__stderr__
@@ -30,7 +30,7 @@ sys.stderr = sys.__stderr__
 import datetime
 import base64
 import os
-from common import *
+from libtisbackup.common import *
 import re
 
 class backup_oracle(backup_generic):
@@ -49,8 +49,8 @@ class backup_oracle(backup_generic):
         try:
             mykey = paramiko.RSAKey.from_private_key_file(self.private_key)
         except paramiko.SSHException:
-            #mykey = paramiko.DSSKey.from_private_key_file(self.private_key)
-			mykey = paramiko.Ed25519Key.from_private_key_file(self.private_key)
+		#mykey = paramiko.DSSKey.from_private_key_file(self.private_key)
+                mykey = paramiko.Ed25519Key.from_private_key_file(self.private_key)
 
         self.ssh = paramiko.SSHClient()
         self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
@@ -66,7 +66,7 @@ class backup_oracle(backup_generic):
             if not self.dry_run:
                 os.makedirs(self.dest_dir)
             else:
-                print 'mkdir "%s"' % self.dest_dir
+                print(('mkdir "%s"' % self.dest_dir))
         else:
             raise Exception('backup destination directory already exists : %s' % self.dest_dir)        
         # dump db
@@ -171,4 +171,4 @@ class backup_oracle(backup_generic):
             if error_code:
                 raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd))
 
-register_driver(backup_oracle)
+register_driver(backup_oracle)

libtisbackup/backup_pgsql.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 # -----------------------------------------------------------------------
 #    This file is part of TISBackup
@@ -21,13 +21,13 @@ import sys
 try:
     sys.stderr = open('/dev/null')       # Silence silly warnings from paramiko
     import paramiko
-except ImportError,e:
-    print "Error : can not load paramiko library %s" % e
+except ImportError as e:
+    print(("Error : can not load paramiko library %s" % e))
     raise
 
 sys.stderr = sys.__stderr__
 
-from common import *
+from .common import *
 
 class backup_pgsql(backup_generic):
     """Backup a postgresql database as gzipped sql file through ssh"""
@@ -46,7 +46,7 @@ class backup_pgsql(backup_generic):
             if not self.dry_run:
                 os.makedirs(self.dest_dir)
             else:
-                print 'mkdir "%s"' % self.dest_dir
+                print(('mkdir "%s"' % self.dest_dir))
         else:
             raise Exception('backup destination directory already exists : %s' % self.dest_dir)
 

libtisbackup/backup_rdiff.py

@@ -1,127 +0,0 @@
-# -----------------------------------------------------------------------
-#    This file is part of TISBackup
-#
-#    TISBackup is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published by
-#    the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
-#
-#    TISBackup is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU General Public License for more details.
-#
-#    You should have received a copy of the GNU General Public License
-#    along with TISBackup.  If not, see <http://www.gnu.org/licenses/>.
-#
-# -----------------------------------------------------------------------
-
-
-import os
-import datetime
-from common import *
-import time
-
-class backup_rdiff:
-  backup_dir=''
-  backup_start_date=None
-  backup_name=''
-  server_name=''
-  exclude_list=''
-  ssh_port='22' 
-  remote_user='root'
-  remote_dir=''
-  dest_dir=''
-  verbose = False
-  dry_run=False
-
-
-
-  def __init__(self, backup_name, backup_base_dir):
-    self.backup_dir = backup_base_dir + '/' + backup_name
-
-    if os.path.isdir(self.backup_dir )==False:
-      os.makedirs(self.backup_dir)
- 
-    self.backup_name = backup_name
-    t = datetime.datetime.now()
-    self.backup_start_date = t.strftime('%Y%m%d-%Hh%Mm%S')
-
-  def get_latest_backup(self):
-    filelist = os.listdir(self.backup_dir)
-    if len(filelist) == 0:
-      return ''
-
-    filelist.sort()
-    
-    return filelist[-1]
-
-  def cleanup_backup(self):
-    filelist = os.listdir(self.backup_dir)
-    if len(filelist) == 0:
-      return ''
-
-    filelist.sort()
-    for backup_date in filelist:
-      today = time.time()
-      print backup_date
-      datestring =  backup_date[0:8] 
-      c = time.strptime(datestring,"%Y%m%d")
-      # TODO: improve
-      if today - c < 60 * 60 * 24* 30:
-        print time.strftime("%Y%m%d",c) + " is to be deleted"
-
-
-  def copy_latest_to_new(self):
-    # TODO check that latest exist
-    # TODO check that new does not exist
-
-
-    last_backup =  self.get_latest_backup()
-    if last_backup=='':
-      print "*********************************"
-      print "*first backup for " + self.backup_name
-    else:
-      latest_backup_path = self.backup_dir + '/' + last_backup 
-      new_backup_path = self.backup_dir + '/' + self.backup_start_date
-      print "#cp -al starting"
-      cmd = 'cp -al ' + latest_backup_path + ' ' + new_backup_path
-      print cmd
-      if self.dry_run==False:
-        call_external_process(cmd)
-      print "#cp -al finished"
-    
-
-  def rsync_to_new(self):
-
-    self.dest_dir = self.backup_dir + '/' + self.backup_start_date + '/'
-    src_server = self.remote_user + '@' + self.server_name + ':"' + self.remote_dir.strip() + '/"'
-
-    print "#starting rsync"
-    verbose_arg=""
-    if self.verbose==True:
-      verbose_arg = "-P "
-    
-    cmd = "rdiff-backup " + verbose_arg + ' --compress-level=9 --numeric-ids -az --partial -e "ssh -o StrictHostKeyChecking=no -p ' + self.ssh_port + ' -i ' + self.private_key + '" --stats --delete-after ' + self.exclude_list + ' ' + src_server + ' ' + self.dest_dir 
-    print cmd
-
-    ## deal with exit code 24 (file vanished)
-    if self.dry_run==False:
-      p = subprocess.call(cmd, shell=True)
-      if (p ==24):
-        print "Note: some files vanished before transfer"
-      if (p != 0 and p != 24 ):
-        raise Exception('shell program exited with error code ' + str(p), cmd)
-
-
-    print "#finished rsync"
-
-  def process_backup(self):
-    print ""
-    print "#========Starting backup item ========="
-    self.copy_latest_to_new()
-
-    self.rsync_to_new()     
-    print "#========Backup item finished=========="
-
-

libtisbackup/backup_rsync.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 # -----------------------------------------------------------------------
 #    This file is part of TISBackup
@@ -20,7 +20,7 @@
 
 import os
 import datetime
-from common import *
+from libtisbackup.common import *
 import time
 import logging
 import re
@@ -69,7 +69,7 @@ class backup_rsync(backup_generic):
                     if not self.dry_run:
                         os.makedirs(dest_dir)
                     else:
-                        print 'mkdir "%s"' % dest_dir
+                        print(('mkdir "%s"' % dest_dir))
                 else:
                     raise Exception('backup destination directory already exists : %s' % dest_dir)
 
@@ -80,7 +80,7 @@ class backup_rsync(backup_generic):
                 if self.dry_run:
                     options.append('-d')
 
-                if self.overload_args <> None:
+                if self.overload_args != None:
                     options.append(self.overload_args)
                 elif not "cygdrive" in self.remote_dir:
                     # we don't preserve owner, group, links, hardlinks, perms for windows/cygwin as it is not reliable nor useful
@@ -118,7 +118,7 @@ class backup_rsync(backup_generic):
                     try:
                         # newsettings with exclude_list='too','titi', parsed as a str python list content
                         excludes = eval('[%s]' % self.exclude_list)
-                    except Exception,e:
+                    except Exception as e:
                         raise Exception('Error reading exclude list : value %s, eval error %s (don\'t forget quotes and comma...)' % (self.exclude_list,e))
                 options.extend(['--exclude="%s"' % x for x in excludes])
 
@@ -146,13 +146,13 @@ class backup_rsync(backup_generic):
                         ssh_params.append('-i %s' % self.private_key)
                     if self.cipher_spec:
                         ssh_params.append('-c %s' % self.cipher_spec)
-                    if self.ssh_port <> 22:
+                    if self.ssh_port != 22:
                         ssh_params.append('-p %i' % self.ssh_port)
                     options.append('-e "/usr/bin/ssh %s"' % (" ".join(ssh_params)))
                     backup_source = '%s@%s:%s' % (self.remote_user,self.server_name,self.remote_dir)
 
                 # ensure there is a slash at end
-                if backup_source[-1] <> '/':
+                if backup_source[-1] != '/':
                     backup_source += '/'
 
                 options_params = " ".join(options)
@@ -165,7 +165,7 @@ class backup_rsync(backup_generic):
                     process = subprocess.Popen(cmd, shell=True,  stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
                     def ondata(data,context):
                         if context.verbose:
-                            print data
+                            print(data)
                         context.logger.debug(data)
 
                     log = monitor_stdout(process,ondata,self)
@@ -195,7 +195,7 @@ class backup_rsync(backup_generic):
                         self.logger.error("[" + self.backup_name + "] shell program exited with error code " + str(returncode))
                         raise Exception("[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd, log[-512:])
                 else:
-                    print cmd
+                    print(cmd)
 
                 #we suppress the .rsync suffix if everything went well
                 finaldest = os.path.join(self.backup_dir,self.backup_start_date)
@@ -203,14 +203,14 @@ class backup_rsync(backup_generic):
                 if not self.dry_run:
                     os.rename(dest_dir, finaldest)
                     self.logger.debug("[%s] touching datetime of target directory %s" ,self.backup_name,finaldest)
-                    print os.popen('touch "%s"' % finaldest).read()
+                    print((os.popen('touch "%s"' % finaldest).read()))
                 else:
-                    print "mv" ,dest_dir,finaldest
+                    print(("mv" ,dest_dir,finaldest))
                 stats['backup_location'] = finaldest
                 stats['status']='OK'
                 stats['log']='ssh+rsync backup from %s OK, %d bytes written for %d changed files' % (backup_source,stats['written_bytes'],stats['written_files_count'])
 
-            except BaseException , e:
+            except BaseException as e:
                 stats['status']='ERROR'
                 stats['log']=str(e)
                 raise
@@ -340,5 +340,5 @@ if __name__=='__main__':
     b = backup_rsync('htouvet','/backup/data/htouvet',dbstat)
     b.read_config(cp)
     b.process_backup()
-    print b.checknagios()
+    print((b.checknagios()))
 

libtisbackup/backup_rsync_btrfs.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 # -----------------------------------------------------------------------
 #    This file is part of TISBackup
@@ -20,13 +20,13 @@
 
 import os
 import datetime
-from common import *
+from .common import *
 import time
 import logging
 import re
 import os.path
 import datetime
-from common import *
+from .common import *
 
 
 class backup_rsync_btrfs(backup_generic):
@@ -78,7 +78,7 @@ class backup_rsync_btrfs(backup_generic):
                         else:
                             self.logger.info("[" + self.backup_name + "] create btrs volume: %s"%dest_dir)
                     else:
-                        print 'btrfs subvolume create "%s"' %dest_dir
+                        print(('btrfs subvolume create "%s"' %dest_dir))
 
 
 
@@ -89,7 +89,7 @@ class backup_rsync_btrfs(backup_generic):
                 if self.dry_run:
                     options.append('-d')
 
-                if self.overload_args <> None:
+                if self.overload_args != None:
                     options.append(self.overload_args)
                 elif not "cygdrive" in self.remote_dir:
                     # we don't preserve owner, group, links, hardlinks, perms for windows/cygwin as it is not reliable nor useful
@@ -128,7 +128,7 @@ class backup_rsync_btrfs(backup_generic):
                     try:
                         # newsettings with exclude_list='too','titi', parsed as a str python list content
                         excludes = eval('[%s]' % self.exclude_list)
-                    except Exception,e:
+                    except Exception as e:
                         raise Exception('Error reading exclude list : value %s, eval error %s (don\'t forget quotes and comma...)' % (self.exclude_list,e))
                 options.extend(['--exclude="%s"' % x for x in excludes])
 
@@ -154,13 +154,13 @@ class backup_rsync_btrfs(backup_generic):
                         ssh_params.append('-i %s' % self.private_key)
                     if self.cipher_spec:
                         ssh_params.append('-c %s' % self.cipher_spec)
-                    if self.ssh_port <> 22:
+                    if self.ssh_port != 22:
                         ssh_params.append('-p %i' % self.ssh_port)
                     options.append('-e "/usr/bin/ssh %s"' % (" ".join(ssh_params)))
                     backup_source = '%s@%s:%s' % (self.remote_user,self.server_name,self.remote_dir)
 
                 # ensure there is a slash at end
-                if backup_source[-1] <> '/':
+                if backup_source[-1] != '/':
                     backup_source += '/'
 
                 options_params = " ".join(options)
@@ -173,7 +173,7 @@ class backup_rsync_btrfs(backup_generic):
                     process = subprocess.Popen(cmd, shell=True,  stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
                     def ondata(data,context):
                         if context.verbose:
-                            print data
+                            print(data)
                         context.logger.debug(data)
 
                     log = monitor_stdout(process,ondata,self)
@@ -203,7 +203,7 @@ class backup_rsync_btrfs(backup_generic):
                         self.logger.error("[" + self.backup_name + "] shell program exited with error code ", str(returncode))
                         raise Exception("[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd, log[-512:])
                 else:
-                    print cmd
+                    print(cmd)
 
                 #we take a snapshot of last_backup if everything went well
                 finaldest = os.path.join(self.backup_dir,self.backup_start_date)
@@ -220,16 +220,16 @@ class backup_rsync_btrfs(backup_generic):
                         else:
                             self.logger.info("[" + self.backup_name + "] snapshot directory created %s"%finaldest)
                     else:
-                        print "btrfs snapshot of %s to %s"%(dest_dir,finaldest)
+                        print(("btrfs snapshot of %s to %s"%(dest_dir,finaldest)))
                 else:
                     raise Exception('snapshot directory already exists : %s' %finaldest)
                 self.logger.debug("[%s] touching datetime of target directory %s" ,self.backup_name,finaldest)
-                print os.popen('touch "%s"' % finaldest).read()
+                print((os.popen('touch "%s"' % finaldest).read()))
                 stats['backup_location'] = finaldest
                 stats['status']='OK'
                 stats['log']='ssh+rsync+btrfs backup from %s OK, %d bytes written for %d changed files' % (backup_source,stats['written_bytes'],stats['written_files_count'])
 
-            except BaseException , e:
+            except BaseException as e:
                 stats['status']='ERROR'
                 stats['log']=str(e)
                 raise
@@ -358,5 +358,5 @@ if __name__=='__main__':
     b = backup_rsync('htouvet','/backup/data/htouvet',dbstat)
     b.read_config(cp)
     b.process_backup()
-    print b.checknagios()
+    print((b.checknagios()))
 

libtisbackup/backup_samba4.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 # -----------------------------------------------------------------------
 #    This file is part of TISBackup
@@ -24,13 +24,13 @@ import sys
 try:
     sys.stderr = open('/dev/null')       # Silence silly warnings from paramiko
     import paramiko
-except ImportError,e:
-    print "Error : can not load paramiko library %s" % e
+except ImportError as e:
+    print("Error : can not load paramiko library %s" % e)
     raise
 
 sys.stderr = sys.__stderr__
 
-from common import *
+from .common import *
 
 class backup_samba4(backup_generic):
     """Backup a samba4 databases as gzipped tdbs file through ssh"""
@@ -47,7 +47,7 @@ class backup_samba4(backup_generic):
             if not self.dry_run:
                 os.makedirs(self.dest_dir)
             else:
-                print 'mkdir "%s"' % self.dest_dir
+                print('mkdir "%s"' % self.dest_dir)
         else:
             raise Exception('backup destination directory already exists : %s' % self.dest_dir)
 
@@ -88,7 +88,7 @@ class backup_samba4(backup_generic):
         self.logger.debug('[%s] Dump DB : %s',self.backup_name,cmd)
         if not self.dry_run:
             (error_code,output) = ssh_exec(cmd,ssh=self.ssh)
-            print output
+            print(output)
             self.logger.debug("[%s] Output of %s :\n%s",self.backup_name,cmd,output)
             if error_code:
                 raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd))

libtisbackup/backup_sqlserver.py

@@ -24,8 +24,8 @@ import sys
 try:
     sys.stderr = open('/dev/null')       # Silence silly warnings from paramiko
     import paramiko
-except ImportError,e:
-    print "Error : can not load paramiko library %s" % e
+except ImportError as e:
+    print("Error : can not load paramiko library %s" % e)
     raise
 
 sys.stderr = sys.__stderr__
@@ -33,7 +33,7 @@ sys.stderr = sys.__stderr__
 import datetime
 import base64
 import os
-from common import *
+from .common import *
 
 class backup_sqlserver(backup_generic):
     """Backup a SQLSERVER database as gzipped sql file through ssh"""
@@ -67,18 +67,17 @@ class backup_sqlserver(backup_generic):
         backup_start_date =  t.strftime('%Y%m%d-%Hh%Mm%S')
 
         backup_file = self.remote_backup_dir + '/' + self.db_name  + '-' + backup_start_date + '.bak'
-    	if not self.db_user == '':
-	   self.userdb = '-U %s -P %s' % ( self.db_user, self.db_password )
+        if not self.db_user == '':
+                self.userdb = '-U %s -P %s' % ( self.db_user, self.db_password )
 
         # dump db
         stats['status']='Dumping'
-	if self.sqlserver_before_2005:
-		cmd =  """osql -E -Q "BACKUP DATABASE [%s]
+        if self.sqlserver_before_2005:
+                cmd =  """osql -E -Q "BACKUP DATABASE [%s]
 				      TO DISK='%s'
 				      WITH FORMAT" """ % ( self.db_name, backup_file )
-
-	else:
-        	cmd =  """sqlcmd %s -S "%s" -d master -Q "BACKUP DATABASE [%s]
+        else:
+                cmd =  """sqlcmd %s -S "%s" -d master -Q "BACKUP DATABASE [%s]
                 	                                       TO DISK = N'%s'
                         	                               WITH INIT, NOUNLOAD ,
                                 	                       NAME = N'Backup %s', NOSKIP ,STATS = 10, NOFORMAT" """ % (self.userdb, self.db_server_name, self.db_name, backup_file ,self.db_name )

libtisbackup/backup_switch.py

@@ -20,15 +20,15 @@
 
 import os
 import datetime
-from common import *
-import XenAPI
+from .common import *
+from . import XenAPI
 import time
 import logging
 import re
 import os.path
 import datetime
 import select
-import urllib2, urllib
+import urllib.request, urllib.error, urllib.parse, urllib.request, urllib.parse, urllib.error
 import base64
 import socket
 import requests
@@ -149,8 +149,9 @@ class backup_switch(backup_generic):
         else:
             child.sendline(self.switch_user)
         child.expect(".*#")
-	child.sendline( "terminal datadump")
-	child.expect("#")
+	
+        child.sendline( "terminal datadump")
+        child.expect("#")
         child.sendline( "show startup-config")
         child.expect("#")
         lines = child.before
@@ -237,7 +238,7 @@ class backup_switch(backup_generic):
             stats['log']='Switch backup from %s OK, %d bytes written' % (self.server_name,stats['written_bytes'])
 
 
-        except BaseException , e:
+        except BaseException as e:
             stats['status']='ERROR'
             stats['log']=str(e)
             raise

libtisbackup/backup_vmdk.py

@@ -17,8 +17,8 @@
 #    along with TISBackup.  If not, see <http://www.gnu.org/licenses/>.
 #
 # -----------------------------------------------------------------------
-from __future__ import with_statement
-from common import *
+
+from .common import *
 import pyVmomi
 from pyVmomi import vim
 from pyVmomi import vmodl
@@ -101,7 +101,7 @@ class backup_vmdk(backup_generic):
         ovfDescParams = vim.OvfManager.CreateDescriptorParams()
         ovf = si.content.ovfManager.CreateDescriptor(vm, ovfDescParams)     
         root = ET.fromstring(ovf.ovfDescriptor)  
-        new_id = root[0][1].attrib.values()[0][1:3]
+        new_id = list(root[0][1].attrib.values())[0][1:3]
         ovfFiles = []
         for vmdk in vmdks:
             old_id =  vmdk['id'][1:3]
@@ -211,7 +211,7 @@ class backup_vmdk(backup_generic):
                 if not self.dry_run:
                     os.makedirs(dest_dir)
                 else:
-                    print 'mkdir "%s"' % dest_dir
+                    print('mkdir "%s"' % dest_dir)
             else:
                 raise Exception('backup destination directory already exists : %s' % dest_dir)            
             os.chdir(dest_dir)
@@ -271,7 +271,7 @@ class backup_vmdk(backup_generic):
             stats['status']='OK'    
             
 
-        except BaseException , e:
+        except BaseException as e:
             stats['status']='ERROR'
             stats['log']=str(e)
             raise

libtisbackup/backup_xcp_metadata.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 # -----------------------------------------------------------------------
 #    This file is part of TISBackup
@@ -20,7 +20,7 @@
 
 
 
-from common import *
+from .common import *
 import paramiko
 
 class backup_xcp_metadata(backup_generic):

libtisbackup/backup_xva.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 # -----------------------------------------------------------------------
 #    This file is part of TISBackup
@@ -17,20 +17,21 @@
 #    along with TISBackup.  If not, see <http://www.gnu.org/licenses/>.
 #
 # -----------------------------------------------------------------------
-from __future__ import with_statement
+
 import logging
 import re
 import os
 import datetime
-import urllib
+import urllib.request, urllib.parse, urllib.error
 import socket
 import tarfile
 import hashlib
 from stat import *
 import ssl
+import requests
 
-from common import *
-import XenAPI
+from .common import *
+from . import XenAPI
 
 if hasattr(ssl, '_create_unverified_context'):
     ssl._create_default_https_context = ssl._create_unverified_context
@@ -72,7 +73,7 @@ class backup_xva(backup_generic):
         session = XenAPI.Session('https://'+self.xcphost)
         try:
             session.login_with_password(user_xen,password_xen)
-        except XenAPI.Failure, error:
+        except XenAPI.Failure as error:
             msg,ip = error.details
 
             if msg == 'HOST_IS_SLAVE':
@@ -117,7 +118,7 @@ class backup_xva(backup_generic):
                                 if not 'NULL' in  vdi:
                                     session.xenapi.VDI.destroy(vdi)
                         session.xenapi.VM.destroy(old_snapshot)
-                    except XenAPI.Failure, error:
+                    except XenAPI.Failure as error:
                         return("error when destroy snapshot %s"%(error))
 
                 now = datetime.datetime.now()
@@ -125,7 +126,7 @@ class backup_xva(backup_generic):
                 try:
                     snapshot = session.xenapi.VM.snapshot(vm,"tisbackup-%s"%(vdi_name))
                     self.logger.debug("[%s] got snapshot %s", vdi_name, snapshot)
-                except XenAPI.Failure, error:
+                except XenAPI.Failure as error:
                     return("error when snapshot %s"%(error))
                 #get snapshot opaqueRef
                 vm = session.xenapi.VM.get_by_name_label("tisbackup-%s"%(vdi_name))[0]
@@ -135,7 +136,7 @@ class backup_xva(backup_generic):
             if status_vm == "Running":
                 self.logger.debug("[%s] Shudown in progress",self.backup_name)
                 if dry_run:
-                    print "session.xenapi.VM.clean_shutdown(vm)"
+                    print("session.xenapi.VM.clean_shutdown(vm)")
                 else:
                     session.xenapi.VM.clean_shutdown(vm)
         try:
@@ -150,8 +151,13 @@ class backup_xva(backup_generic):
                     scheme = "https://"
                 url = scheme+user_xen+":"+password_xen+"@"+self.xcphost+"/export?use_compression="+self.use_compression+"&uuid="+session.xenapi.VM.get_uuid(vm)
 
-                urllib.urlretrieve(url, filename_temp)
-                urllib.urlcleanup()
+
+
+
+                top_level_url = scheme+self.xcphost+"/export?use_compression="+self.use_compression+"&uuid="+session.xenapi.VM.get_uuid(vm)
+                r = requests.get(top_level_url, auth=(user_xen, password_xen))
+                open(filename_temp, 'wb').write(r.content)
+
 
             except Exception as e:
                 self.logger.error("[%s] error when fetching snap: %s", "tisbackup-%s"%(vdi_name), e)
@@ -171,13 +177,13 @@ class backup_xva(backup_generic):
                             if not 'NULL' in  vdi:
                                 session.xenapi.VDI.destroy(vdi)
                     session.xenapi.VM.destroy(snapshot)
-                except XenAPI.Failure, error:
+                except XenAPI.Failure as error:
                     return("error when destroy snapshot %s"%(error))
 
             elif status_vm == "Running":
                 self.logger.debug("[%s] Starting in progress",self.backup_name)
                 if dry_run:
-                    print "session.xenapi.Async.VM.start(vm,False,True)"
+                    print("session.xenapi.Async.VM.start(vm,False,True)")
                 else:
                     session.xenapi.Async.VM.start(vm,False,True)
 
@@ -219,7 +225,7 @@ class backup_xva(backup_generic):
             else:
                 raise Exception(cmd)
 
-        except BaseException , e:
+        except BaseException as e:
             stats['status']='ERROR'
             stats['log']=str(e)
             raise

libtisbackup/common.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 # -----------------------------------------------------------------------
 #    This file is part of TISBackup
@@ -18,6 +18,7 @@
 #
 # -----------------------------------------------------------------------
 
+from abc import ABC, abstractmethod
 import os
 import subprocess
 import re
@@ -34,8 +35,8 @@ import sys
 try:
     sys.stderr = open('/dev/null')       # Silence silly warnings from paramiko
     import paramiko
-except ImportError,e:
-    print "Error : can not load paramiko library %s" % e
+except ImportError as e:
+    print(("Error : can not load paramiko library %s" % e))
     raise
 
 sys.stderr = sys.__stderr__
@@ -121,7 +122,7 @@ def check_string(test_string):
     pattern = r'[^\.A-Za-z0-9\-_]'
     if re.search(pattern, test_string):
         #Character other then . a-z 0-9 was found
-        print 'Invalid : %r' % (test_string,)
+        print(('Invalid : %r' % (test_string,)))
 
 def convert_bytes(bytes):
     if bytes is None:
@@ -207,7 +208,7 @@ def html_table(cur,callback=None):
             yield dict((cur.description[idx][0], value)
                        for idx, value in enumerate(row))
 
-    head=u"<tr>"+"".join(["<th>"+c[0]+"</th>" for c in cur.description])+"</tr>"
+    head="<tr>"+"".join(["<th>"+c[0]+"</th>" for c in cur.description])+"</tr>"
     lines=""
     if callback:
         for r in itermap(cur):
@@ -237,7 +238,7 @@ def monitor_stdout(aprocess, onoutputdata,context):
     while read_set:
         try:
             rlist, wlist, xlist = select.select(read_set, [], [])
-        except select.error, e:
+        except select.error as e:
             if e.args[0] == errno.EINTR:
                 continue
             raise
@@ -245,12 +246,14 @@ def monitor_stdout(aprocess, onoutputdata,context):
         # Reads one line from stdout
         if aprocess.stdout in rlist:
             data = os.read(aprocess.stdout.fileno(), 1)
+            data = data.decode('utf-8')
             if data == "":
                 aprocess.stdout.close()
                 read_set.remove(aprocess.stdout)
             while data and not data in ('\n','\r'):
                 line += data
                 data = os.read(aprocess.stdout.fileno(), 1)
+                data = data.decode('utf-8')
             if line or data in ('\n','\r'):
                 stdout.append(line)
                 if onoutputdata:
@@ -260,12 +263,14 @@ def monitor_stdout(aprocess, onoutputdata,context):
         # Reads one line from stderr
         if aprocess.stderr in rlist:
             data = os.read(aprocess.stderr.fileno(), 1)
+            data = data.decode('utf-8')
             if data == "":
                 aprocess.stderr.close()
                 read_set.remove(aprocess.stderr)
             while data and not data in ('\n','\r'):
                 line += data
                 data = os.read(aprocess.stderr.fileno(), 1)
+                data = data.decode('utf-8')
             if line or data in ('\n','\r'):
                 stdout.append(line)
                 if onoutputdata:
@@ -442,7 +447,7 @@ CREATE INDEX idx_stats_backup_name_start on stats(backup_name,backup_start);""")
                 return value
 
         #for r in self.query('select  * from stats where backup_name=? order by backup_end desc limit ?',(backup_name,count)):
-        print pp(cur,None,1,fcb)
+        print((pp(cur,None,1,fcb)))
 
 
     def fcb(self,fields,fieldname,value):
@@ -492,12 +497,13 @@ def ssh_exec(command,ssh=None,server_name='',remote_user='',private_key='',ssh_p
 
     chan.exec_command(command)
     stdout.flush()
-    output = stdout.read()
+    output_base = stdout.read()
+    output = output_base.decode(encoding='UTF-8').replace("'","")
     exit_code = chan.recv_exit_status()
     return (exit_code,output)
 
 
-class backup_generic:
+class backup_generic(ABC):
     """Generic ancestor class for backups, not registered"""
     type = 'generic'
     required_params = ['type','backup_name','backup_dir','server_name','backup_retention_time','maximum_backup_age']
@@ -696,7 +702,7 @@ class backup_generic:
             self.logger.info('[%s] ######### Backup finished : %s',self.backup_name,stats['log'])
             return stats
 
-        except BaseException, e:
+        except BaseException as e:
             stats['status']='ERROR'
             stats['log']=str(e)
             endtime = time.time()
@@ -798,7 +804,7 @@ class backup_generic:
                         if not self.dry_run:
                             self.dbstat.db.execute('update stats set TYPE="CLEAN" where backup_name=? and backup_location=?',(self.backup_name,oldbackup_location))
                             self.dbstat.db.commit()
-                    except BaseException,e:
+                    except BaseException as e:
                         self.logger.error('cleanup_backup : Unable to remove directory/file "%s". Error %s', oldbackup_location,e)
                     removed.append((self.backup_name,oldbackup_location))
             else:
@@ -809,10 +815,12 @@ class backup_generic:
         self.logger.info('[%s] Cleanup finished : removed : %s' , self.backup_name,','.join([('[%s]-"%s"') % r for r in removed]) or 'Nothing')
         return removed
 
+    @abstractmethod
     def register_existingbackups(self):
-        """scan existing backups and insert stats in database"""
-        registered = [b['backup_location'] for b in self.dbstat.query('select distinct backup_location from stats where backup_name=?',[self.backup_name])]
-        raise Exception('Abstract method')
+        pass
+        # """scan existing backups and insert stats in database"""
+        # registered = [b['backup_location'] for b in self.dbstat.query('select distinct backup_location from stats where backup_name=?',[self.backup_name])]
+        # raise Exception('Abstract method')
 
     def export_latestbackup(self,destdir):
         """Copy (rsync) latest OK backup to external storage located at locally mounted "destdir"
@@ -846,9 +854,9 @@ class backup_generic:
                     raise Exception('Backup source %s doesn\'t exists' % backup_source)
 
                 # ensure there is a slash at end
-                if os.path.isdir(backup_source) and backup_source[-1] <> '/':
+                if os.path.isdir(backup_source) and backup_source[-1] != '/':
                     backup_source += '/'
-                    if backup_dest[-1] <> '/':
+                    if backup_dest[-1] != '/':
                         backup_dest += '/'
 
                 if not os.path.isdir(backup_dest):
@@ -874,7 +882,7 @@ class backup_generic:
                     process = subprocess.Popen(cmd, shell=True,  stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
                     def ondata(data,context):
                         if context.verbose:
-                            print data
+                            print(data)
                         context.logger.debug(data)
 
                     log = monitor_stdout(process,ondata,self)
@@ -898,7 +906,7 @@ class backup_generic:
                         self.logger.error("[" + self.backup_name + "] shell program exited with error code ")
                         raise Exception("[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd)
                 else:
-                    print cmd
+                    print(cmd)
 
                 stats['status']='OK'
                 self.logger.info('export backup from %s to %s OK, %d bytes written for %d changed files' % (backup_source,backup_dest,stats['written_bytes'],stats['written_files_count']))

libtisbackup/copy_vm_xcp.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 # -----------------------------------------------------------------------
 #    This file is part of TISBackup
@@ -20,8 +20,8 @@
 
 import os
 import datetime
-from common import *
-import XenAPI
+from .common import *
+from . import XenAPI
 import time
 import logging
 import re
@@ -29,7 +29,7 @@ import os.path
 import os
 import datetime
 import select
-import urllib2
+import urllib.request, urllib.error, urllib.parse
 import base64
 import socket
 from stat import *
@@ -66,7 +66,7 @@ class copy_vm_xcp(backup_generic):
             session = XenAPI.Session('https://'+self.server_name)
             try:
                 session.login_with_password(user_xen,password_xen)
-            except XenAPI.Failure, error:
+            except XenAPI.Failure as error:
                 msg,ip = error.details
     
                 if msg == 'HOST_IS_SLAVE':
@@ -81,7 +81,7 @@ class copy_vm_xcp(backup_generic):
             #get storage opaqueRef
             try:
                 storage = session.xenapi.SR.get_by_name_label(storage_name)[0]
-            except IndexError,error:
+            except IndexError as error:
                 result = (1,"error get SR opaqueref %s"%(error))
                 return result
             
@@ -89,14 +89,14 @@ class copy_vm_xcp(backup_generic):
             #get vm to copy opaqueRef    
             try:
                 vm = session.xenapi.VM.get_by_name_label(vm_name)[0]
-            except IndexError,error:
+            except IndexError as error:
                 result = (1,"error get VM opaqueref %s"%(error))
                 return result
 
             # get vm backup network opaqueRef
             try:
                 networkRef = session.xenapi.network.get_by_name_label(self.network_name)[0]
-            except IndexError, error:
+            except IndexError as error:
                 result = (1, "error get VM network opaqueref %s" % (error))
                 return result
             
@@ -104,9 +104,9 @@ class copy_vm_xcp(backup_generic):
                 status_vm = session.xenapi.VM.get_power_state(vm)                
                 self.logger.debug("[%s] Status of VM: %s",self.backup_name,status_vm)
                 if status_vm == "Running":
-                    self.logger.debug("[%s] Shudown in progress",self.backup_name)
+                    self.logger.debug("[%s] Shutdown in progress",self.backup_name)
                     if dry_run:
-                        print "session.xenapi.VM.clean_shutdown(vm)" 
+                        print("session.xenapi.VM.clean_shutdown(vm)") 
                     else:
                         session.xenapi.VM.clean_shutdown(vm)     
                 snapshot = vm
@@ -115,7 +115,7 @@ class copy_vm_xcp(backup_generic):
                 self.logger.debug("[%s] Snapshot in progress",self.backup_name)
                 try:
                     snapshot = session.xenapi.VM.snapshot(vm,"tisbackup-%s"%(vm_name))
-                except XenAPI.Failure, error:
+                except XenAPI.Failure as error:
                     result = (1,"error when snapshot %s"%(error))
                     return result
             
@@ -165,7 +165,7 @@ class copy_vm_xcp(backup_generic):
                                         session.xenapi.VDI.destroy(vdi)
                                 
                             session.xenapi.VM.destroy(oldest_backup_vm)                
-                        except XenAPI.Failure, error:
+                        except XenAPI.Failure as error:
                             result = (1,"error when destroy old backup vm %s"%(error))
                             return result
                     
@@ -173,7 +173,7 @@ class copy_vm_xcp(backup_generic):
             self.logger.debug("[%s] Copy %s in progress on %s",self.backup_name,vm_name,storage_name)
             try:
                 backup_vm = session.xenapi.VM.copy(snapshot,vm_backup_name+now.strftime("%Y-%m-%d %H:%M"),storage)
-            except XenAPI.Failure, error:
+            except XenAPI.Failure as error:
                 result = (1,"error when copy %s"%(error))
                 return result
             
@@ -184,7 +184,7 @@ class copy_vm_xcp(backup_generic):
             #change the network of the new VM
             try:
                 vifDestroy = session.xenapi.VM.get_VIFs(backup_vm)
-            except IndexError,error:
+            except IndexError as error:
                 result = (1,"error get VIF opaqueref %s"%(error))
                 return result
             
@@ -213,7 +213,7 @@ class copy_vm_xcp(backup_generic):
                         }
                 try:
                     session.xenapi.VIF.create(data)
-                except Exception, error:
+                except Exception as error:
                     result = (1,error)
                     return result
             
@@ -237,7 +237,7 @@ class copy_vm_xcp(backup_generic):
                 return result
             
             #Disable automatic boot
-            if session.xenapi.VM.get_other_config(backup_vm).has_key('auto_poweron'):
+            if 'auto_poweron' in session.xenapi.VM.get_other_config(backup_vm):
                 session.xenapi.VM.remove_from_other_config(backup_vm, "auto_poweron")
 
             if not str2bool(self.halt_vm):
@@ -251,14 +251,14 @@ class copy_vm_xcp(backup_generic):
                             if not 'NULL' in  vdi:
                                 session.xenapi.VDI.destroy(vdi)
                     session.xenapi.VM.destroy(snapshot)
-                except XenAPI.Failure, error:
+                except XenAPI.Failure as error:
                     result = (1,"error when destroy snapshot %s"%(error))
                     return result
             else:
                 if status_vm == "Running":
                     self.logger.debug("[%s] Starting in progress",self.backup_name)
                     if dry_run:
-                        print "session.xenapi.VM.start(vm,False,True)" 
+                        print("session.xenapi.VM.start(vm,False,True)") 
                     else:
                         session.xenapi.VM.start(vm,False,True)
                 
@@ -282,9 +282,14 @@ class copy_vm_xcp(backup_generic):
                 stats['status']='ERROR'
                 stats['log']=cmd[1]
 
-        except BaseException,e:
+        except BaseException as e:
             stats['status']='ERROR'
             stats['log']=str(e)
             raise
 
+    def register_existingbackups(self):
+        """scan backup dir and insert stats in database"""
+        #This backup is on target server, no data available on this server
+        pass
+
 register_driver(copy_vm_xcp)

libtisbackup/iniparse/__init__.py

@@ -0,0 +1,25 @@
+# Copyright (c) 2001, 2002, 2003 Python Software Foundation
+# Copyright (c) 2004-2008 Paramjit Oberoi <param.cs.wisc.edu>
+# Copyright (c) 2007 Tim Lauridsen <tla@rasmil.dk>
+# All Rights Reserved.  See LICENSE-PSF & LICENSE for details.
+
+from .ini import INIConfig, change_comment_syntax
+from .config import BasicConfig, ConfigNamespace
+from .compat import RawConfigParser, ConfigParser, SafeConfigParser
+from .utils import tidy
+
+from .configparser import DuplicateSectionError,    \
+                   NoSectionError, NoOptionError,   \
+                   InterpolationMissingOptionError, \
+                   InterpolationDepthError,         \
+                   InterpolationSyntaxError,        \
+                   DEFAULTSECT, MAX_INTERPOLATION_DEPTH
+
+__all__ = [
+    'BasicConfig', 'ConfigNamespace',
+    'INIConfig', 'tidy', 'change_comment_syntax',
+    'RawConfigParser', 'ConfigParser', 'SafeConfigParser',
+    'DuplicateSectionError', 'NoSectionError', 'NoOptionError',
+    'InterpolationMissingOptionError', 'InterpolationDepthError',
+    'InterpolationSyntaxError', 'DEFAULTSECT', 'MAX_INTERPOLATION_DEPTH',
+]

libtisbackup/iniparse/compat.py

@@ -0,0 +1,343 @@
+# Copyright (c) 2001, 2002, 2003 Python Software Foundation
+# Copyright (c) 2004-2008 Paramjit Oberoi <param.cs.wisc.edu>
+# All Rights Reserved.  See LICENSE-PSF & LICENSE for details.
+
+"""Compatibility interfaces for ConfigParser
+
+Interfaces of ConfigParser, RawConfigParser and SafeConfigParser
+should be completely identical to the Python standard library
+versions.  Tested with the unit tests included with Python-2.3.4
+
+The underlying INIConfig object can be accessed as cfg.data
+"""
+
+import re
+from .configparser import DuplicateSectionError,    \
+                   NoSectionError, NoOptionError,   \
+                   InterpolationMissingOptionError, \
+                   InterpolationDepthError,         \
+                   InterpolationSyntaxError,        \
+                   DEFAULTSECT, MAX_INTERPOLATION_DEPTH
+
+# These are imported only for compatiability.
+# The code below does not reference them directly.
+from .configparser import Error, InterpolationError, \
+                   MissingSectionHeaderError, ParsingError
+
+import six
+
+from . import ini
+
+
+class RawConfigParser(object):
+    def __init__(self, defaults=None, dict_type=dict):
+        if dict_type != dict:
+            raise ValueError('Custom dict types not supported')
+        self.data = ini.INIConfig(defaults=defaults, optionxformsource=self)
+
+    def optionxform(self, optionstr):
+        return optionstr.lower()
+
+    def defaults(self):
+        d = {}
+        secobj = self.data._defaults
+        for name in secobj._options:
+            d[name] = secobj._compat_get(name)
+        return d
+
+    def sections(self):
+        """Return a list of section names, excluding [DEFAULT]"""
+        return list(self.data)
+
+    def add_section(self, section):
+        """Create a new section in the configuration.
+
+        Raise DuplicateSectionError if a section by the specified name
+        already exists.  Raise ValueError if name is DEFAULT or any of
+        its case-insensitive variants.
+        """
+        # The default section is the only one that gets the case-insensitive
+        # treatment - so it is special-cased here.
+        if section.lower() == "default":
+            raise ValueError('Invalid section name: %s' % section)
+
+        if self.has_section(section):
+            raise DuplicateSectionError(section)
+        else:
+            self.data._new_namespace(section)
+
+    def has_section(self, section):
+        """Indicate whether the named section is present in the configuration.
+
+        The DEFAULT section is not acknowledged.
+        """
+        return section in self.data
+
+    def options(self, section):
+        """Return a list of option names for the given section name."""
+        if section in self.data:
+            return list(self.data[section])
+        else:
+            raise NoSectionError(section)
+
+    def read(self, filenames):
+        """Read and parse a filename or a list of filenames.
+
+        Files that cannot be opened are silently ignored; this is
+        designed so that you can specify a list of potential
+        configuration file locations (e.g. current directory, user's
+        home directory, systemwide directory), and all existing
+        configuration files in the list will be read.  A single
+        filename may also be given.
+        """
+        files_read = []
+        if isinstance(filenames, six.string_types):
+            filenames = [filenames]
+        for filename in filenames:
+            try:
+                fp = open(filename)
+            except IOError:
+                continue
+            files_read.append(filename)
+            self.data._readfp(fp)
+            fp.close()
+        return files_read
+
+    def readfp(self, fp, filename=None):
+        """Like read() but the argument must be a file-like object.
+
+        The `fp' argument must have a `readline' method.  Optional
+        second argument is the `filename', which if not given, is
+        taken from fp.name.  If fp has no `name' attribute, `<???>' is
+        used.
+        """
+        self.data._readfp(fp)
+
+    def get(self, section, option, vars=None):
+        if not self.has_section(section):
+            raise NoSectionError(section)
+
+        sec = self.data[section]
+        if option in sec:
+            return sec._compat_get(option)
+        else:
+            raise NoOptionError(option, section)
+
+    def items(self, section):
+        if section in self.data:
+            ans = []
+            for opt in self.data[section]:
+                ans.append((opt, self.get(section, opt)))
+            return ans
+        else:
+            raise NoSectionError(section)
+
+    def getint(self, section, option):
+        return int(self.get(section, option))
+
+    def getfloat(self, section, option):
+        return float(self.get(section, option))
+
+    _boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
+                       '0': False, 'no': False, 'false': False, 'off': False}
+
+    def getboolean(self, section, option):
+        v = self.get(section, option)
+        if v.lower() not in self._boolean_states:
+            raise ValueError('Not a boolean: %s' % v)
+        return self._boolean_states[v.lower()]
+
+    def has_option(self, section, option):
+        """Check for the existence of a given option in a given section."""
+        if section in self.data:
+            sec = self.data[section]
+        else:
+            raise NoSectionError(section)
+        return (option in sec)
+
+    def set(self, section, option, value):
+        """Set an option."""
+        if section in self.data:
+            self.data[section][option] = value
+        else:
+            raise NoSectionError(section)
+
+    def write(self, fp):
+        """Write an .ini-format representation of the configuration state."""
+        fp.write(str(self.data))
+
+    def remove_option(self, section, option):
+        """Remove an option."""
+        if section in self.data:
+            sec = self.data[section]
+        else:
+            raise NoSectionError(section)
+        if option in sec:
+            del sec[option]
+            return 1
+        else:
+            return 0
+
+    def remove_section(self, section):
+        """Remove a file section."""
+        if not self.has_section(section):
+            return False
+        del self.data[section]
+        return True
+
+
+class ConfigDict(object):
+    """Present a dict interface to a ini section."""
+
+    def __init__(self, cfg, section, vars):
+        self.cfg = cfg
+        self.section = section
+        self.vars = vars
+
+    def __getitem__(self, key):
+        try:
+            return RawConfigParser.get(self.cfg, self.section, key, self.vars)
+        except (NoOptionError, NoSectionError):
+            raise KeyError(key)
+
+
+class ConfigParser(RawConfigParser):
+
+    def get(self, section, option, raw=False, vars=None):
+        """Get an option value for a given section.
+
+        All % interpolations are expanded in the return values, based on the
+        defaults passed into the constructor, unless the optional argument
+        `raw' is true.  Additional substitutions may be provided using the
+        `vars' argument, which must be a dictionary whose contents overrides
+        any pre-existing defaults.
+
+        The section DEFAULT is special.
+        """
+        if section != DEFAULTSECT and not self.has_section(section):
+            raise NoSectionError(section)
+
+        option = self.optionxform(option)
+        value = RawConfigParser.get(self, section, option, vars)
+
+        if raw:
+            return value
+        else:
+            d = ConfigDict(self, section, vars)
+            return self._interpolate(section, option, value, d)
+
+    def _interpolate(self, section, option, rawval, vars):
+        # do the string interpolation
+        value = rawval
+        depth = MAX_INTERPOLATION_DEPTH
+        while depth:                    # Loop through this until it's done
+            depth -= 1
+            if "%(" in value:
+                try:
+                    value = value % vars
+                except KeyError as e:
+                    raise InterpolationMissingOptionError(
+                        option, section, rawval, e.args[0])
+            else:
+                break
+        if value.find("%(") != -1:
+            raise InterpolationDepthError(option, section, rawval)
+        return value
+
+    def items(self, section, raw=False, vars=None):
+        """Return a list of tuples with (name, value) for each option
+        in the section.
+
+        All % interpolations are expanded in the return values, based on the
+        defaults passed into the constructor, unless the optional argument
+        `raw' is true.  Additional substitutions may be provided using the
+        `vars' argument, which must be a dictionary whose contents overrides
+        any pre-existing defaults.
+
+        The section DEFAULT is special.
+        """
+        if section != DEFAULTSECT and not self.has_section(section):
+            raise NoSectionError(section)
+        if vars is None:
+            options = list(self.data[section])
+        else:
+            options = []
+            for x in self.data[section]:
+                if x not in vars:
+                    options.append(x)
+            options.extend(vars.keys())
+
+        if "__name__" in options:
+            options.remove("__name__")
+
+        d = ConfigDict(self, section, vars)
+        if raw:
+            return [(option, d[option])
+                    for option in options]
+        else:
+            return [(option, self._interpolate(section, option, d[option], d))
+                    for option in options]
+
+
+class SafeConfigParser(ConfigParser):
+    _interpvar_re = re.compile(r"%\(([^)]+)\)s")
+    _badpercent_re = re.compile(r"%[^%]|%$")
+
+    def set(self, section, option, value):
+        if not isinstance(value, six.string_types):
+            raise TypeError("option values must be strings")
+        # check for bad percent signs:
+        # first, replace all "good" interpolations
+        tmp_value = self._interpvar_re.sub('', value)
+        # then, check if there's a lone percent sign left
+        m = self._badpercent_re.search(tmp_value)
+        if m:
+            raise ValueError("invalid interpolation syntax in %r at "
+                             "position %d" % (value, m.start()))
+
+        ConfigParser.set(self, section, option, value)
+
+    def _interpolate(self, section, option, rawval, vars):
+        # do the string interpolation
+        L = []
+        self._interpolate_some(option, L, rawval, section, vars, 1)
+        return ''.join(L)
+
+    _interpvar_match = re.compile(r"%\(([^)]+)\)s").match
+
+    def _interpolate_some(self, option, accum, rest, section, map, depth):
+        if depth > MAX_INTERPOLATION_DEPTH:
+            raise InterpolationDepthError(option, section, rest)
+        while rest:
+            p = rest.find("%")
+            if p < 0:
+                accum.append(rest)
+                return
+            if p > 0:
+                accum.append(rest[:p])
+                rest = rest[p:]
+            # p is no longer used
+            c = rest[1:2]
+            if c == "%":
+                accum.append("%")
+                rest = rest[2:]
+            elif c == "(":
+                m = self._interpvar_match(rest)
+                if m is None:
+                    raise InterpolationSyntaxError(option, section, "bad interpolation variable reference %r" % rest)
+                var = m.group(1)
+                rest = rest[m.end():]
+                try:
+                    v = map[var]
+                except KeyError:
+                    raise InterpolationMissingOptionError(
+                        option, section, rest, var)
+                if "%" in v:
+                    self._interpolate_some(option, accum, v,
+                                           section, map, depth + 1)
+                else:
+                    accum.append(v)
+            else:
+                raise InterpolationSyntaxError(
+                    option, section,
+                    "'%' must be followed by '%' or '(', found: " + repr(rest))

libtisbackup/iniparse/config.py

@@ -0,0 +1,292 @@
+class ConfigNamespace(object):
+    """Abstract class representing the interface of Config objects.
+
+    A ConfigNamespace is a collection of names mapped to values, where
+    the values may be nested namespaces.  Values can be accessed via
+    container notation - obj[key] - or via dotted notation - obj.key.
+    Both these access methods are equivalent.
+
+    To minimize name conflicts between namespace keys and class members,
+    the number of class members should be minimized, and the names of
+    all class members should start with an underscore.
+
+    Subclasses must implement the methods for container-like access,
+    and this class will automatically provide dotted access.
+
+    """
+
+    # Methods that must be implemented by subclasses
+
+    def _getitem(self, key):
+        return NotImplementedError(key)
+
+    def __setitem__(self, key, value):
+        raise NotImplementedError(key, value)
+
+    def __delitem__(self, key):
+        raise NotImplementedError(key)
+
+    def __iter__(self):
+        return NotImplementedError()
+
+    def _new_namespace(self, name):
+        raise NotImplementedError(name)
+
+    def __contains__(self, key):
+        try:
+            self._getitem(key)
+        except KeyError:
+            return False
+        return True
+
+    # Machinery for converting dotted access into container access,
+    # and automatically creating new sections/namespaces.
+    #
+    # To distinguish between accesses of class members and namespace
+    # keys, we first call object.__getattribute__().  If that succeeds,
+    # the name is assumed to be a class member.  Otherwise it is
+    # treated as a namespace key.
+    #
+    # Therefore, member variables should be defined in the class,
+    # not just in the __init__() function.  See BasicNamespace for
+    # an example.
+
+    def __getitem__(self, key):
+        try:
+            return self._getitem(key)
+        except KeyError:
+            return Undefined(key, self)
+
+    def __getattr__(self, name):
+        try:
+            return self._getitem(name)
+        except KeyError:
+            if name.startswith('__') and name.endswith('__'):
+                raise AttributeError
+            return Undefined(name, self)
+
+    def __setattr__(self, name, value):
+        try:
+            object.__getattribute__(self, name)
+            object.__setattr__(self, name, value)
+        except AttributeError:
+            self.__setitem__(name, value)
+
+    def __delattr__(self, name):
+        try:
+            object.__getattribute__(self, name)
+            object.__delattr__(self, name)
+        except AttributeError:
+            self.__delitem__(name)
+
+    # During unpickling, Python checks if the class has a __setstate__
+    # method.  But, the data dicts have not been initialised yet, which
+    # leads to  _getitem and hence __getattr__ raising an exception.  So
+    # we explicitly impement default __setstate__ behavior.
+    def __setstate__(self, state):
+        self.__dict__.update(state)
+
+
+class Undefined(object):
+    """Helper class used to hold undefined names until assignment.
+
+    This class helps create any undefined subsections when an
+    assignment is made to a nested value.  For example, if the
+    statement is "cfg.a.b.c = 42", but "cfg.a.b" does not exist yet.
+    """
+
+    def __init__(self, name, namespace):
+        object.__setattr__(self, 'name', name)
+        object.__setattr__(self, 'namespace', namespace)
+
+    def __setattr__(self, name, value):
+        obj = self.namespace._new_namespace(self.name)
+        obj[name] = value
+
+    def __setitem__(self, name, value):
+        obj = self.namespace._new_namespace(self.name)
+        obj[name] = value
+
+
+# ---- Basic implementation of a ConfigNamespace
+
+class BasicConfig(ConfigNamespace):
+    """Represents a hierarchical collection of named values.
+
+    Values are added using dotted notation:
+
+    >>> n = BasicConfig()
+    >>> n.x = 7
+    >>> n.name.first = 'paramjit'
+    >>> n.name.last = 'oberoi'
+
+    ...and accessed the same way, or with [...]:
+
+    >>> n.x
+    7
+    >>> n.name.first
+    'paramjit'
+    >>> n.name.last
+    'oberoi'
+    >>> n['x']
+    7
+    >>> n['name']['first']
+    'paramjit'
+
+    Iterating over the namespace object returns the keys:
+
+    >>> l = list(n)
+    >>> l.sort()
+    >>> l
+    ['name', 'x']
+
+    Values can be deleted using 'del' and printed using 'print'.
+
+    >>> n.aaa = 42
+    >>> del n.x
+    >>> print(n)
+    aaa = 42
+    name.first = paramjit
+    name.last = oberoi
+
+    Nested namespaces are also namespaces:
+
+    >>> isinstance(n.name, ConfigNamespace)
+    True
+    >>> print(n.name)
+    first = paramjit
+    last = oberoi
+    >>> sorted(list(n.name))
+    ['first', 'last']
+
+    Finally, values can be read from a file as follows:
+
+    >>> from six import StringIO
+    >>> sio = StringIO('''
+    ... # comment
+    ... ui.height = 100
+    ... ui.width = 150
+    ... complexity = medium
+    ... have_python
+    ... data.secret.password = goodness=gracious me
+    ... ''')
+    >>> n = BasicConfig()
+    >>> n._readfp(sio)
+    >>> print(n)
+    complexity = medium
+    data.secret.password = goodness=gracious me
+    have_python
+    ui.height = 100
+    ui.width = 150
+    """
+
+    # this makes sure that __setattr__ knows this is not a namespace key
+    _data = None
+
+    def __init__(self):
+        self._data = {}
+
+    def _getitem(self, key):
+        return self._data[key]
+
+    def __setitem__(self, key, value):
+        self._data[key] = value
+
+    def __delitem__(self, key):
+        del self._data[key]
+
+    def __iter__(self):
+        return iter(self._data)
+
+    def __str__(self, prefix=''):
+        lines = []
+        keys = list(self._data.keys())
+        keys.sort()
+        for name in keys:
+            value = self._data[name]
+            if isinstance(value, ConfigNamespace):
+                lines.append(value.__str__(prefix='%s%s.' % (prefix,name)))
+            else:
+                if value is None:
+                    lines.append('%s%s' % (prefix, name))
+                else:
+                    lines.append('%s%s = %s' % (prefix, name, value))
+        return '\n'.join(lines)
+
+    def _new_namespace(self, name):
+        obj = BasicConfig()
+        self._data[name] = obj
+        return obj
+
+    def _readfp(self, fp):
+        while True:
+            line = fp.readline()
+            if not line:
+                break
+
+            line = line.strip()
+            if not line: continue
+            if line[0] == '#': continue
+            data = line.split('=', 1)
+            if len(data) == 1:
+                name = line
+                value = None
+            else:
+                name = data[0].strip()
+                value = data[1].strip()
+            name_components = name.split('.')
+            ns = self
+            for n in name_components[:-1]:
+                if n in ns:
+                    ns = ns[n]
+                    if not isinstance(ns, ConfigNamespace):
+                        raise TypeError('value-namespace conflict', n)
+                else:
+                    ns = ns._new_namespace(n)
+            ns[name_components[-1]] = value
+
+
+# ---- Utility functions
+
+def update_config(target, source):
+    """Imports values from source into target.
+
+    Recursively walks the <source> ConfigNamespace and inserts values
+    into the <target> ConfigNamespace.  For example:
+
+    >>> n = BasicConfig()
+    >>> n.playlist.expand_playlist = True
+    >>> n.ui.display_clock = True
+    >>> n.ui.display_qlength = True
+    >>> n.ui.width = 150
+    >>> print(n)
+    playlist.expand_playlist = True
+    ui.display_clock = True
+    ui.display_qlength = True
+    ui.width = 150
+
+    >>> from iniparse import ini
+    >>> i = ini.INIConfig()
+    >>> update_config(i, n)
+    >>> print(i)
+    [playlist]
+    expand_playlist = True
+    <BLANKLINE>
+    [ui]
+    display_clock = True
+    display_qlength = True
+    width = 150
+
+    """
+    for name in sorted(source):
+        value = source[name]
+        if isinstance(value, ConfigNamespace):
+            if name in target:
+                myns = target[name]
+                if not isinstance(myns, ConfigNamespace):
+                    raise TypeError('value-namespace conflict')
+            else:
+                myns = target._new_namespace(name)
+            update_config(myns, value)
+        else:
+            target[name] = value

libtisbackup/iniparse/configparser.py

@@ -0,0 +1,7 @@
+try:
+    from ConfigParser import *
+    # not all objects get imported with __all__
+    from ConfigParser import Error, InterpolationMissingOptionError
+except ImportError:
+    from configparser import *
+    from configparser import Error, InterpolationMissingOptionError

libtisbackup/iniparse/ini.py

@@ -0,0 +1,652 @@
+"""Access and/or modify INI files
+
+* Compatiable with ConfigParser
+* Preserves order of sections & options
+* Preserves comments/blank lines/etc
+* More conveninet access to data
+
+Example:
+
+    >>> from six import StringIO
+    >>> sio = StringIO('''# configure foo-application
+    ... [foo]
+    ... bar1 = qualia
+    ... bar2 = 1977
+    ... [foo-ext]
+    ... special = 1''')
+
+    >>> cfg = INIConfig(sio)
+    >>> print(cfg.foo.bar1)
+    qualia
+    >>> print(cfg['foo-ext'].special)
+    1
+    >>> cfg.foo.newopt = 'hi!'
+    >>> cfg.baz.enabled = 0
+
+    >>> print(cfg)
+    # configure foo-application
+    [foo]
+    bar1 = qualia
+    bar2 = 1977
+    newopt = hi!
+    [foo-ext]
+    special = 1
+    <BLANKLINE>
+    [baz]
+    enabled = 0
+
+"""
+
+# An ini parser that supports ordered sections/options
+# Also supports updates, while preserving structure
+# Backward-compatiable with ConfigParser
+
+import re
+from .configparser import DEFAULTSECT, ParsingError, MissingSectionHeaderError
+
+import six
+
+from . import config
+
+
+class LineType(object):
+    line = None
+
+    def __init__(self, line=None):
+        if line is not None:
+            self.line = line.strip('\n')
+
+    # Return the original line for unmodified objects
+    # Otherwise construct using the current attribute values
+    def __str__(self):
+        if self.line is not None:
+            return self.line
+        else:
+            return self.to_string()
+
+    # If an attribute is modified after initialization
+    # set line to None since it is no longer accurate.
+    def __setattr__(self, name, value):
+        if hasattr(self,name):
+            self.__dict__['line'] = None
+        self.__dict__[name] = value
+
+    def to_string(self):
+        raise Exception('This method must be overridden in derived classes')
+
+
+class SectionLine(LineType):
+    regex = re.compile(r'^\['
+                       r'(?P<name>[^]]+)'
+                       r'\]\s*'
+                       r'((?P<csep>;|#)(?P<comment>.*))?$')
+
+    def __init__(self, name, comment=None, comment_separator=None,
+                             comment_offset=-1, line=None):
+        super(SectionLine, self).__init__(line)
+        self.name = name
+        self.comment = comment
+        self.comment_separator = comment_separator
+        self.comment_offset = comment_offset
+
+    def to_string(self):
+        out = '[' + self.name + ']'
+        if self.comment is not None:
+            # try to preserve indentation of comments
+            out = (out+' ').ljust(self.comment_offset)
+            out = out + self.comment_separator + self.comment
+        return out
+
+    def parse(cls, line):
+        m = cls.regex.match(line.rstrip())
+        if m is None:
+            return None
+        return cls(m.group('name'), m.group('comment'),
+                   m.group('csep'), m.start('csep'),
+                   line)
+    parse = classmethod(parse)
+
+
+class OptionLine(LineType):
+    def __init__(self, name, value, separator=' = ', comment=None,
+                 comment_separator=None, comment_offset=-1, line=None):
+        super(OptionLine, self).__init__(line)
+        self.name = name
+        self.value = value
+        self.separator = separator
+        self.comment = comment
+        self.comment_separator = comment_separator
+        self.comment_offset = comment_offset
+
+    def to_string(self):
+        out = '%s%s%s' % (self.name, self.separator, self.value)
+        if self.comment is not None:
+            # try to preserve indentation of comments
+            out = (out+' ').ljust(self.comment_offset)
+            out = out + self.comment_separator + self.comment
+        return out
+
+    regex = re.compile(r'^(?P<name>[^:=\s[][^:=]*)'
+                       r'(?P<sep>[:=]\s*)'
+                       r'(?P<value>.*)$')
+
+    def parse(cls, line):
+        m = cls.regex.match(line.rstrip())
+        if m is None:
+            return None
+
+        name = m.group('name').rstrip()
+        value = m.group('value')
+        sep = m.group('name')[len(name):] + m.group('sep')
+
+        # comments are not detected in the regex because
+        # ensuring total compatibility with ConfigParser
+        # requires that:
+        #     option = value    ;comment   // value=='value'
+        #     option = value;1  ;comment   // value=='value;1  ;comment'
+        #
+        # Doing this in a regex would be complicated.  I
+        # think this is a bug.  The whole issue of how to
+        # include ';' in the value needs to be addressed.
+        # Also, '#' doesn't mark comments in options...
+
+        coff = value.find(';')
+        if coff != -1 and value[coff-1].isspace():
+            comment = value[coff+1:]
+            csep = value[coff]
+            value = value[:coff].rstrip()
+            coff = m.start('value') + coff
+        else:
+            comment = None
+            csep = None
+            coff = -1
+
+        return cls(name, value, sep, comment, csep, coff, line)
+    parse = classmethod(parse)
+
+
+def change_comment_syntax(comment_chars='%;#', allow_rem=False):
+    comment_chars = re.sub(r'([\]\-\^])', r'\\\1', comment_chars)
+    regex = r'^(?P<csep>[%s]' % comment_chars
+    if allow_rem:
+        regex += '|[rR][eE][mM]'
+    regex += r')(?P<comment>.*)$'
+    CommentLine.regex = re.compile(regex)
+
+
+class CommentLine(LineType):
+    regex = re.compile(r'^(?P<csep>[;#])'
+                       r'(?P<comment>.*)$')
+
+    def __init__(self, comment='', separator='#', line=None):
+        super(CommentLine, self).__init__(line)
+        self.comment = comment
+        self.separator = separator
+
+    def to_string(self):
+        return self.separator + self.comment
+
+    def parse(cls, line):
+        m = cls.regex.match(line.rstrip())
+        if m is None:
+            return None
+        return cls(m.group('comment'), m.group('csep'), line)
+
+    parse = classmethod(parse)
+
+
+class EmptyLine(LineType):
+    # could make this a singleton
+    def to_string(self):
+        return ''
+
+    value = property(lambda self: '')
+
+    def parse(cls, line):
+        if line.strip():
+            return None
+        return cls(line)
+
+    parse = classmethod(parse)
+
+
+class ContinuationLine(LineType):
+    regex = re.compile(r'^\s+(?P<value>.*)$')
+
+    def __init__(self, value, value_offset=None, line=None):
+        super(ContinuationLine, self).__init__(line)
+        self.value = value
+        if value_offset is None:
+            value_offset = 8
+        self.value_offset = value_offset
+
+    def to_string(self):
+        return ' '*self.value_offset + self.value
+
+    def parse(cls, line):
+        m = cls.regex.match(line.rstrip())
+        if m is None:
+            return None
+        return cls(m.group('value'), m.start('value'), line)
+
+    parse = classmethod(parse)
+
+
+class LineContainer(object):
+    def __init__(self, d=None):
+        self.contents = []
+        self.orgvalue = None
+        if d:
+            if isinstance(d, list): self.extend(d)
+            else: self.add(d)
+
+    def add(self, x):
+        self.contents.append(x)
+
+    def extend(self, x):
+        for i in x: self.add(i)
+
+    def get_name(self):
+        return self.contents[0].name
+
+    def set_name(self, data):
+        self.contents[0].name = data
+
+    def get_value(self):
+        if self.orgvalue is not None:
+            return self.orgvalue
+        elif len(self.contents) == 1:
+            return self.contents[0].value
+        else:
+            return '\n'.join([('%s' % x.value) for x in self.contents
+                              if not isinstance(x, CommentLine)])
+
+    def set_value(self, data):
+        self.orgvalue = data
+        lines = ('%s' % data).split('\n')
+
+        # If there is an existing ContinuationLine, use its offset
+        value_offset = None
+        for v in self.contents:
+            if isinstance(v, ContinuationLine):
+                value_offset = v.value_offset
+                break
+
+        # Rebuild contents list, preserving initial OptionLine
+        self.contents = self.contents[0:1]
+        self.contents[0].value = lines[0]
+        del lines[0]
+        for line in lines:
+            if line.strip():
+                self.add(ContinuationLine(line, value_offset))
+            else:
+                self.add(EmptyLine())
+
+    name = property(get_name, set_name)
+
+    value = property(get_value, set_value)
+
+    def __str__(self):
+        s = [x.__str__() for x in self.contents]
+        return '\n'.join(s)
+
+    def finditer(self, key):
+        for x in self.contents[::-1]:
+            if hasattr(x, 'name') and x.name==key:
+                yield x
+
+    def find(self, key):
+        for x in self.finditer(key):
+            return x
+        raise KeyError(key)
+
+
+def _make_xform_property(myattrname, srcattrname=None):
+    private_attrname = myattrname + 'value'
+    private_srcname = myattrname + 'source'
+    if srcattrname is None:
+        srcattrname = myattrname
+
+    def getfn(self):
+        srcobj = getattr(self, private_srcname)
+        if srcobj is not None:
+            return getattr(srcobj, srcattrname)
+        else:
+            return getattr(self, private_attrname)
+
+    def setfn(self, value):
+        srcobj = getattr(self, private_srcname)
+        if srcobj is not None:
+            setattr(srcobj, srcattrname, value)
+        else:
+            setattr(self, private_attrname, value)
+
+    return property(getfn, setfn)
+
+
+class INISection(config.ConfigNamespace):
+    _lines = None
+    _options = None
+    _defaults = None
+    _optionxformvalue = None
+    _optionxformsource = None
+    _compat_skip_empty_lines = set()
+
+    def __init__(self, lineobj, defaults=None, optionxformvalue=None, optionxformsource=None):
+        self._lines = [lineobj]
+        self._defaults = defaults
+        self._optionxformvalue = optionxformvalue
+        self._optionxformsource = optionxformsource
+        self._options = {}
+
+    _optionxform = _make_xform_property('_optionxform')
+
+    def _compat_get(self, key):
+        # identical to __getitem__ except that _compat_XXX
+        # is checked for backward-compatible handling
+        if key == '__name__':
+            return self._lines[-1].name
+        if self._optionxform: key = self._optionxform(key)
+        try:
+            value = self._options[key].value
+            del_empty = key in self._compat_skip_empty_lines
+        except KeyError:
+            if self._defaults and key in self._defaults._options:
+                value = self._defaults._options[key].value
+                del_empty = key in self._defaults._compat_skip_empty_lines
+            else:
+                raise
+        if del_empty:
+            value = re.sub('\n+', '\n', value)
+        return value
+
+    def _getitem(self, key):
+        if key == '__name__':
+            return self._lines[-1].name
+        if self._optionxform: key = self._optionxform(key)
+        try:
+            return self._options[key].value
+        except KeyError:
+            if self._defaults and key in self._defaults._options:
+                return self._defaults._options[key].value
+            else:
+                raise
+
+    def __setitem__(self, key, value):
+        if self._optionxform: xkey = self._optionxform(key)
+        else: xkey = key
+        if xkey in self._compat_skip_empty_lines:
+            self._compat_skip_empty_lines.remove(xkey)
+        if xkey not in self._options:
+            # create a dummy object - value may have multiple lines
+            obj = LineContainer(OptionLine(key, ''))
+            self._lines[-1].add(obj)
+            self._options[xkey] = obj
+        # the set_value() function in LineContainer
+        # automatically handles multi-line values
+        self._options[xkey].value = value
+
+    def __delitem__(self, key):
+        if self._optionxform: key = self._optionxform(key)
+        if key in self._compat_skip_empty_lines:
+            self._compat_skip_empty_lines.remove(key)
+        for l in self._lines:
+            remaining = []
+            for o in l.contents:
+                if isinstance(o, LineContainer):
+                    n = o.name
+                    if self._optionxform: n = self._optionxform(n)
+                    if key != n: remaining.append(o)
+                else:
+                    remaining.append(o)
+            l.contents = remaining
+        del self._options[key]
+
+    def __iter__(self):
+        d = set()
+        for l in self._lines:
+            for x in l.contents:
+                if isinstance(x, LineContainer):
+                    if self._optionxform:
+                        ans = self._optionxform(x.name)
+                    else:
+                        ans = x.name
+                    if ans not in d:
+                        yield ans
+                        d.add(ans)
+        if self._defaults:
+            for x in self._defaults:
+                if x not in d:
+                    yield x
+                    d.add(x)
+
+    def _new_namespace(self, name):
+        raise Exception('No sub-sections allowed', name)
+
+
+def make_comment(line):
+    return CommentLine(line.rstrip('\n'))
+
+
+def readline_iterator(f):
+    """iterate over a file by only using the file object's readline method"""
+
+    have_newline = False
+    while True:
+        line = f.readline()
+
+        if not line:
+            if have_newline:
+                yield ""
+            return
+
+        if line.endswith('\n'):
+            have_newline = True
+        else:
+            have_newline = False
+
+        yield line
+
+
+def lower(x):
+    return x.lower()
+
+
+class INIConfig(config.ConfigNamespace):
+    _data = None
+    _sections = None
+    _defaults = None
+    _optionxformvalue = None
+    _optionxformsource = None
+    _sectionxformvalue = None
+    _sectionxformsource = None
+    _parse_exc = None
+    _bom = False
+
+    def __init__(self, fp=None, defaults=None, parse_exc=True,
+                 optionxformvalue=lower, optionxformsource=None,
+                 sectionxformvalue=None, sectionxformsource=None):
+        self._data = LineContainer()
+        self._parse_exc = parse_exc
+        self._optionxformvalue = optionxformvalue
+        self._optionxformsource = optionxformsource
+        self._sectionxformvalue = sectionxformvalue
+        self._sectionxformsource = sectionxformsource
+        self._sections = {}
+        if defaults is None: defaults = {}
+        self._defaults = INISection(LineContainer(), optionxformsource=self)
+        for name, value in defaults.items():
+            self._defaults[name] = value
+        if fp is not None:
+            self._readfp(fp)
+
+    _optionxform = _make_xform_property('_optionxform', 'optionxform')
+    _sectionxform = _make_xform_property('_sectionxform', 'optionxform')
+
+    def _getitem(self, key):
+        if key == DEFAULTSECT:
+            return self._defaults
+        if self._sectionxform: key = self._sectionxform(key)
+        return self._sections[key]
+
+    def __setitem__(self, key, value):
+        raise Exception('Values must be inside sections', key, value)
+
+    def __delitem__(self, key):
+        if self._sectionxform: key = self._sectionxform(key)
+        for line in self._sections[key]._lines:
+            self._data.contents.remove(line)
+        del self._sections[key]
+
+    def __iter__(self):
+        d = set()
+        d.add(DEFAULTSECT)
+        for x in self._data.contents:
+            if isinstance(x, LineContainer):
+                if x.name not in d:
+                    yield x.name
+                    d.add(x.name)
+
+    def _new_namespace(self, name):
+        if self._data.contents:
+            self._data.add(EmptyLine())
+        obj = LineContainer(SectionLine(name))
+        self._data.add(obj)
+        if self._sectionxform: name = self._sectionxform(name)
+        if name in self._sections:
+            ns = self._sections[name]
+            ns._lines.append(obj)
+        else:
+            ns = INISection(obj, defaults=self._defaults,
+                            optionxformsource=self)
+            self._sections[name] = ns
+        return ns
+
+    def __str__(self):
+        if self._bom:
+            fmt = u'\ufeff%s'
+        else:
+            fmt = '%s'
+        return fmt % self._data.__str__()
+
+    __unicode__ = __str__
+
+    _line_types = [EmptyLine, CommentLine,
+                   SectionLine, OptionLine,
+                   ContinuationLine]
+
+    def _parse(self, line):
+        for linetype in self._line_types:
+            lineobj = linetype.parse(line)
+            if lineobj:
+                return lineobj
+        else:
+            # can't parse line
+            return None
+
+    def _readfp(self, fp):
+        cur_section = None
+        cur_option = None
+        cur_section_name = None
+        cur_option_name = None
+        pending_lines = []
+        pending_empty_lines = False
+        try:
+            fname = fp.name
+        except AttributeError:
+            fname = '<???>'
+        line_count = 0
+        exc = None
+        line = None
+
+        for line in readline_iterator(fp):
+            # Check for BOM on first line
+            if line_count == 0 and isinstance(line, six.text_type):
+                if line[0] == u'\ufeff':
+                    line = line[1:]
+                    self._bom = True
+
+            line_obj = self._parse(line)
+            line_count += 1
+
+            if not cur_section and not isinstance(line_obj, (CommentLine, EmptyLine, SectionLine)):
+                if self._parse_exc:
+                    raise MissingSectionHeaderError(fname, line_count, line)
+                else:
+                    line_obj = make_comment(line)
+
+            if line_obj is None:
+                if self._parse_exc:
+                    if exc is None:
+                        exc = ParsingError(fname)
+                    exc.append(line_count, line)
+                line_obj = make_comment(line)
+
+            if isinstance(line_obj, ContinuationLine):
+                if cur_option:
+                    if pending_lines:
+                        cur_option.extend(pending_lines)
+                        pending_lines = []
+                        if pending_empty_lines:
+                            optobj._compat_skip_empty_lines.add(cur_option_name)
+                            pending_empty_lines = False
+                    cur_option.add(line_obj)
+                else:
+                    # illegal continuation line - convert to comment
+                    if self._parse_exc:
+                        if exc is None:
+                            exc = ParsingError(fname)
+                        exc.append(line_count, line)
+                    line_obj = make_comment(line)
+
+            if isinstance(line_obj, OptionLine):
+                if pending_lines:
+                    cur_section.extend(pending_lines)
+                    pending_lines = []
+                    pending_empty_lines = False
+                cur_option = LineContainer(line_obj)
+                cur_section.add(cur_option)
+                if self._optionxform:
+                    cur_option_name = self._optionxform(cur_option.name)
+                else:
+                    cur_option_name = cur_option.name
+                if cur_section_name == DEFAULTSECT:
+                    optobj = self._defaults
+                else:
+                    optobj = self._sections[cur_section_name]
+                optobj._options[cur_option_name] = cur_option
+
+            if isinstance(line_obj, SectionLine):
+                self._data.extend(pending_lines)
+                pending_lines = []
+                pending_empty_lines = False
+                cur_section = LineContainer(line_obj)
+                self._data.add(cur_section)
+                cur_option = None
+                cur_option_name = None
+                if cur_section.name == DEFAULTSECT:
+                    self._defaults._lines.append(cur_section)
+                    cur_section_name = DEFAULTSECT
+                else:
+                    if self._sectionxform:
+                        cur_section_name = self._sectionxform(cur_section.name)
+                    else:
+                        cur_section_name = cur_section.name
+                    if cur_section_name not in self._sections:
+                        self._sections[cur_section_name] = \
+                                INISection(cur_section, defaults=self._defaults,
+                                           optionxformsource=self)
+                    else:
+                        self._sections[cur_section_name]._lines.append(cur_section)
+
+            if isinstance(line_obj, (CommentLine, EmptyLine)):
+                pending_lines.append(line_obj)
+                if isinstance(line_obj, EmptyLine):
+                    pending_empty_lines = True
+
+        self._data.extend(pending_lines)
+        if line and line[-1] == '\n':
+            self._data.add(EmptyLine())
+
+        if exc:
+            raise exc

libtisbackup/iniparse/utils.py

@@ -0,0 +1,48 @@
+from . import compat
+from .ini import LineContainer, EmptyLine
+
+
+def tidy(cfg):
+    """Clean up blank lines.
+
+    This functions makes the configuration look clean and
+    handwritten - consecutive empty lines and empty lines at
+    the start of the file are removed, and one is guaranteed
+    to be at the end of the file.
+    """
+
+    if isinstance(cfg, compat.RawConfigParser):
+        cfg = cfg.data
+    cont = cfg._data.contents
+    i = 1
+    while i < len(cont):
+        if isinstance(cont[i], LineContainer):
+            tidy_section(cont[i])
+            i += 1
+        elif (isinstance(cont[i-1], EmptyLine) and
+              isinstance(cont[i], EmptyLine)):
+            del cont[i]
+        else:
+            i += 1
+
+    # Remove empty first line
+    if cont and isinstance(cont[0], EmptyLine):
+        del cont[0]
+
+    # Ensure a last line
+    if cont and not isinstance(cont[-1], EmptyLine):
+        cont.append(EmptyLine())
+
+
+def tidy_section(lc):
+    cont = lc.contents
+    i = 1
+    while i < len(cont):
+        if isinstance(cont[i-1], EmptyLine) and isinstance(cont[i], EmptyLine):
+            del cont[i]
+        else:
+            i += 1
+
+    # Remove empty first line
+    if len(cont) > 1 and isinstance(cont[1], EmptyLine):
+        del cont[1]