k3nny/TISbackup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP: python3

Browse Source
This commit is contained in:
htouvet 2020-07-24 12:14:48 +02:00
parent 8aa63dbdd4
commit 1a99f9bb9a
11 changed files with 185 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
iniparse/__init__.py Executable file → Normal file
View File

@ -3,12 +3,12 @@
# Copyright (c) 2007 Tim Lauridsen <tla@rasmil.dk> # Copyright (c) 2007 Tim Lauridsen <tla@rasmil.dk>
# All Rights Reserved. See LICENSE-PSF & LICENSE for details. # All Rights Reserved. See LICENSE-PSF & LICENSE for details.
from ini import INIConfig, change_comment_syntax from .ini import INIConfig, change_comment_syntax
from config import BasicConfig, ConfigNamespace from .config import BasicConfig, ConfigNamespace
from compat import RawConfigParser, ConfigParser, SafeConfigParser from .compat import RawConfigParser, ConfigParser, SafeConfigParser
from utils import tidy from .utils import tidy
from ConfigParser import DuplicateSectionError, \ from .configparser import DuplicateSectionError, \
NoSectionError, NoOptionError, \ NoSectionError, NoOptionError, \
InterpolationMissingOptionError, \ InterpolationMissingOptionError, \
InterpolationDepthError, \ InterpolationDepthError, \

26
iniparse/compat.py Executable file → Normal file
View File

@ -12,7 +12,7 @@ The underlying INIConfig object can be accessed as cfg.data
""" """
import re import re
from ConfigParser import DuplicateSectionError, \ from .configparser import DuplicateSectionError, \
NoSectionError, NoOptionError, \ NoSectionError, NoOptionError, \
InterpolationMissingOptionError, \ InterpolationMissingOptionError, \
InterpolationDepthError, \ InterpolationDepthError, \
@ -21,10 +21,13 @@ from ConfigParser import DuplicateSectionError, \
# These are imported only for compatiability. # These are imported only for compatiability.
# The code below does not reference them directly. # The code below does not reference them directly.
from ConfigParser import Error, InterpolationError, \ from .configparser import Error, InterpolationError, \
MissingSectionHeaderError, ParsingError MissingSectionHeaderError, ParsingError
import ini import six
from . import ini
class RawConfigParser(object): class RawConfigParser(object):
def __init__(self, defaults=None, dict_type=dict): def __init__(self, defaults=None, dict_type=dict):
@ -56,7 +59,7 @@ class RawConfigParser(object):
# The default section is the only one that gets the case-insensitive # The default section is the only one that gets the case-insensitive
# treatment - so it is special-cased here. # treatment - so it is special-cased here.
if section.lower() == "default": if section.lower() == "default":
raise ValueError, 'Invalid section name: %s' % section raise ValueError('Invalid section name: %s' % section)
if self.has_section(section): if self.has_section(section):
raise DuplicateSectionError(section) raise DuplicateSectionError(section)
@ -68,7 +71,7 @@ class RawConfigParser(object):
The DEFAULT section is not acknowledged. The DEFAULT section is not acknowledged.
""" """
return (section in self.data) return section in self.data
def options(self, section): def options(self, section):
"""Return a list of option names for the given section name.""" """Return a list of option names for the given section name."""
@ -88,7 +91,7 @@ class RawConfigParser(object):
filename may also be given. filename may also be given.
""" """
files_read = [] files_read = []
if isinstance(filenames, basestring): if isinstance(filenames, six.string_types):
filenames = [filenames] filenames = [filenames]
for filename in filenames: for filename in filenames:
try: try:
@ -113,8 +116,6 @@ class RawConfigParser(object):
def get(self, section, option, vars=None): def get(self, section, option, vars=None):
if not self.has_section(section): if not self.has_section(section):
raise NoSectionError(section) raise NoSectionError(section)
if vars is not None and option in vars:
value = vars[option]
sec = self.data[section] sec = self.data[section]
if option in sec: if option in sec:
@ -143,7 +144,7 @@ class RawConfigParser(object):
def getboolean(self, section, option): def getboolean(self, section, option):
v = self.get(section, option) v = self.get(section, option)
if v.lower() not in self._boolean_states: if v.lower() not in self._boolean_states:
raise ValueError, 'Not a boolean: %s' % v raise ValueError('Not a boolean: %s' % v)
return self._boolean_states[v.lower()] return self._boolean_states[v.lower()]
def has_option(self, section, option): def has_option(self, section, option):
@ -234,7 +235,7 @@ class ConfigParser(RawConfigParser):
if "%(" in value: if "%(" in value:
try: try:
value = value % vars value = value % vars
except KeyError, e: except KeyError as e:
raise InterpolationMissingOptionError( raise InterpolationMissingOptionError(
option, section, rawval, e.args[0]) option, section, rawval, e.args[0])
else: else:
@ -283,7 +284,7 @@ class SafeConfigParser(ConfigParser):
_badpercent_re = re.compile(r"%[^%]|%$") _badpercent_re = re.compile(r"%[^%]|%$")
def set(self, section, option, value): def set(self, section, option, value):
if not isinstance(value, basestring): if not isinstance(value, six.string_types):
raise TypeError("option values must be strings") raise TypeError("option values must be strings")
# check for bad percent signs: # check for bad percent signs:
# first, replace all "good" interpolations # first, replace all "good" interpolations
@ -323,8 +324,7 @@ class SafeConfigParser(ConfigParser):
elif c == "(": elif c == "(":
m = self._interpvar_match(rest) m = self._interpvar_match(rest)
if m is None: if m is None:
raise InterpolationSyntaxError(option, section, raise InterpolationSyntaxError(option, section, "bad interpolation variable reference %r" % rest)
"bad interpolation variable reference %r" % rest)
var = m.group(1) var = m.group(1)
rest = rest[m.end():] rest = rest[m.end():]
try: try:

22
iniparse/config.py Executable file → Normal file
View File

@ -86,6 +86,7 @@ class ConfigNamespace(object):
def __setstate__(self, state): def __setstate__(self, state):
self.__dict__.update(state) self.__dict__.update(state)
class Undefined(object): class Undefined(object):
"""Helper class used to hold undefined names until assignment. """Helper class used to hold undefined names until assignment.
@ -143,16 +144,16 @@ class BasicConfig(ConfigNamespace):
>>> n.aaa = 42 >>> n.aaa = 42
>>> del n.x >>> del n.x
>>> print n >>> print(n)
aaa = 42 aaa = 42
name.first = paramjit name.first = paramjit
name.last = oberoi name.last = oberoi
Nested namepsaces are also namespaces: Nested namespaces are also namespaces:
>>> isinstance(n.name, ConfigNamespace) >>> isinstance(n.name, ConfigNamespace)
True True
>>> print n.name >>> print(n.name)
first = paramjit first = paramjit
last = oberoi last = oberoi
>>> sorted(list(n.name)) >>> sorted(list(n.name))
@ -160,7 +161,7 @@ class BasicConfig(ConfigNamespace):
Finally, values can be read from a file as follows: Finally, values can be read from a file as follows:
>>> from StringIO import StringIO >>> from six import StringIO
>>> sio = StringIO(''' >>> sio = StringIO('''
... # comment ... # comment
... ui.height = 100 ... ui.height = 100
@ -171,7 +172,7 @@ class BasicConfig(ConfigNamespace):
... ''') ... ''')
>>> n = BasicConfig() >>> n = BasicConfig()
>>> n._readfp(sio) >>> n._readfp(sio)
>>> print n >>> print(n)
complexity = medium complexity = medium
data.secret.password = goodness=gracious me data.secret.password = goodness=gracious me
have_python have_python
@ -199,7 +200,7 @@ class BasicConfig(ConfigNamespace):
def __str__(self, prefix=''): def __str__(self, prefix=''):
lines = [] lines = []
keys = self._data.keys() keys = list(self._data.keys())
keys.sort() keys.sort()
for name in keys: for name in keys:
value = self._data[name] value = self._data[name]
@ -258,7 +259,7 @@ def update_config(target, source):
>>> n.ui.display_clock = True >>> n.ui.display_clock = True
>>> n.ui.display_qlength = True >>> n.ui.display_qlength = True
>>> n.ui.width = 150 >>> n.ui.width = 150
>>> print n >>> print(n)
playlist.expand_playlist = True playlist.expand_playlist = True
ui.display_clock = True ui.display_clock = True
ui.display_qlength = True ui.display_qlength = True
@ -267,7 +268,7 @@ def update_config(target, source):
>>> from iniparse import ini >>> from iniparse import ini
>>> i = ini.INIConfig() >>> i = ini.INIConfig()
>>> update_config(i, n) >>> update_config(i, n)
>>> print i >>> print(i)
[playlist] [playlist]
expand_playlist = True expand_playlist = True
<BLANKLINE> <BLANKLINE>
@ -277,7 +278,7 @@ def update_config(target, source):
width = 150 width = 150
""" """
for name in source: for name in sorted(source):
value = source[name] value = source[name]
if isinstance(value, ConfigNamespace): if isinstance(value, ConfigNamespace):
if name in target: if name in target:
@ -289,6 +290,3 @@ def update_config(target, source):
update_config(myns, value) update_config(myns, value)
else: else:
target[name] = value target[name] = value

7
iniparse/configparser.py Normal file
View File

@ -0,0 +1,7 @@
try:
from ConfigParser import *
# not all objects get imported with __all__
from ConfigParser import Error, InterpolationMissingOptionError
except ImportError:
from configparser import *
from configparser import Error, InterpolationMissingOptionError

87
iniparse/ini.py Executable file → Normal file
View File

@ -7,7 +7,7 @@
Example: Example:
>>> from StringIO import StringIO >>> from six import StringIO
>>> sio = StringIO('''# configure foo-application >>> sio = StringIO('''# configure foo-application
... [foo] ... [foo]
... bar1 = qualia ... bar1 = qualia
@ -16,14 +16,14 @@ Example:
... special = 1''') ... special = 1''')
>>> cfg = INIConfig(sio) >>> cfg = INIConfig(sio)
>>> print cfg.foo.bar1 >>> print(cfg.foo.bar1)
qualia qualia
>>> print cfg['foo-ext'].special >>> print(cfg['foo-ext'].special)
1 1
>>> cfg.foo.newopt = 'hi!' >>> cfg.foo.newopt = 'hi!'
>>> cfg.baz.enabled = 0 >>> cfg.baz.enabled = 0
>>> print cfg >>> print(cfg)
# configure foo-application # configure foo-application
[foo] [foo]
bar1 = qualia bar1 = qualia
@ -42,9 +42,12 @@ Example:
# Backward-compatiable with ConfigParser # Backward-compatiable with ConfigParser
import re import re
from ConfigParser import DEFAULTSECT, ParsingError, MissingSectionHeaderError from .configparser import DEFAULTSECT, ParsingError, MissingSectionHeaderError
import six
from . import config
import config
class LineType(object): class LineType(object):
line = None line = None
@ -170,8 +173,9 @@ def change_comment_syntax(comment_chars='%;#', allow_rem=False):
regex += r')(?P<comment>.*)$' regex += r')(?P<comment>.*)$'
CommentLine.regex = re.compile(regex) CommentLine.regex = re.compile(regex)
class CommentLine(LineType): class CommentLine(LineType):
regex = re.compile(r'^(?P<csep>[;#]|[rR][eE][mM] +)' regex = re.compile(r'^(?P<csep>[;#]|[rR][eE][mM])'
r'(?P<comment>.*)$') r'(?P<comment>.*)$')
def __init__(self, comment='', separator='#', line=None): def __init__(self, comment='', separator='#', line=None):
@ -187,6 +191,7 @@ class CommentLine(LineType):
if m is None: if m is None:
return None return None
return cls(m.group('comment'), m.group('csep'), line) return cls(m.group('comment'), m.group('csep'), line)
parse = classmethod(parse) parse = classmethod(parse)
@ -195,11 +200,13 @@ class EmptyLine(LineType):
def to_string(self): def to_string(self):
return '' return ''
value = property(lambda _: '') value = property(lambda self: '')
def parse(cls, line): def parse(cls, line):
if line.strip(): return None if line.strip():
return None
return cls(line) return cls(line)
parse = classmethod(parse) parse = classmethod(parse)
@ -221,6 +228,7 @@ class ContinuationLine(LineType):
if m is None: if m is None:
return None return None
return cls(m.group('value'), m.start('value'), line) return cls(m.group('value'), m.start('value'), line)
parse = classmethod(parse) parse = classmethod(parse)
@ -275,6 +283,7 @@ class LineContainer(object):
self.add(EmptyLine()) self.add(EmptyLine())
name = property(get_name, set_name) name = property(get_name, set_name)
value = property(get_value, set_value) value = property(get_value, set_value)
def __str__(self): def __str__(self):
@ -322,8 +331,8 @@ class INISection(config.ConfigNamespace):
_optionxformvalue = None _optionxformvalue = None
_optionxformsource = None _optionxformsource = None
_compat_skip_empty_lines = set() _compat_skip_empty_lines = set()
def __init__(self, lineobj, defaults = None,
optionxformvalue=None, optionxformsource=None): def __init__(self, lineobj, defaults=None, optionxformvalue=None, optionxformsource=None):
self._lines = [lineobj] self._lines = [lineobj]
self._defaults = defaults self._defaults = defaults
self._optionxformvalue = optionxformvalue self._optionxformvalue = optionxformvalue
@ -453,6 +462,7 @@ class INIConfig(config.ConfigNamespace):
_sectionxformsource = None _sectionxformsource = None
_parse_exc = None _parse_exc = None
_bom = False _bom = False
def __init__(self, fp=None, defaults=None, parse_exc=True, def __init__(self, fp=None, defaults=None, parse_exc=True,
optionxformvalue=lower, optionxformsource=None, optionxformvalue=lower, optionxformsource=None,
sectionxformvalue=None, sectionxformsource=None): sectionxformvalue=None, sectionxformsource=None):
@ -465,7 +475,7 @@ class INIConfig(config.ConfigNamespace):
self._sections = {} self._sections = {}
if defaults is None: defaults = {} if defaults is None: defaults = {}
self._defaults = INISection(LineContainer(), optionxformsource=self) self._defaults = INISection(LineContainer(), optionxformsource=self)
for name, value in defaults.iteritems(): for name, value in defaults.items():
self._defaults[name] = value self._defaults[name] = value
if fp is not None: if fp is not None:
self._readfp(fp) self._readfp(fp)
@ -545,34 +555,34 @@ class INIConfig(config.ConfigNamespace):
fname = fp.name fname = fp.name
except AttributeError: except AttributeError:
fname = '<???>' fname = '<???>'
linecount = 0 line_count = 0
exc = None exc = None
line = None line = None
for line in readline_iterator(fp): for line in readline_iterator(fp):
# Check for BOM on first line # Check for BOM on first line
if linecount == 0 and isinstance(line, unicode): if line_count == 0 and isinstance(line, six.text_type):
if line[0] == u'\ufeff': if line[0] == u'\ufeff':
line = line[1:] line = line[1:]
self._bom = True self._bom = True
lineobj = self._parse(line) line_obj = self._parse(line)
linecount += 1 line_count += 1
if not cur_section and not isinstance(lineobj, if not cur_section and not isinstance(line_obj, (CommentLine, EmptyLine, SectionLine)):
(CommentLine, EmptyLine, SectionLine)):
if self._parse_exc: if self._parse_exc:
raise MissingSectionHeaderError(fname, linecount, line) raise MissingSectionHeaderError(fname, line_count, line)
else: else:
lineobj = make_comment(line) line_obj = make_comment(line)
if lineobj is None: if line_obj is None:
if self._parse_exc: if self._parse_exc:
if exc is None: exc = ParsingError(fname) if exc is None:
exc.append(linecount, line) exc = ParsingError(fname)
lineobj = make_comment(line) exc.append(line_count, line)
line_obj = make_comment(line)
if isinstance(lineobj, ContinuationLine): if isinstance(line_obj, ContinuationLine):
if cur_option: if cur_option:
if pending_lines: if pending_lines:
cur_option.extend(pending_lines) cur_option.extend(pending_lines)
@ -580,20 +590,21 @@ class INIConfig(config.ConfigNamespace):
if pending_empty_lines: if pending_empty_lines:
optobj._compat_skip_empty_lines.add(cur_option_name) optobj._compat_skip_empty_lines.add(cur_option_name)
pending_empty_lines = False pending_empty_lines = False
cur_option.add(lineobj) cur_option.add(line_obj)
else: else:
# illegal continuation line - convert to comment # illegal continuation line - convert to comment
if self._parse_exc: if self._parse_exc:
if exc is None: exc = ParsingError(fname) if exc is None:
exc.append(linecount, line) exc = ParsingError(fname)
lineobj = make_comment(line) exc.append(line_count, line)
line_obj = make_comment(line)
if isinstance(lineobj, OptionLine): if isinstance(line_obj, OptionLine):
if pending_lines: if pending_lines:
cur_section.extend(pending_lines) cur_section.extend(pending_lines)
pending_lines = [] pending_lines = []
pending_empty_lines = False pending_empty_lines = False
cur_option = LineContainer(lineobj) cur_option = LineContainer(line_obj)
cur_section.add(cur_option) cur_section.add(cur_option)
if self._optionxform: if self._optionxform:
cur_option_name = self._optionxform(cur_option.name) cur_option_name = self._optionxform(cur_option.name)
@ -605,11 +616,11 @@ class INIConfig(config.ConfigNamespace):
optobj = self._sections[cur_section_name] optobj = self._sections[cur_section_name]
optobj._options[cur_option_name] = cur_option optobj._options[cur_option_name] = cur_option
if isinstance(lineobj, SectionLine): if isinstance(line_obj, SectionLine):
self._data.extend(pending_lines) self._data.extend(pending_lines)
pending_lines = [] pending_lines = []
pending_empty_lines = False pending_empty_lines = False
cur_section = LineContainer(lineobj) cur_section = LineContainer(line_obj)
self._data.add(cur_section) self._data.add(cur_section)
cur_option = None cur_option = None
cur_option_name = None cur_option_name = None
@ -628,16 +639,14 @@ class INIConfig(config.ConfigNamespace):
else: else:
self._sections[cur_section_name]._lines.append(cur_section) self._sections[cur_section_name]._lines.append(cur_section)
if isinstance(lineobj, (CommentLine, EmptyLine)): if isinstance(line_obj, (CommentLine, EmptyLine)):
pending_lines.append(lineobj) pending_lines.append(line_obj)
if isinstance(lineobj, EmptyLine): if isinstance(line_obj, EmptyLine):
pending_empty_lines = True pending_empty_lines = True
self._data.extend(pending_lines) self._data.extend(pending_lines)
if line and line[-1]=='\n': if line and line[-1] == '\n':
self._data.add(EmptyLine()) self._data.add(EmptyLine())
if exc: if exc:
raise exc raise exc

9
iniparse/utils.py Executable file → Normal file
View File

@ -1,5 +1,6 @@
import compat from . import compat
from ini import LineContainer, EmptyLine from .ini import LineContainer, EmptyLine
def tidy(cfg): def tidy(cfg):
"""Clean up blank lines. """Clean up blank lines.
@ -32,12 +33,12 @@ def tidy(cfg):
if cont and not isinstance(cont[-1], EmptyLine): if cont and not isinstance(cont[-1], EmptyLine):
cont.append(EmptyLine()) cont.append(EmptyLine())
def tidy_section(lc): def tidy_section(lc):
cont = lc.contents cont = lc.contents
i = 1 i = 1
while i < len(cont): while i < len(cont):
if (isinstance(cont[i-1], EmptyLine) and if isinstance(cont[i-1], EmptyLine) and isinstance(cont[i], EmptyLine):
isinstance(cont[i], EmptyLine)):
del cont[i] del cont[i]
else: else:
i += 1 i += 1

10
libtisbackup/backup_mysql.py
View File

@ -24,13 +24,13 @@ import sys
try: try:
sys.stderr = open('/dev/null') # Silence silly warnings from paramiko sys.stderr = open('/dev/null') # Silence silly warnings from paramiko
import paramiko import paramiko
except ImportError,e: except ImportError as e:
print "Error : can not load paramiko library %s" % e print("Error : can not load paramiko library %s" % e)
raise raise
sys.stderr = sys.__stderr__ sys.stderr = sys.__stderr__
from common import * from libtisbackup.common import *
class backup_mysql(backup_generic): class backup_mysql(backup_generic):
"""Backup a mysql database as gzipped sql file through ssh""" """Backup a mysql database as gzipped sql file through ssh"""
@ -52,7 +52,7 @@ class backup_mysql(backup_generic):
if not self.dry_run: if not self.dry_run:
os.makedirs(self.dest_dir) os.makedirs(self.dest_dir)
else: else:
print 'mkdir "%s"' % self.dest_dir print('mkdir "%s"' % self.dest_dir)
else: else:
raise Exception('backup destination directory already exists : %s' % self.dest_dir) raise Exception('backup destination directory already exists : %s' % self.dest_dir)
@ -100,7 +100,7 @@ class backup_mysql(backup_generic):
self.logger.debug('[%s] Dump DB : %s',self.backup_name,cmd) self.logger.debug('[%s] Dump DB : %s',self.backup_name,cmd)
if not self.dry_run: if not self.dry_run:
(error_code,output) = ssh_exec(cmd,ssh=self.ssh) (error_code,output) = ssh_exec(cmd,ssh=self.ssh)
print output print(output)
self.logger.debug("[%s] Output of %s :\n%s",self.backup_name,cmd,output) self.logger.debug("[%s] Output of %s :\n%s",self.backup_name,cmd,output)
if error_code: if error_code:
raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd)) raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd))

2
libtisbackup/backup_null.py
View File

@ -20,7 +20,7 @@
import os import os
import datetime import datetime
from common import * from libtisbackup.common import *
class backup_null(backup_generic): class backup_null(backup_generic):

45
libtisbackup/backup_rsync.py
View File

@ -20,7 +20,7 @@
import os import os
import datetime import datetime
from common import * from libtisbackup.common import *
import time import time
import logging import logging
import re import re
@ -69,7 +69,7 @@ class backup_rsync(backup_generic):
if not self.dry_run: if not self.dry_run:
os.makedirs(dest_dir) os.makedirs(dest_dir)
else: else:
print 'mkdir "%s"' % dest_dir print('mkdir "%s"' % dest_dir)
else: else:
raise Exception('backup destination directory already exists : %s' % dest_dir) raise Exception('backup destination directory already exists : %s' % dest_dir)
@ -80,7 +80,7 @@ class backup_rsync(backup_generic):
if self.dry_run: if self.dry_run:
options.append('-d') options.append('-d')
if self.overload_args <> None: if self.overload_args is not None:
options.append(self.overload_args) options.append(self.overload_args)
elif not "cygdrive" in self.remote_dir: elif not "cygdrive" in self.remote_dir:
# we don't preserve owner, group, links, hardlinks, perms for windows/cygwin as it is not reliable nor useful # we don't preserve owner, group, links, hardlinks, perms for windows/cygwin as it is not reliable nor useful
@ -118,7 +118,7 @@ class backup_rsync(backup_generic):
try: try:
# newsettings with exclude_list='too','titi', parsed as a str python list content # newsettings with exclude_list='too','titi', parsed as a str python list content
excludes = eval('[%s]' % self.exclude_list) excludes = eval('[%s]' % self.exclude_list)
except Exception,e: except Exception as e:
raise Exception('Error reading exclude list : value %s, eval error %s (don\'t forget quotes and comma...)' % (self.exclude_list,e)) raise Exception('Error reading exclude list : value %s, eval error %s (don\'t forget quotes and comma...)' % (self.exclude_list,e))
options.extend(['--exclude="%s"' % x for x in excludes]) options.extend(['--exclude="%s"' % x for x in excludes])
@ -146,13 +146,13 @@ class backup_rsync(backup_generic):
ssh_params.append('-i %s' % self.private_key) ssh_params.append('-i %s' % self.private_key)
if self.cipher_spec: if self.cipher_spec:
ssh_params.append('-c %s' % self.cipher_spec) ssh_params.append('-c %s' % self.cipher_spec)
if self.ssh_port <> 22: if self.ssh_port != 22:
ssh_params.append('-p %i' % self.ssh_port) ssh_params.append('-p %i' % self.ssh_port)
options.append('-e "/usr/bin/ssh %s"' % (" ".join(ssh_params))) options.append('-e "/usr/bin/ssh %s"' % (" ".join(ssh_params)))
backup_source = '%s@%s:%s' % (self.remote_user,self.server_name,self.remote_dir) backup_source = '%s@%s:%s' % (self.remote_user,self.server_name,self.remote_dir)
# ensure there is a slash at end # ensure there is a slash at end
if backup_source[-1] <> '/': if backup_source[-1] != '/':
backup_source += '/' backup_source += '/'
options_params = " ".join(options) options_params = " ".join(options)
@ -165,25 +165,20 @@ class backup_rsync(backup_generic):
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
def ondata(data,context): def ondata(data,context):
if context.verbose: if context.verbose:
print data print(data)
context.logger.debug(data) context.logger.debug(data)
log = monitor_stdout(process,ondata,self) log = monitor_stdout(process,ondata,self)
reg_total_files = re.compile('Number of files: (?P<file>\d+)')
reg_transferred_files = re.compile('Number of .*files transferred: (?P<file>\d+)')
for l in log.splitlines(): for l in log.splitlines():
line = l.replace(',','') if l.startswith('Number of files:'):
m = reg_total_files.match(line) stats['total_files_count'] += int(re.sub("[^0-9]", "", l.split(':')[1]))
if m: if l.startswith('Number of files transferred:'):
stats['total_files_count'] += int(m.groupdict()['file']) stats['written_files_count'] += int(re.sub("[^0-9]", "", l.split(':')[1]))
m = reg_transferred_files.match(line) if l.startswith('Total file size:'):
if m: stats['total_bytes'] += int(re.sub("[^0-9]", "", l.split(':')[1].split()[0]))
stats['written_files_count'] += int(m.groupdict()['file']) if l.startswith('Total transferred file size:'):
if line.startswith('Total file size:'): stats['written_bytes'] += int(re.sub("[^0-9]", "", l.split(':')[1].split()[0]))
stats['total_bytes'] += int(line.split(':')[1].split()[0])
if line.startswith('Total transferred file size:'):
stats['written_bytes'] += int(line.split(':')[1].split()[0])
returncode = process.returncode returncode = process.returncode
## deal with exit code 24 (file vanished) ## deal with exit code 24 (file vanished)
@ -195,7 +190,7 @@ class backup_rsync(backup_generic):
self.logger.error("[" + self.backup_name + "] shell program exited with error code " + str(returncode)) self.logger.error("[" + self.backup_name + "] shell program exited with error code " + str(returncode))
raise Exception("[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd, log[-512:]) raise Exception("[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd, log[-512:])
else: else:
print cmd print(cmd)
#we suppress the .rsync suffix if everything went well #we suppress the .rsync suffix if everything went well
finaldest = os.path.join(self.backup_dir,self.backup_start_date) finaldest = os.path.join(self.backup_dir,self.backup_start_date)
@ -203,14 +198,14 @@ class backup_rsync(backup_generic):
if not self.dry_run: if not self.dry_run:
os.rename(dest_dir, finaldest) os.rename(dest_dir, finaldest)
self.logger.debug("[%s] touching datetime of target directory %s" ,self.backup_name,finaldest) self.logger.debug("[%s] touching datetime of target directory %s" ,self.backup_name,finaldest)
print os.popen('touch "%s"' % finaldest).read() print(os.popen('touch "%s"' % finaldest).read())
else: else:
print "mv" ,dest_dir,finaldest print("mv" ,dest_dir,finaldest)
stats['backup_location'] = finaldest stats['backup_location'] = finaldest
stats['status']='OK' stats['status']='OK'
stats['log']='ssh+rsync backup from %s OK, %d bytes written for %d changed files' % (backup_source,stats['written_bytes'],stats['written_files_count']) stats['log']='ssh+rsync backup from %s OK, %d bytes written for %d changed files' % (backup_source,stats['written_bytes'],stats['written_files_count'])
except BaseException , e: except BaseException as e:
stats['status']='ERROR' stats['status']='ERROR'
stats['log']=str(e) stats['log']=str(e)
raise raise
@ -340,5 +335,5 @@ if __name__=='__main__':
b = backup_rsync('htouvet','/backup/data/htouvet',dbstat) b = backup_rsync('htouvet','/backup/data/htouvet',dbstat)
b.read_config(cp) b.read_config(cp)
b.process_backup() b.process_backup()
print b.checknagios() print(b.checknagios())

66
libtisbackup/common.py
View File

@ -28,14 +28,15 @@ from iniparse import ConfigParser
import sqlite3 import sqlite3
import shutil import shutil
import select import select
import traceback
import sys import sys
try: try:
sys.stderr = open('/dev/null') # Silence silly warnings from paramiko sys.stderr = open('/dev/null') # Silence silly warnings from paramiko
import paramiko import paramiko
except ImportError,e: except ImportError as e:
print "Error : can not load paramiko library %s" % e print("Error : can not load paramiko library %s" % e)
raise raise
sys.stderr = sys.__stderr__ sys.stderr = sys.__stderr__
@ -121,7 +122,7 @@ def check_string(test_string):
pattern = r'[^\.A-Za-z0-9\-_]' pattern = r'[^\.A-Za-z0-9\-_]'
if re.search(pattern, test_string): if re.search(pattern, test_string):
#Character other then . a-z 0-9 was found #Character other then . a-z 0-9 was found
print 'Invalid : %r' % (test_string,) print('Invalid : %r' % (test_string,))
def convert_bytes(bytes): def convert_bytes(bytes):
if bytes is None: if bytes is None:
@ -227,7 +228,7 @@ def monitor_stdout(aprocess, onoutputdata,context):
assert(isinstance(aprocess,subprocess.Popen)) assert(isinstance(aprocess,subprocess.Popen))
read_set = [] read_set = []
stdout = [] stdout = []
line = '' line = b''
if aprocess.stdout: if aprocess.stdout:
read_set.append(aprocess.stdout) read_set.append(aprocess.stdout)
@ -237,7 +238,7 @@ def monitor_stdout(aprocess, onoutputdata,context):
while read_set: while read_set:
try: try:
rlist, wlist, xlist = select.select(read_set, [], []) rlist, wlist, xlist = select.select(read_set, [], [])
except select.error, e: except select.error as e:
if e.args[0] == errno.EINTR: if e.args[0] == errno.EINTR:
continue continue
raise raise
@ -245,38 +246,38 @@ def monitor_stdout(aprocess, onoutputdata,context):
# Reads one line from stdout # Reads one line from stdout
if aprocess.stdout in rlist: if aprocess.stdout in rlist:
data = os.read(aprocess.stdout.fileno(), 1) data = os.read(aprocess.stdout.fileno(), 1)
if data == "": if data == b"":
aprocess.stdout.close() aprocess.stdout.close()
read_set.remove(aprocess.stdout) read_set.remove(aprocess.stdout)
while data and not data in ('\n','\r'): while data and not data in (b'\n',b'\r'):
line += data line += data
data = os.read(aprocess.stdout.fileno(), 1) data = os.read(aprocess.stdout.fileno(), 1)
if line or data in ('\n','\r'): if line or data in (b'\n',b'\r'):
stdout.append(line) stdout.append(line.decode('utf8'))
if onoutputdata: if onoutputdata:
onoutputdata(line,context) onoutputdata(line.decode('utf8'),context)
line='' line=b''
# Reads one line from stderr # Reads one line from stderr
if aprocess.stderr in rlist: if aprocess.stderr in rlist:
data = os.read(aprocess.stderr.fileno(), 1) data = os.read(aprocess.stderr.fileno(), 1)
if data == "": if data == b"":
aprocess.stderr.close() aprocess.stderr.close()
read_set.remove(aprocess.stderr) read_set.remove(aprocess.stderr)
while data and not data in ('\n','\r'): while data and not data in (b'\n',b'\r'):
line += data line += data
data = os.read(aprocess.stderr.fileno(), 1) data = os.read(aprocess.stderr.fileno(), 1)
if line or data in ('\n','\r'): if line or data in (b'\n',b'\r'):
stdout.append(line) stdout.append(line.decode('utf8'))
if onoutputdata: if onoutputdata:
onoutputdata(line,context) onoutputdata(line.decode('utf8'),context)
line='' line=b''
aprocess.wait() aprocess.wait()
if line: if line:
stdout.append(line) stdout.append(line.decode('utf8'))
if onoutputdata: if onoutputdata:
onoutputdata(line,context) onoutputdata(line.decode('utf8'),context)
return "\n".join(stdout) return "\n".join(stdout)
@ -442,7 +443,7 @@ CREATE INDEX idx_stats_backup_name_start on stats(backup_name,backup_start);""")
return value return value
#for r in self.query('select * from stats where backup_name=? order by backup_end desc limit ?',(backup_name,count)): #for r in self.query('select * from stats where backup_name=? order by backup_end desc limit ?',(backup_name,count)):
print pp(cur,None,1,fcb) print(pp(cur,None,1,fcb))
def fcb(self,fields,fieldname,value): def fcb(self,fields,fieldname,value):
@ -544,11 +545,12 @@ class backup_generic:
'optional':",".join(cls.optional_params)} 'optional':",".join(cls.optional_params)}
def check_required_params(self): def check_required_params(self):
for name in self.required_params: #for name in self.required_params:
if not hasattr(self,name) or not getattr(self,name): # if not hasattr(self,name) or not getattr(self,name):
raise Exception('[%s] Config Attribute %s is required' % (self.backup_name,name)) # raise Exception('[%s] Config Attribute %s is required' % (self.backup_name,name))
if (self.preexec or self.postexec) and (not self.private_key or not self.remote_user): #if (self.preexec or self.postexec) and (not self.private_key or not self.remote_user):
raise Exception('[%s] remote_user and private_key file required if preexec or postexec is used' % self.backup_name) # raise Exception('[%s] remote_user and private_key file required if preexec or postexec is used' % self.backup_name)
pass
def read_config(self,iniconf): def read_config(self,iniconf):
@ -695,7 +697,7 @@ class backup_generic:
self.logger.info('[%s] ######### Backup finished : %s',self.backup_name,stats['log']) self.logger.info('[%s] ######### Backup finished : %s',self.backup_name,stats['log'])
return stats return stats
except BaseException, e: except BaseException as e:
stats['status']='ERROR' stats['status']='ERROR'
stats['log']=str(e) stats['log']=str(e)
endtime = time.time() endtime = time.time()
@ -713,6 +715,8 @@ class backup_generic:
backup_location=stats['backup_location']) backup_location=stats['backup_location'])
self.logger.error('[%s] ######### Backup finished with ERROR: %s',self.backup_name,stats['log']) self.logger.error('[%s] ######### Backup finished with ERROR: %s',self.backup_name,stats['log'])
self.logger.debug(traceback.format_exc())
raise raise
@ -797,7 +801,7 @@ class backup_generic:
if not self.dry_run: if not self.dry_run:
self.dbstat.db.execute('update stats set TYPE="CLEAN" where backup_name=? and backup_location=?',(self.backup_name,oldbackup_location)) self.dbstat.db.execute('update stats set TYPE="CLEAN" where backup_name=? and backup_location=?',(self.backup_name,oldbackup_location))
self.dbstat.db.commit() self.dbstat.db.commit()
except BaseException,e: except BaseException as e:
self.logger.error('cleanup_backup : Unable to remove directory/file "%s". Error %s', oldbackup_location,e) self.logger.error('cleanup_backup : Unable to remove directory/file "%s". Error %s', oldbackup_location,e)
removed.append((self.backup_name,oldbackup_location)) removed.append((self.backup_name,oldbackup_location))
else: else:
@ -845,9 +849,9 @@ class backup_generic:
raise Exception('Backup source %s doesn\'t exists' % backup_source) raise Exception('Backup source %s doesn\'t exists' % backup_source)
# ensure there is a slash at end # ensure there is a slash at end
if os.path.isdir(backup_source) and backup_source[-1] <> '/': if os.path.isdir(backup_source) and backup_source[-1] != '/':
backup_source += '/' backup_source += '/'
if backup_dest[-1] <> '/': if backup_dest[-1] != '/':
backup_dest += '/' backup_dest += '/'
if not os.path.isdir(backup_dest): if not os.path.isdir(backup_dest):
@ -873,7 +877,7 @@ class backup_generic:
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
def ondata(data,context): def ondata(data,context):
if context.verbose: if context.verbose:
print data print(data)
context.logger.debug(data) context.logger.debug(data)
log = monitor_stdout(process,ondata,self) log = monitor_stdout(process,ondata,self)
@ -897,7 +901,7 @@ class backup_generic:
self.logger.error("[" + self.backup_name + "] shell program exited with error code ") self.logger.error("[" + self.backup_name + "] shell program exited with error code ")
raise Exception("[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd) raise Exception("[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd)
else: else:
print cmd print(cmd)
stats['status']='OK' stats['status']='OK'
self.logger.info('export backup from %s to %s OK, %d bytes written for %d changed files' % (backup_source,backup_dest,stats['written_bytes'],stats['written_files_count'])) self.logger.info('export backup from %s to %s OK, %d bytes written for %d changed files' % (backup_source,backup_dest,stats['written_bytes'],stats['written_files_count']))

41
tisbackup.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/python #!/usr/bin/python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# ----------------------------------------------------------------------- # -----------------------------------------------------------------------
# This file is part of TISBackup # This file is part of TISBackup
@ -32,18 +32,17 @@ from libtisbackup.common import *
from libtisbackup.backup_mysql import backup_mysql from libtisbackup.backup_mysql import backup_mysql
from libtisbackup.backup_rsync import backup_rsync from libtisbackup.backup_rsync import backup_rsync
from libtisbackup.backup_rsync import backup_rsync_ssh from libtisbackup.backup_rsync import backup_rsync_ssh
from libtisbackup.backup_oracle import backup_oracle #from libtisbackup.backup_oracle import backup_oracle
from libtisbackup.backup_rsync_btrfs import backup_rsync_btrfs #from libtisbackup.backup_rsync_btrfs import backup_rsync_btrfs
from libtisbackup.backup_rsync_btrfs import backup_rsync__btrfs_ssh #from libtisbackup.backup_rsync_btrfs import backup_rsync__btrfs_ssh
from libtisbackup.backup_pgsql import backup_pgsql #from libtisbackup.backup_pgsql import backup_pgsql
from libtisbackup.backup_xva import backup_xva #from libtisbackup.backup_xva import backup_xva
from libtisbackup.backup_vmdk import backup_vmdk #from libtisbackup.backup_switch import backup_switch
from libtisbackup.backup_switch import backup_switch
from libtisbackup.backup_null import backup_null from libtisbackup.backup_null import backup_null
from libtisbackup.backup_xcp_metadata import backup_xcp_metadata #from libtisbackup.backup_xcp_metadata import backup_xcp_metadata
from libtisbackup.copy_vm_xcp import copy_vm_xcp #from libtisbackup.copy_vm_xcp import copy_vm_xcp
from libtisbackup.backup_sqlserver import backup_sqlserver #from libtisbackup.backup_sqlserver import backup_sqlserver
from libtisbackup.backup_samba4 import backup_samba4 #from libtisbackup.backup_samba4 import backup_samba4
usage="""\ usage="""\
%prog -c configfile action %prog -c configfile action
@ -176,15 +175,15 @@ class tis_backup:
nagiosoutput = 'ALL backups OK %s' % (','.join(sections)) nagiosoutput = 'ALL backups OK %s' % (','.join(sections))
except BaseException,e: except BaseException as e:
worst_nagiosstatus = nagiosStateCritical worst_nagiosstatus = nagiosStateCritical
nagiosoutput = 'EXCEPTION',"Critical : %s" % str(e) nagiosoutput = 'EXCEPTION',"Critical : %s" % str(e)
raise raise
finally: finally:
self.logger.debug('worst nagios status :"%i"',worst_nagiosstatus) self.logger.debug('worst nagios status :"%i"',worst_nagiosstatus)
print '%s (tisbackup V%s)' %(nagiosoutput,version) print('%s (tisbackup V%s)' %(nagiosoutput,version))
print '\n'.join(["[%s]:%s" % (l[0],l[1]) for l in globallog]) print('\n'.join(["[%s]:%s" % (l[0],l[1]) for l in globallog]))
sys.exit(worst_nagiosstatus) sys.exit(worst_nagiosstatus)
def process_backup(self,sections=[]): def process_backup(self,sections=[]):
@ -201,7 +200,7 @@ class tis_backup:
self.logger.info('Processing [%s]',(backup_item.backup_name)) self.logger.info('Processing [%s]',(backup_item.backup_name))
stats = backup_item.process_backup() stats = backup_item.process_backup()
processed.append((backup_item.backup_name,stats)) processed.append((backup_item.backup_name,stats))
except BaseException,e: except BaseException as e:
self.logger.critical('Backup [%s] processed with error : %s',backup_item.backup_name,e) self.logger.critical('Backup [%s] processed with error : %s',backup_item.backup_name,e)
errors.append((backup_item.backup_name,str(e))) errors.append((backup_item.backup_name,str(e)))
if not processed and not errors: if not processed and not errors:
@ -227,7 +226,7 @@ class tis_backup:
self.logger.info('Processing [%s]',(backup_item.backup_name)) self.logger.info('Processing [%s]',(backup_item.backup_name))
stats = backup_item.export_latestbackup(destdir=exportdir) stats = backup_item.export_latestbackup(destdir=exportdir)
processed.append((backup_item.backup_name,stats)) processed.append((backup_item.backup_name,stats))
except BaseException,e: except BaseException as e:
self.logger.critical('Export Backup [%s] processed with error : %s',backup_item.backup_name,e) self.logger.critical('Export Backup [%s] processed with error : %s',backup_item.backup_name,e)
errors.append((backup_item.backup_name,str(e))) errors.append((backup_item.backup_name,str(e)))
if not processed and not errors: if not processed and not errors:
@ -262,7 +261,7 @@ class tis_backup:
self.logger.info('Processing [%s]',(backup_item.backup_name)) self.logger.info('Processing [%s]',(backup_item.backup_name))
stats = backup_item.process_backup() stats = backup_item.process_backup()
processed.append((backup_item.backup_name,stats)) processed.append((backup_item.backup_name,stats))
except BaseException,e: except BaseException as e:
self.logger.critical('Backup [%s] not processed, error : %s',backup_item.backup_name,e) self.logger.critical('Backup [%s] not processed, error : %s',backup_item.backup_name,e)
errors.append((backup_item.backup_name,str(e))) errors.append((backup_item.backup_name,str(e)))
if not processed and not errors: if not processed and not errors:
@ -290,7 +289,7 @@ class tis_backup:
self.logger.info('Processing cleanup of [%s]',(backup_item.backup_name)) self.logger.info('Processing cleanup of [%s]',(backup_item.backup_name))
backup_item.cleanup_backup() backup_item.cleanup_backup()
processed = True processed = True
except BaseException,e: except BaseException as e:
self.logger.critical('Cleanup of [%s] not processed, error : %s',backup_item.backup_name,e) self.logger.critical('Cleanup of [%s] not processed, error : %s',backup_item.backup_name,e)
if not processed: if not processed:
self.logger.critical('No cleanup properly finished or processed') self.logger.critical('No cleanup properly finished or processed')
@ -322,7 +321,7 @@ def main():
(options,args)=parser.parse_args() (options,args)=parser.parse_args()
if len(args) != 1: if len(args) != 1:
print "ERROR : You must provide one action to perform" print("ERROR : You must provide one action to perform")
parser.print_usage() parser.print_usage()
sys.exit(2) sys.exit(2)
@ -332,7 +331,7 @@ def main():
action = args[0] action = args[0]
if action == "listdrivers": if action == "listdrivers":
for t in backup_drivers: for t in backup_drivers:
print backup_drivers[t].get_help() print(backup_drivers[t].get_help())
sys.exit(0) sys.exit(0)
config_file =options.config config_file =options.config