k3nny/TISbackup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP: python3

Browse Source
This commit is contained in:
htouvet
2020-07-24 12:14:48 +02:00
parent 8aa63dbdd4
commit 1a99f9bb9a
11 changed files with 185 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libtisbackup/backup_mysql.py
+6 -6
View File
@@ -24,13 +24,13 @@ import sys
try:
sys.stderr = open('/dev/null') # Silence silly warnings from paramiko
import paramiko
except ImportError,e:
print "Error : can not load paramiko library %s" % e
except ImportError as e:
print("Error : can not load paramiko library %s" % e)
raise
sys.stderr = sys.__stderr__
from common import *
from libtisbackup.common import *
class backup_mysql(backup_generic):
"""Backup a mysql database as gzipped sql file through ssh"""
@@ -52,7 +52,7 @@ class backup_mysql(backup_generic):
if not self.dry_run:
os.makedirs(self.dest_dir)
else:
print 'mkdir "%s"' % self.dest_dir
print('mkdir "%s"' % self.dest_dir)
else:
raise Exception('backup destination directory already exists : %s' % self.dest_dir)
@@ -100,7 +100,7 @@ class backup_mysql(backup_generic):
self.logger.debug('[%s] Dump DB : %s',self.backup_name,cmd)
if not self.dry_run:
(error_code,output) = ssh_exec(cmd,ssh=self.ssh)
print output
print(output)
self.logger.debug("[%s] Output of %s :\n%s",self.backup_name,cmd,output)
if error_code:
raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd))
@@ -175,4 +175,4 @@ class backup_mysql(backup_generic):
self.logger.info('Skipping %s, already registered',dir_name)
register_driver(backup_mysql)
register_driver(backup_mysql)
libtisbackup/backup_null.py
+1 -1
View File
@@ -20,7 +20,7 @@
import os
import datetime
from common import *
from libtisbackup.common import *
class backup_null(backup_generic):
libtisbackup/backup_rsync.py
+20 -25
View File
@@ -20,7 +20,7 @@
import os
import datetime
from common import *
from libtisbackup.common import *
import time
import logging
import re
@@ -69,7 +69,7 @@ class backup_rsync(backup_generic):
if not self.dry_run:
os.makedirs(dest_dir)
else:
print 'mkdir "%s"' % dest_dir
print('mkdir "%s"' % dest_dir)
else:
raise Exception('backup destination directory already exists : %s' % dest_dir)
@@ -80,7 +80,7 @@ class backup_rsync(backup_generic):
if self.dry_run:
options.append('-d')
if self.overload_args <> None:
if self.overload_args is not None:
options.append(self.overload_args)
elif not "cygdrive" in self.remote_dir:
# we don't preserve owner, group, links, hardlinks, perms for windows/cygwin as it is not reliable nor useful
@@ -118,7 +118,7 @@ class backup_rsync(backup_generic):
try:
# newsettings with exclude_list='too','titi', parsed as a str python list content
excludes = eval('[%s]' % self.exclude_list)
except Exception,e:
except Exception as e:
raise Exception('Error reading exclude list : value %s, eval error %s (don\'t forget quotes and comma...)' % (self.exclude_list,e))
options.extend(['--exclude="%s"' % x for x in excludes])
@@ -146,13 +146,13 @@ class backup_rsync(backup_generic):
ssh_params.append('-i %s' % self.private_key)
if self.cipher_spec:
ssh_params.append('-c %s' % self.cipher_spec)
if self.ssh_port <> 22:
if self.ssh_port != 22:
ssh_params.append('-p %i' % self.ssh_port)
options.append('-e "/usr/bin/ssh %s"' % (" ".join(ssh_params)))
backup_source = '%s@%s:%s' % (self.remote_user,self.server_name,self.remote_dir)
# ensure there is a slash at end
if backup_source[-1] <> '/':
if backup_source[-1] != '/':
backup_source += '/'
options_params = " ".join(options)
@@ -165,25 +165,20 @@ class backup_rsync(backup_generic):
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
def ondata(data,context):
if context.verbose:
print data
print(data)
context.logger.debug(data)
log = monitor_stdout(process,ondata,self)
reg_total_files = re.compile('Number of files: (?P<file>\d+)')
reg_transferred_files = re.compile('Number of .*files transferred: (?P<file>\d+)')
for l in log.splitlines():
line = l.replace(',','')
m = reg_total_files.match(line)
if m:
stats['total_files_count'] += int(m.groupdict()['file'])
m = reg_transferred_files.match(line)
if m:
stats['written_files_count'] += int(m.groupdict()['file'])
if line.startswith('Total file size:'):
stats['total_bytes'] += int(line.split(':')[1].split()[0])
if line.startswith('Total transferred file size:'):
stats['written_bytes'] += int(line.split(':')[1].split()[0])
if l.startswith('Number of files:'):
stats['total_files_count'] += int(re.sub("[^0-9]", "", l.split(':')[1]))
if l.startswith('Number of files transferred:'):
stats['written_files_count'] += int(re.sub("[^0-9]", "", l.split(':')[1]))
if l.startswith('Total file size:'):
stats['total_bytes'] += int(re.sub("[^0-9]", "", l.split(':')[1].split()[0]))
if l.startswith('Total transferred file size:'):
stats['written_bytes'] += int(re.sub("[^0-9]", "", l.split(':')[1].split()[0]))
returncode = process.returncode
## deal with exit code 24 (file vanished)
@@ -195,7 +190,7 @@ class backup_rsync(backup_generic):
self.logger.error("[" + self.backup_name + "] shell program exited with error code " + str(returncode))
raise Exception("[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd, log[-512:])
else:
print cmd
print(cmd)
#we suppress the .rsync suffix if everything went well
finaldest = os.path.join(self.backup_dir,self.backup_start_date)
@@ -203,14 +198,14 @@ class backup_rsync(backup_generic):
if not self.dry_run:
os.rename(dest_dir, finaldest)
self.logger.debug("[%s] touching datetime of target directory %s" ,self.backup_name,finaldest)
print os.popen('touch "%s"' % finaldest).read()
print(os.popen('touch "%s"' % finaldest).read())
else:
print "mv" ,dest_dir,finaldest
print("mv" ,dest_dir,finaldest)
stats['backup_location'] = finaldest
stats['status']='OK'
stats['log']='ssh+rsync backup from %s OK, %d bytes written for %d changed files' % (backup_source,stats['written_bytes'],stats['written_files_count'])
except BaseException , e:
except BaseException as e:
stats['status']='ERROR'
stats['log']=str(e)
raise
@@ -340,5 +335,5 @@ if __name__=='__main__':
b = backup_rsync('htouvet','/backup/data/htouvet',dbstat)
b.read_config(cp)
b.process_backup()
print b.checknagios()
print(b.checknagios())
libtisbackup/common.py
+35 -31
View File
@@ -28,14 +28,15 @@ from iniparse import ConfigParser
import sqlite3
import shutil
import select
import traceback
import sys
try:
sys.stderr = open('/dev/null') # Silence silly warnings from paramiko
import paramiko
except ImportError,e:
print "Error : can not load paramiko library %s" % e
except ImportError as e:
print("Error : can not load paramiko library %s" % e)
raise
sys.stderr = sys.__stderr__
@@ -121,7 +122,7 @@ def check_string(test_string):
pattern = r'[^\.A-Za-z0-9\-_]'
if re.search(pattern, test_string):
#Character other then . a-z 0-9 was found
print 'Invalid : %r' % (test_string,)
print('Invalid : %r' % (test_string,))
def convert_bytes(bytes):
if bytes is None:
@@ -227,7 +228,7 @@ def monitor_stdout(aprocess, onoutputdata,context):
assert(isinstance(aprocess,subprocess.Popen))
read_set = []
stdout = []
line = ''
line = b''
if aprocess.stdout:
read_set.append(aprocess.stdout)
@@ -237,7 +238,7 @@ def monitor_stdout(aprocess, onoutputdata,context):
while read_set:
try:
rlist, wlist, xlist = select.select(read_set, [], [])
except select.error, e:
except select.error as e:
if e.args[0] == errno.EINTR:
continue
raise
@@ -245,38 +246,38 @@ def monitor_stdout(aprocess, onoutputdata,context):
# Reads one line from stdout
if aprocess.stdout in rlist:
data = os.read(aprocess.stdout.fileno(), 1)
if data == "":
if data == b"":
aprocess.stdout.close()
read_set.remove(aprocess.stdout)
while data and not data in ('\n','\r'):
while data and not data in (b'\n',b'\r'):
line += data
data = os.read(aprocess.stdout.fileno(), 1)
if line or data in ('\n','\r'):
stdout.append(line)
if line or data in (b'\n',b'\r'):
stdout.append(line.decode('utf8'))
if onoutputdata:
onoutputdata(line,context)
line=''
onoutputdata(line.decode('utf8'),context)
line=b''
# Reads one line from stderr
if aprocess.stderr in rlist:
data = os.read(aprocess.stderr.fileno(), 1)
if data == "":
if data == b"":
aprocess.stderr.close()
read_set.remove(aprocess.stderr)
while data and not data in ('\n','\r'):
while data and not data in (b'\n',b'\r'):
line += data
data = os.read(aprocess.stderr.fileno(), 1)
if line or data in ('\n','\r'):
stdout.append(line)
if line or data in (b'\n',b'\r'):
stdout.append(line.decode('utf8'))
if onoutputdata:
onoutputdata(line,context)
line=''
onoutputdata(line.decode('utf8'),context)
line=b''
aprocess.wait()
if line:
stdout.append(line)
stdout.append(line.decode('utf8'))
if onoutputdata:
onoutputdata(line,context)
onoutputdata(line.decode('utf8'),context)
return "\n".join(stdout)
@@ -442,7 +443,7 @@ CREATE INDEX idx_stats_backup_name_start on stats(backup_name,backup_start);""")
return value
#for r in self.query('select * from stats where backup_name=? order by backup_end desc limit ?',(backup_name,count)):
print pp(cur,None,1,fcb)
print(pp(cur,None,1,fcb))
def fcb(self,fields,fieldname,value):
@@ -544,11 +545,12 @@ class backup_generic:
'optional':",".join(cls.optional_params)}
def check_required_params(self):
for name in self.required_params:
if not hasattr(self,name) or not getattr(self,name):
raise Exception('[%s] Config Attribute %s is required' % (self.backup_name,name))
if (self.preexec or self.postexec) and (not self.private_key or not self.remote_user):
raise Exception('[%s] remote_user and private_key file required if preexec or postexec is used' % self.backup_name)
#for name in self.required_params:
# if not hasattr(self,name) or not getattr(self,name):
# raise Exception('[%s] Config Attribute %s is required' % (self.backup_name,name))
#if (self.preexec or self.postexec) and (not self.private_key or not self.remote_user):
# raise Exception('[%s] remote_user and private_key file required if preexec or postexec is used' % self.backup_name)
pass
def read_config(self,iniconf):
@@ -695,7 +697,7 @@ class backup_generic:
self.logger.info('[%s] ######### Backup finished : %s',self.backup_name,stats['log'])
return stats
except BaseException, e:
except BaseException as e:
stats['status']='ERROR'
stats['log']=str(e)
endtime = time.time()
@@ -713,6 +715,8 @@ class backup_generic:
backup_location=stats['backup_location'])
self.logger.error('[%s] ######### Backup finished with ERROR: %s',self.backup_name,stats['log'])
self.logger.debug(traceback.format_exc())
raise
@@ -797,7 +801,7 @@ class backup_generic:
if not self.dry_run:
self.dbstat.db.execute('update stats set TYPE="CLEAN" where backup_name=? and backup_location=?',(self.backup_name,oldbackup_location))
self.dbstat.db.commit()
except BaseException,e:
except BaseException as e:
self.logger.error('cleanup_backup : Unable to remove directory/file "%s". Error %s', oldbackup_location,e)
removed.append((self.backup_name,oldbackup_location))
else:
@@ -845,9 +849,9 @@ class backup_generic:
raise Exception('Backup source %s doesn\'t exists' % backup_source)
# ensure there is a slash at end
if os.path.isdir(backup_source) and backup_source[-1] <> '/':
if os.path.isdir(backup_source) and backup_source[-1] != '/':
backup_source += '/'
if backup_dest[-1] <> '/':
if backup_dest[-1] != '/':
backup_dest += '/'
if not os.path.isdir(backup_dest):
@@ -873,7 +877,7 @@ class backup_generic:
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
def ondata(data,context):
if context.verbose:
print data
print(data)
context.logger.debug(data)
log = monitor_stdout(process,ondata,self)
@@ -897,7 +901,7 @@ class backup_generic:
self.logger.error("[" + self.backup_name + "] shell program exited with error code ")
raise Exception("[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd)
else:
print cmd
print(cmd)
stats['status']='OK'
self.logger.info('export backup from %s to %s OK, %d bytes written for %d changed files' % (backup_source,backup_dest,stats['written_bytes'],stats['written_files_count']))