From cae9a1a4dc9114ba316110abf1663274a3e1198f Mon Sep 17 00:00:00 2001 From: ssamson-tis Date: Fri, 10 Jul 2015 10:56:22 +0200 Subject: [PATCH] Type mysql: - add multiple database support --- libtisbackup/backup_mysql.py | 103 +++++++++++++++++++++++++---------- 1 file changed, 75 insertions(+), 28 deletions(-) diff --git a/libtisbackup/backup_mysql.py b/libtisbackup/backup_mysql.py index 050f988..6a70d86 100644 --- a/libtisbackup/backup_mysql.py +++ b/libtisbackup/backup_mysql.py @@ -38,12 +38,26 @@ from common import * class backup_mysql(backup_generic): """Backup a mysql database as gzipped sql file through ssh""" type = 'mysql+ssh' - required_params = backup_generic.required_params + ['db_name','db_user','db_passwd','private_key'] + required_params = backup_generic.required_params + ['db_user','db_passwd','private_key'] + optional_params = backup_generic.optional_params + ['db_name'] + db_name='' db_user='' db_passwd='' + dest_dir = "" + def do_backup(self,stats): + self.dest_dir = os.path.join(self.backup_dir,self.backup_start_date) + + + if not os.path.isdir(self.dest_dir): + if not self.dry_run: + os.makedirs(self.dest_dir) + else: + print 'mkdir "%s"' % self.dest_dir + else: + raise Exception('backup destination directory already exists : %s' % self.dest_dir) self.logger.debug('[%s] Connecting to %s with user root and key %s',self.backup_name,self.server_name,self.private_key) try: @@ -51,19 +65,44 @@ class backup_mysql(backup_generic): except paramiko.SSHException: mykey = paramiko.DSSKey.from_private_key_file(self.private_key) - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(self.server_name,username='root',pkey = mykey, port=self.ssh_port) + self.ssh = paramiko.SSHClient() + self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.ssh.connect(self.server_name,username='root',pkey = mykey, port=self.ssh_port) + + if not self.db_name: + stats['log']= "Successfully backuping processed to the following databases :" + stats['status']='List' + cmd = 'mysql -N -B -p -e "SHOW DATABASES;" -u ' + self.db_user +' -p' + self.db_passwd + self.logger.debug('[%s] List databases: %s',self.backup_name,cmd) + (error_code,output) = ssh_exec(cmd,ssh=self.ssh) + self.logger.debug("[%s] Output of %s :\n%s",self.backup_name,cmd,output) + if error_code: + raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd)) + databases = output.split('\n') + for database in databases: + if database != "": + self.db_name = database.rstrip() + self.do_mysqldump(stats) + + else: + stats['log']= "Successfully backup processed to the following database :" + self.do_mysqldump(stats) + + + def do_mysqldump(self,stats): + + t = datetime.datetime.now() backup_start_date = t.strftime('%Y%m%d-%Hh%Mm%S') # dump db stats['status']='Dumping' - cmd = 'mysqldump -u' + self.db_user +' -p' + self.db_passwd + ' ' + self.db_name + ' > /tmp/' + self.db_name + '-' + backup_start_date + '.sql' + cmd = 'mysqldump --single-transaction -u' + self.db_user +' -p' + self.db_passwd + ' ' + self.db_name + ' > /tmp/' + self.db_name + '-' + backup_start_date + '.sql' self.logger.debug('[%s] Dump DB : %s',self.backup_name,cmd) if not self.dry_run: - (error_code,output) = ssh_exec(cmd,ssh=ssh) + (error_code,output) = ssh_exec(cmd,ssh=self.ssh) + print output self.logger.debug("[%s] Output of %s :\n%s",self.backup_name,cmd,output) if error_code: raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd)) @@ -73,7 +112,7 @@ class backup_mysql(backup_generic): cmd = 'gzip /tmp/' + self.db_name + '-' + backup_start_date + '.sql' self.logger.debug('[%s] Compress backup : %s',self.backup_name,cmd) if not self.dry_run: - (error_code,output) = ssh_exec(cmd,ssh=ssh) + (error_code,output) = ssh_exec(cmd,ssh=self.ssh) self.logger.debug("[%s] Output of %s :\n%s",self.backup_name,cmd,output) if error_code: raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd)) @@ -81,27 +120,27 @@ class backup_mysql(backup_generic): # get the file stats['status']='SFTP' filepath = '/tmp/' + self.db_name + '-' + backup_start_date + '.sql.gz' - localpath = os.path.join(self.backup_dir , self.db_name + '-' + backup_start_date + '.sql.gz') + localpath = os.path.join(self.dest_dir , self.db_name + '.sql.gz') self.logger.debug('[%s] Get gz backup with sftp on %s from %s to %s',self.backup_name,self.server_name,filepath,localpath) if not self.dry_run: - transport = ssh.get_transport() + transport = self.ssh.get_transport() sftp = paramiko.SFTPClient.from_transport(transport) sftp.get(filepath, localpath) sftp.close() if not self.dry_run: - stats['total_files_count']=1 - stats['written_files_count']=1 - stats['total_bytes']=os.stat(localpath).st_size - stats['written_bytes']=os.stat(localpath).st_size - stats['log']='gzip dump of DB %s:%s (%d bytes) to %s' % (self.server_name,self.db_name, stats['written_bytes'], localpath) - stats['backup_location'] = localpath + stats['total_files_count']=1 + stats.get('total_files_count', 0) + stats['written_files_count']=1 + stats.get('written_files_count', 0) + stats['total_bytes']=os.stat(localpath).st_size + stats.get('total_bytes', 0) + stats['written_bytes']=os.stat(localpath).st_size + stats.get('written_bytes', 0) + stats['log'] = '%s "%s"' % (stats['log'] ,self.db_name) + stats['backup_location'] = self.dest_dir stats['status']='RMTemp' cmd = 'rm -f /tmp/' + self.db_name + '-' + backup_start_date + '.sql.gz' self.logger.debug('[%s] Remove temp gzip : %s',self.backup_name,cmd) if not self.dry_run: - (error_code,output) = ssh_exec(cmd,ssh=ssh) + (error_code,output) = ssh_exec(cmd,ssh=self.ssh) self.logger.debug("[%s] Output of %s :\n%s",self.backup_name,cmd,output) if error_code: raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd)) @@ -114,20 +153,28 @@ class backup_mysql(backup_generic): filelist = os.listdir(self.backup_dir) filelist.sort() - p = re.compile('^%s-(?P\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}).sql.gz$' % self.db_name) + p = re.compile('^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}$') for item in filelist: - sr = p.match(item) - if sr: - file_name = os.path.join(self.backup_dir,item) - start = datetime.datetime.strptime(sr.groups()[0],'%Y%m%d-%Hh%Mm%S').isoformat() - if not file_name in registered: - self.logger.info('Registering %s from %s',file_name,fileisodate(file_name)) - size_bytes = int(os.popen('du -sb "%s"' % file_name).read().split('\t')[0]) + if p.match(item): + dir_name = os.path.join(self.backup_dir,item) + if not dir_name in registered: + start = datetime.datetime.strptime(item,'%Y%m%d-%Hh%Mm%S').isoformat() + if fileisodate(dir_name)>start: + stop = fileisodate(dir_name) + else: + stop = start + self.logger.info('Registering %s started on %s',dir_name,start) + self.logger.debug(' Disk usage %s','du -sb "%s"' % dir_name) + if not self.dry_run: + size_bytes = int(os.popen('du -sb "%s"' % dir_name).read().split('\t')[0]) + else: + size_bytes = 0 self.logger.debug(' Size in bytes : %i',size_bytes) - if not self.dry_run: + if not self.dry_run: self.dbstat.add(self.backup_name,self.server_name,'',\ - backup_start=start,backup_end=fileisodate(file_name),status='OK',total_bytes=size_bytes,backup_location=file_name) + backup_start=start,backup_end = stop,status='OK',total_bytes=size_bytes,backup_location=dir_name) else: - self.logger.info('Skipping %s from %s, already registered',file_name,fileisodate(file_name)) + self.logger.info('Skipping %s, already registered',dir_name) -register_driver(backup_mysql) + +register_driver(backup_mysql) \ No newline at end of file