From 690292ece9934f1e1f8065675a0a665581e1394b Mon Sep 17 00:00:00 2001 From: ssamson-tis Date: Fri, 25 Jul 2014 15:01:18 +0200 Subject: [PATCH] Ajout support des backups via btrfs --- libtisbackup/backup_rsync_btrfs.py | 359 +++++++++++++++++++++++++++++ 1 file changed, 359 insertions(+) create mode 100644 libtisbackup/backup_rsync_btrfs.py diff --git a/libtisbackup/backup_rsync_btrfs.py b/libtisbackup/backup_rsync_btrfs.py new file mode 100644 index 0000000..a89f2b2 --- /dev/null +++ b/libtisbackup/backup_rsync_btrfs.py @@ -0,0 +1,359 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------- +# This file is part of TISBackup +# +# TISBackup is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# TISBackup is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with TISBackup. If not, see . +# +# ----------------------------------------------------------------------- + +import os +import datetime +from common import * +import time +import logging +import re +import os.path +import datetime +from common import * + + +class backup_rsync_btrfs(backup_generic): + """Backup a directory on remote server with rsync and btrfs protocol (requires running remote rsync daemon)""" + type = 'rsync+btrfs' + required_params = backup_generic.required_params + ['remote_user','remote_dir','rsync_module','password_file'] + optional_params = backup_generic.optional_params + ['compressionlevel','compression','bwlimit','exclude_list','protect_args','overload_args'] + + remote_user='root' + remote_dir='' + + exclude_list='' + rsync_module='' + password_file = '' + compression = '' + bwlimit = 0 + protect_args = '1' + overload_args = None + compressionlevel = 0 + + + + def read_config(self,iniconf): + assert(isinstance(iniconf,ConfigParser)) + backup_generic.read_config(self,iniconf) + if not self.bwlimit and iniconf.has_option('global','bw_limit'): + self.bwlimit = iniconf.getint('global','bw_limit') + if not self.compressionlevel and iniconf.has_option('global','compression_level'): + self.compressionlevel = iniconf.getint('global','compression_level') + + def do_backup(self,stats): + if not self.set_lock(): + self.logger.error("[%s] a lock file is set, a backup maybe already running!!",self.backup_name) + return False + + try: + try: + backup_source = 'undefined' + dest_dir = os.path.join(self.backup_dir,'last_backup') + if not os.path.isdir(dest_dir): + if not self.dry_run: + cmd = "/sbin/btrfs subvolume create %s"%dest_dir + process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) + log = monitor_stdout(process,'',self) + returncode = process.returncode + if (returncode != 0): + self.logger.error("[" + self.backup_name + "] shell program exited with error code: %s"%log) + raise Exception("[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd) + else: + self.logger.info("[" + self.backup_name + "] create btrs volume: %s"%dest_dir) + else: + print 'btrfs subvolume create "%s"' %dest_dir + + + + options = ['-rt','--stats','--delete-excluded','--numeric-ids','--delete-after'] + if self.logger.level: + options.append('-P') + + if self.dry_run: + options.append('-d') + + if self.overload_args <> None: + options.append(self.overload_args) + elif not "cygdrive" in self.remote_dir: + # we don't preserve owner, group, links, hardlinks, perms for windows/cygwin as it is not reliable nor useful + options.append('-lpgoD') + + # the protect-args option is not available in all rsync version + if not self.protect_args.lower() in ('false','no','0'): + options.append('--protect-args') + + if self.compression.lower() in ('true','yes','1'): + options.append('-z') + + if self.compressionlevel: + options.append('--compress-level=%s' % self.compressionlevel) + + if self.bwlimit: + options.append('--bwlimit %s' % self.bwlimit) + + latest = self.get_latest_backup(self.backup_start_date) + #remove link-dest replace by btrfs + #if latest: + # options.extend(['--link-dest="%s"' % os.path.join('..',b,'') for b in latest]) + + def strip_quotes(s): + if s[0] == '"': + s = s[1:] + if s[-1] == '"': + s = s[:-1] + return s + + # Add excludes + if "--exclude" in self.exclude_list: + # old settings with exclude_list=--exclude toto --exclude=titi + excludes = [strip_quotes(s).strip() for s in self.exclude_list.replace('--exclude=','').replace('--exclude ','').split()] + else: + try: + # newsettings with exclude_list='too','titi', parsed as a str python list content + excludes = eval('[%s]' % self.exclude_list) + except Exception,e: + raise Exception('Error reading exclude list : value %s, eval error %s (don\'t forget quotes and comma...)' % (self.exclude_list,e)) + options.extend(['--exclude="%s"' % x for x in excludes]) + + if (self.rsync_module and not self.password_file): + raise Exception('You must specify a password file if you specify a rsync module') + + if (not self.rsync_module and not self.private_key): + raise Exception('If you don''t use SSH, you must specify a rsync module') + + #rsync_re = re.compile('(?P[^:]*)::(?P[^/]*)/(?P.*)') + #ssh_re = re.compile('((?P.*)@)?(?P[^:]*):(?P/.*)') + + # Add ssh connection params + if self.rsync_module: + # Case of rsync exports + if self.password_file: + options.append('--password-file="%s"' % self.password_file) + backup_source = '%s@%s::%s%s' % (self.remote_user, self.server_name, self.rsync_module, self.remote_dir) + else: + # case of rsync + ssh + ssh_params = ['-o StrictHostKeyChecking=no','-c blowfish'] + if self.private_key: + ssh_params.append('-i %s' % self.private_key) + if self.ssh_port <> 22: + ssh_params.append('-p %i' % self.ssh_port) + options.append('-e "/usr/bin/ssh %s"' % (" ".join(ssh_params))) + backup_source = '%s@%s:%s' % (self.remote_user,self.server_name,self.remote_dir) + + # ensure there is a slash at end + if backup_source[-1] <> '/': + backup_source += '/' + + options_params = " ".join(options) + + cmd = '/usr/bin/rsync %s %s %s 2>&1' % (options_params,backup_source,dest_dir) + self.logger.debug("[%s] rsync : %s",self.backup_name,cmd) + + if not self.dry_run: + self.line = '' + process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) + def ondata(data,context): + if context.verbose: + print data + context.logger.debug(data) + + log = monitor_stdout(process,ondata,self) + + reg_total_files = re.compile('Number of files: (?P\d+)') + reg_transferred_files = re.compile('Number of .*files transferred: (?P\d+)') + for l in log.splitlines(): + line = l.replace(',','') + m = reg_total_files.match(line) + if m: + stats['total_files_count'] += int(m.groupdict()['file']) + m = reg_transferred_files.match(line) + if m: + stats['written_files_count'] += int(m.groupdict()['file']) + if line.startswith('Total file size:'): + stats['total_bytes'] += int(line.split(':')[1].split()[0]) + if line.startswith('Total transferred file size:'): + stats['written_bytes'] += int(line.split(':')[1].split()[0]) + + returncode = process.returncode + ## deal with exit code 24 (file vanished) + if (returncode == 24): + self.logger.warning("[" + self.backup_name + "] Note: some files vanished before transfer") + elif (returncode == 23): + self.logger.warning("[" + self.backup_name + "] unable so set uid on some files") + elif (returncode != 0): + self.logger.error("[" + self.backup_name + "] shell program exited with error code ") + raise Exception("[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd) + else: + print cmd + + #we take a snapshot of last_backup if everything went well + finaldest = os.path.join(self.backup_dir,self.backup_start_date) + self.logger.debug("[%s] snapshoting last_backup directory from %s to %s" ,self.backup_name,dest_dir,finaldest) + if not os.path.isdir(finaldest): + if not self.dry_run: + cmd = "/sbin/btrfs subvolume snapshot %s %s"%(dest_dir,finaldest) + process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) + log = monitor_stdout(process,'',self) + returncode = process.returncode + if (returncode != 0): + self.logger.error("[" + self.backup_name + "] shell program exited with error code: %s"%log) + raise Exception("[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd) + else: + self.logger.info("[" + self.backup_name + "] snapshot directory created %s"%finaldest) + else: + print "btrfs snapshot of %s to %s"%(dest_dir,finaldest) + else: + raise Exception('snapshot directory already exists : %s' %finaldest) + self.logger.debug("[%s] touching datetime of target directory %s" ,self.backup_name,finaldest) + print os.popen('touch "%s"' % finaldest).read() + stats['backup_location'] = finaldest + stats['status']='OK' + stats['log']='ssh+rsync+btrfs backup from %s OK, %d bytes written for %d changed files' % (backup_source,stats['written_bytes'],stats['written_files_count']) + + except BaseException , e: + stats['status']='ERROR' + stats['log']=str(e) + raise + + + finally: + self.remove_lock() + + def get_latest_backup(self,current): + result = [] + filelist = os.listdir(self.backup_dir) + filelist.sort() + filelist.reverse() + full = '' + r_full = re.compile('^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}$') + r_partial = re.compile('^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}.rsync$') + # we take all latest partials younger than the latest full and the latest full + for item in filelist: + if r_partial.match(item) and itemstart: + stop = fileisodate(dir_name) + else: + stop = start + self.logger.info('Registering %s started on %s',dir_name,start) + self.logger.debug(' Disk usage %s','du -sb "%s"' % dir_name) + if not self.dry_run: + size_bytes = int(os.popen('du -sb "%s"' % dir_name).read().split('\t')[0]) + else: + size_bytes = 0 + self.logger.debug(' Size in bytes : %i',size_bytes) + if not self.dry_run: + self.dbstat.add(self.backup_name,self.server_name,'',\ + backup_start=start,backup_end = stop,status='OK',total_bytes=size_bytes,backup_location=dir_name) + else: + self.logger.info('Skipping %s, already registered',dir_name) + + + def is_pid_still_running(self,lockfile): + f = open(lockfile) + lines = f.readlines() + f.close() + if len(lines)==0 : + self.logger.info("[" + self.backup_name + "] empty lock file, removing...") + return False + + for line in lines: + if line.startswith('pid='): + pid = line.split('=')[1].strip() + if os.path.exists("/proc/" + pid): + self.logger.info("[" + self.backup_name + "] process still there") + return True + else: + self.logger.info("[" + self.backup_name + "] process not there anymore remove lock") + return False + else: + self.logger.info("[" + self.backup_name + "] incorrrect lock file : no pid line") + return False + + + def set_lock(self): + self.logger.debug("[" + self.backup_name + "] setting lock") + + #TODO: improve for race condition + #TODO: also check if process is really there + if os.path.isfile(self.backup_dir + '/lock'): + self.logger.debug("[" + self.backup_name + "] File " + self.backup_dir + '/lock already exist') + if self.is_pid_still_running(self.backup_dir + '/lock')==False: + self.logger.info("[" + self.backup_name + "] removing lock file " + self.backup_dir + '/lock') + os.unlink(self.backup_dir + '/lock') + else: + return False + + lockfile = open(self.backup_dir + '/lock',"w") + # Write all the lines at once: + lockfile.write('pid='+str(os.getpid())) + lockfile.write('\nbackup_time=' + self.backup_start_date) + lockfile.close() + return True + + def remove_lock(self): + self.logger.debug("[%s] removing lock",self.backup_name ) + os.unlink(self.backup_dir + '/lock') + +class backup_rsync__btrfs_ssh(backup_rsync_btrfs): + """Backup a directory on remote server with rsync,ssh and btrfs protocol (requires rsync software on remote host)""" + type = 'rsync+btrfs+ssh' + required_params = backup_generic.required_params + ['remote_user','remote_dir','private_key'] + optional_params = backup_generic.optional_params + ['compression','bwlimit','ssh_port','exclude_list','protect_args','overload_args'] + + +register_driver(backup_rsync_btrfs) +register_driver(backup_rsync__btrfs_ssh) + +if __name__=='__main__': + logger = logging.getLogger('tisbackup') + logger.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger.addHandler(handler) + + cp = ConfigParser() + cp.read('/opt/tisbackup/configtest.ini') + dbstat = BackupStat('/backup/data/log/tisbackup.sqlite') + b = backup_rsync('htouvet','/backup/data/htouvet',dbstat) + b.read_config(cp) + b.process_backup() + print b.checknagios() +