168 lines
7.6 KiB
Python
168 lines
7.6 KiB
Python
#!/usr/bin/python
|
|
# -*- coding: utf-8 -*-
|
|
# -----------------------------------------------------------------------
|
|
# This file is part of TISBackup
|
|
#
|
|
# TISBackup is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# TISBackup is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with TISBackup. If not, see <http://www.gnu.org/licenses/>.
|
|
#
|
|
# -----------------------------------------------------------------------
|
|
import sys
|
|
try:
|
|
sys.stderr = open('/dev/null') # Silence silly warnings from paramiko
|
|
import paramiko
|
|
except ImportError,e:
|
|
print "Error : can not load paramiko library %s" % e
|
|
raise
|
|
|
|
sys.stderr = sys.__stderr__
|
|
|
|
import datetime
|
|
import base64
|
|
import os
|
|
import logging
|
|
import re
|
|
from common import *
|
|
|
|
class backup_pgsql(backup_generic):
|
|
"""Backup a postgresql database as gzipped sql file through ssh"""
|
|
type = 'pgsql+ssh'
|
|
required_params = backup_generic.required_params + ['private_key']
|
|
optional_params = backup_generic.optional_params + ['db_name']
|
|
|
|
db_name=''
|
|
|
|
def do_backup(self,stats):
|
|
self.dest_dir = os.path.join(self.backup_dir,self.backup_start_date)
|
|
|
|
if not os.path.isdir(self.dest_dir):
|
|
if not self.dry_run:
|
|
os.makedirs(self.dest_dir)
|
|
else:
|
|
print 'mkdir "%s"' % self.dest_dir
|
|
else:
|
|
raise Exception('backup destination directory already exists : %s' % self.dest_dir)
|
|
|
|
try:
|
|
mykey = paramiko.RSAKey.from_private_key_file(self.private_key)
|
|
except paramiko.SSHException:
|
|
mykey = paramiko.DSSKey.from_private_key_file(self.private_key)
|
|
|
|
self.logger.debug('[%s] Trying to connect to "%s" with username root and key "%s"',self.backup_name,self.server_name,self.private_key)
|
|
self.ssh = paramiko.SSHClient()
|
|
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
self.ssh.connect(self.server_name,username='root',pkey = mykey,port=self.ssh_port)
|
|
|
|
|
|
if self.db_name:
|
|
stats['log']= "Successfully backup processed to the following database :"
|
|
self.do_pgsqldump(stats)
|
|
else:
|
|
stats['log']= "Successfully backuping processed to the following databases :"
|
|
stats['status']='List'
|
|
cmd = "su - postgres -c 'psql -l -t' | cut -d'|' -f1 | sed -e 's/ //g' -e '/^$/d'"
|
|
self.logger.debug('[%s] List databases: %s',self.backup_name,cmd)
|
|
(error_code,output) = ssh_exec(cmd,ssh=self.ssh)
|
|
self.logger.debug("[%s] Output of %s :\n%s",self.backup_name,cmd,output)
|
|
if error_code:
|
|
raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd))
|
|
databases = output.split('\n')
|
|
for database in databases:
|
|
if database.rstrip() not in ("", "template0", "template1"):
|
|
self.db_name = database.rstrip()
|
|
self.do_pgsqldump(stats)
|
|
|
|
|
|
stats['status']='OK'
|
|
|
|
|
|
def do_pgsqldump(self,stats):
|
|
t = datetime.datetime.now()
|
|
backup_start_date = t.strftime('%Y%m%d-%Hh%Mm%S')
|
|
# dump db
|
|
cmd = " su - postgres -c 'pg_dump "+ self.db_name + ' > /tmp/' + self.db_name + '-' + backup_start_date + ".sql '"
|
|
self.logger.debug('[%s] %s ',self.backup_name,cmd)
|
|
if not self.dry_run:
|
|
(error_code,output) = ssh_exec(cmd,ssh=self.ssh)
|
|
self.logger.debug("[%s] Output of %s :\n%s",self.backup_name,cmd,output)
|
|
if error_code:
|
|
raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd))
|
|
|
|
# zip the file
|
|
cmd = 'gzip /tmp/' + self.db_name + '-' + backup_start_date + '.sql'
|
|
self.logger.debug('[%s] %s ',self.backup_name,cmd)
|
|
if not self.dry_run:
|
|
(error_code,output) = ssh_exec(cmd,ssh=self.ssh)
|
|
self.logger.debug("[%s] Output of %s :\n%s",self.backup_name,cmd,output)
|
|
if error_code:
|
|
raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd))
|
|
|
|
# get the file
|
|
filepath = '/tmp/' + self.db_name + '-' + backup_start_date + '.sql.gz'
|
|
localpath = self.dest_dir + '/' + self.db_name + '-' + backup_start_date + '.sql.gz'
|
|
self.logger.debug('[%s] get the file using sftp from "%s" to "%s" ',self.backup_name,filepath,localpath)
|
|
if not self.dry_run:
|
|
transport = self.ssh.get_transport()
|
|
sftp = paramiko.SFTPClient.from_transport(transport)
|
|
sftp.get(filepath, localpath)
|
|
sftp.close()
|
|
|
|
if not self.dry_run:
|
|
stats['total_files_count']=1 + stats.get('total_files_count', 0)
|
|
stats['written_files_count']=1 + stats.get('written_files_count', 0)
|
|
stats['total_bytes']=os.stat(localpath).st_size + stats.get('total_bytes', 0)
|
|
stats['written_bytes']=os.stat(localpath).st_size + stats.get('written_bytes', 0)
|
|
stats['log'] = '%s "%s"' % (stats['log'] ,self.db_name)
|
|
stats['backup_location'] = self.dest_dir
|
|
|
|
cmd = 'rm -f /tmp/' + self.db_name + '-' + backup_start_date + '.sql.gz'
|
|
self.logger.debug('[%s] %s ',self.backup_name,cmd)
|
|
if not self.dry_run:
|
|
(error_code,output) = ssh_exec(cmd,ssh=self.ssh)
|
|
self.logger.debug("[%s] Output of %s :\n%s",self.backup_name,cmd,output)
|
|
if error_code:
|
|
raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd))
|
|
|
|
|
|
|
|
def register_existingbackups(self):
|
|
"""scan backup dir and insert stats in database"""
|
|
|
|
registered = [b['backup_location'] for b in self.dbstat.query('select distinct backup_location from stats where backup_name=?',(self.backup_name,))]
|
|
|
|
filelist = os.listdir(self.backup_dir)
|
|
filelist.sort()
|
|
p = re.compile('^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}$')
|
|
for item in filelist:
|
|
if p.match(item):
|
|
dir_name = os.path.join(self.backup_dir,item)
|
|
if not dir_name in registered:
|
|
start = datetime.datetime.strptime(item,'%Y%m%d-%Hh%Mm%S').isoformat()
|
|
if fileisodate(dir_name)>start:
|
|
stop = fileisodate(dir_name)
|
|
else:
|
|
stop = start
|
|
self.logger.info('Registering %s started on %s',dir_name,start)
|
|
self.logger.debug(' Disk usage %s','du -sb "%s"' % dir_name)
|
|
if not self.dry_run:
|
|
size_bytes = int(os.popen('du -sb "%s"' % dir_name).read().split('\t')[0])
|
|
else:
|
|
size_bytes = 0
|
|
self.logger.debug(' Size in bytes : %i',size_bytes)
|
|
if not self.dry_run:
|
|
self.dbstat.add(self.backup_name,self.server_name,'',\
|
|
backup_start=start,backup_end = stop,status='OK',total_bytes=size_bytes,backup_location=dir_name)
|
|
else:
|
|
self.logger.info('Skipping %s, already registered',dir_name)
|
|
|
|
register_driver(backup_pgsql) |