k3nny/TISbackup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use huey to spool exports backups

Browse Source
This commit is contained in:
ssamson-tis 2015-07-06 18:01:49 +02:00
parent ba022d64d3
commit 4c188d8f94
2 changed files with 88 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
templates/export_backup.html
View File

@ -1,18 +1,25 @@
{% extends "layout.html" %} {% extends "layout.html" %}
{% block content %} {% block content %}
<script src="/static/js/bootbox.js"></script>
<h2 class="title">Hard drive verifications:</h2> <h2 class="title">Hard drive verifications:</h2>
{% with messages = get_flashed_messages() %} {% with messages = get_flashed_messages() %}
{% if messages %} {% if messages %}
{% for message in messages %} {% for message in messages %}
<h4>{{ message }} <img src="/static/images/check.png" width="15" height="15"/></h4> <div class="alert alert-success fade in">
<a href="#" class="close" data-dismiss="alert">&times;</a>
<strong>Success!</strong> {{ message }}
</div>
{% endfor %} {% endfor %}
{% endif %} {% endif %}
{% endwith %} {% endwith %}
{% if error %} {% if error %}
<p> <p>
<div class=error><img class="center-img" src='/static/images/important.gif' title="Notice" width="24" height="24"/> <strong>Error:</strong> {{ error }}</div> <div class="alert alert-danger fade in"><strong>Error</strong>: {{ error }}</div>
<div class=info><img class="center-img" src='/static/images/info.gif' title="Notice" width="24" height="24"/> <strong>Notice:</strong> {{ info }}</div> <div class="alert alert-warning"><strong>Notice</strong>: {{ info }}</div>
<h4>Also, you can contact your <a href="mailto:{{ email }}?Subject=TISBACKUP%20Export"> System Administrator</a> for more details </h4> <h4>Also, you can contact your <a href="mailto:{{ email }}?Subject=TISBACKUP%20Export"> System Administrator</a> for more details </h4>
</p> </p>
{% elif not start %} {% elif not start %}
@ -39,16 +46,14 @@
}); });
</script> </script>
<form id="backup" action='/export_backup'> <form id="backup" action='/export_backup'>
<p> Select backups to save : <br/> <br/> <p> Select backups to save : <br/>
Select all <input type="checkbox" id="selectall" checked="true"><br /><br/> <div class="checkbox"><label><input type="checkbox" class="checkbox1" id="selectall" checked>Select all</label></div>
{% for entry in sections|sort %} {% for entry in sections|sort %}
<div class="col-xs-6 col-md-4"> <div class="col-xs-6 col-md-4">
<div class="form-group"> <div class="form-group">
<div class="checkbox"><label><input type="checkbox" class="checkbox1" value="{{entry}}" checked>{{entry}}</label></div> <div class="checkbox"><label><input type="checkbox" name="sections" class="checkbox1" value="{{entry}}" checked>{{entry}}</label></div>
</div>
</div> </div>
</div>
<input type="checkbox" name="sections" class="checkbox1" value="{{entry}}" checked="true">{{entry}} <br />
{% endfor %} {% endfor %}
<p> <p>
<input type="hidden" name="start" value="true" /> <input type="hidden" name="start" value="true" />
@ -57,7 +62,7 @@
{% else %} {% else %}
<h2 class="title">Backups is running: </h2> <h2 class="title">Backups is running: </h2>
<table id="table-design"> <table id="table" class='table'>
<thead> <thead>
<th>Server</th> <th>Server</th>
<th>Backup</th> <th>Backup</th>
@ -75,9 +80,9 @@ var refresh = 10;
var done = false; var done = false;
function status(){ function status(){
$.getJSON("/status.json", function(data) { $.getJSON("/status.json", function(data) {
$("#table-design tbody").remove(); $("#table tbody").remove();
$.each(data.data, function(key,val){ $.each(data.data, function(key,val){
$('#table-design').append('<tr>'); $('#table').append('<tr>');
$('tbody').append('<td>'+val.server_name+'</td>'); $('tbody').append('<td>'+val.server_name+'</td>');
$('tbody').append('<td>'+val.backup_name+'</td>'); $('tbody').append('<td>'+val.backup_name+'</td>');
if(val.status != 'OK'){ if(val.status != 'OK'){

119
tisbackup_gui.py
View File

@ -26,7 +26,7 @@ from shutil import *
from iniparse import ConfigParser from iniparse import ConfigParser
from libtisbackup.common import * from libtisbackup.common import *
import time import time
from flask import request, Flask, session, g, redirect, url_for, abort, render_template, flash, jsonify, Response from flask import request, Flask, session, g, appcontext_pushed, redirect, url_for, abort, render_template, flash, jsonify, Response
from urlparse import urlparse from urlparse import urlparse
import json import json
import glob import glob
@ -39,15 +39,13 @@ import logging
import re import re
cp = ConfigParser()
cp.read("/etc/tis/tisbackup_gui.ini")
CONFIG = cp.get('uwsgi','config_tisbackup').split(",")
SECTIONS = cp.get('uwsgi','sections')
ADMIN_EMAIL = cp.get('uwsgi','ADMIN_EMAIL')
"""CONFIG = uwsgi.opt['config_tisbackup'].split(",")
SECTIONS = uwsgi.opt['sections']
ADMIN_EMAIL = uwsgi.opt.get('ADMIN_EMAIL',uwsgi.opt.get('admin_email'))
spooler = uwsgi.opt['spooler']
"""
CONFIG="/home/homes/ssamson/projects/tisbackup/configtest.ini".split(",")
ADMIN_EMAIL="toot@test.fr"
SECTIONS=''
tisbackup_config_file= CONFIG[0] tisbackup_config_file= CONFIG[0]
config_number=0 config_number=0
@ -63,9 +61,11 @@ app = Flask(__name__)
app.secret_key = 'fsiqefiuqsefARZ4Zfesfe34234dfzefzfe' app.secret_key = 'fsiqefiuqsefARZ4Zfesfe34234dfzefzfe'
app.config['PROPAGATE_EXCEPTIONS'] = True app.config['PROPAGATE_EXCEPTIONS'] = True
queue = SqliteQueue('tisbackups',os.path.join(tisbackup_root_dir,"tasks.sqlite")) tasks_db = os.path.join(tisbackup_root_dir,"tasks.sqlite")
result_store = SqliteDataStore('tisbackups',os.path.join(tisbackup_root_dir,"tasks.sqlite")) queue = SqliteQueue('tisbackups',tasks_db)
huey = Huey(queue,result_store) result_store = SqliteDataStore('tisbackups',tasks_db)
huey = Huey(queue,result_store,always_eager=False)
def read_config(): def read_config():
config_file = CONFIG[config_number] config_file = CONFIG[config_number]
@ -172,9 +172,8 @@ def check_usb_disk():
usb_disk_list += [ name ] usb_disk_list += [ name ]
if len(usb_disk_list) == 0: if len(usb_disk_list) == 0:
raise_error("cannot find external usb disk", "You should plug the usb hard drive into the server") raise_error("Cannot find any external usb disk", "You should plug the usb hard drive into the server")
return "" return ""
print usb_disk_list print usb_disk_list
usb_partition_list = [] usb_partition_list = []
@ -187,8 +186,13 @@ def check_usb_disk():
if '/devices/pci' in output: if '/devices/pci' in output:
#flash("partition found: %s1" % usb_disk) #flash("partition found: %s1" % usb_disk)
usb_partition_list.append(usb_disk + "1") usb_partition_list.append(usb_disk + "1")
print usb_partition_list print usb_partition_list
if len(usb_partition_list) ==0:
raise_error("The dribe %s has no partition" % (usb_disk_list[0] ), "You should initialize the usb drive and format an ext4 partition with TISBACKUP label")
return ""
tisbackup_partition_list = [] tisbackup_partition_list = []
for usb_partition in usb_partition_list: for usb_partition in usb_partition_list:
if "tisbackup" in os.popen("/sbin/dumpe2fs -h %s 2>&1 |/bin/grep 'volume name'" % usb_partition).read().lower(): if "tisbackup" in os.popen("/sbin/dumpe2fs -h %s 2>&1 |/bin/grep 'volume name'" % usb_partition).read().lower():
@ -198,7 +202,7 @@ def check_usb_disk():
print tisbackup_partition_list print tisbackup_partition_list
if len(tisbackup_partition_list) ==0: if len(tisbackup_partition_list) ==0:
raise_error("No tisbackup partition exist on external disk", "You should initialize the usb drive and set TISBACKUP label on ext4 partition") raise_error("No tisbackup partition exist on disk %s" % (usb_disk_list[0] ), "You should initialize the usb drive and format an ext4 partition with TISBACKUP label")
return "" return ""
if len(tisbackup_partition_list) > 1: if len(tisbackup_partition_list) > 1:
@ -209,44 +213,53 @@ def check_usb_disk():
return tisbackup_partition_list[0] return tisbackup_partition_list[0]
def check_already_mount(partition_name):
with open('/proc/mounts') as f:
mount_point = ""
for line in f.readlines():
if line.startswith(partition_name):
mount_point = line.split(' ')[1]
run_command("/bin/umount %s" % mount_point)
os.rmdir(mount_point)
def check_mount_disk(partition_name, refresh): def run_command(cmd, info=""):
flash("Executing: %s"% cmd)
flash("check if disk is mounted") from subprocess import CalledProcessError, check_output
result =""
try:
already_mounted=False result = check_output(cmd, stderr=subprocess.STDOUT,shell=True)
f = open('/proc/mounts') except CalledProcessError as e:
lines = f.readlines() raise_error(result,info)
mount_point = "" return result
for line in lines:
if line.startswith(partition_name):
already_mounted = True
mount_point = line.split(' ')[1]
break
f.close()
def check_mount_disk(partition_name, refresh):
if not refresh: if not refresh:
if already_mounted == True: check_already_mount(partition_name)
os.system("/bin/umount %s" % mount_point)
os.rmdir(mount_point)
mount_point = "/mnt/" + str(time.time()) mount_point = "/mnt/TISBACKUP-" +str(time.time())
os.mkdir(mount_point) os.mkdir(mount_point)
flash("must mount " + partition_name ) flash("must mount " + partition_name )
cmd = "mount %s %s" % (partition_name, mount_point) cmd = "mount %s %s" % (partition_name, mount_point)
flash("executing : " + cmd) if run_command(cmd,"You should manualy mount the usb drive") != "":
result = os.popen(cmd+" 2>&1").read() flash("Remove directory: %s" % mount_point)
if len(result) > 1: os.rmdir(mount_point)
raise_error(result, "You should manualy mount the usb drive") return ""
return ""
return mount_point return mount_point
@app.route('/status.json') @app.route('/status.json')
def export_backup_status(): def export_backup_status():
exports = dbstat.query('select * from stats where TYPE="EXPORT" and backup_start>="%s"' % mindate) exports = dbstat.query('select * from stats where TYPE="EXPORT" and backup_start>="%s"' % mindate)
return jsonify(data=exports,finish=True) return jsonify(data=exports,finish=(not runnings_backups()))
def runnings_backups():
conn = sqlite3.connect(tasks_db)
c = conn.cursor()
c.execute('SELECT count(*) FROM huey_queue_tisbackups')
count = c.fetchall()[0][0]
conn.close()
return ( int(count) > 0 )
@app.route('/backups.json') @app.route('/backups.json')
def last_backup_json(): def last_backup_json():
@ -264,9 +277,11 @@ def last_backup():
@app.route('/export_backup') @app.route('/export_backup')
def export_backup(): def export_backup():
raise_error("", "") raise_error("", "")
backup_dict = read_config() backup_dict = read_config()
sections = [] sections = []
backup_sections = []
for backup_types in backup_dict: for backup_types in backup_dict:
if backup_types == "null_list": if backup_types == "null_list":
continue continue
@ -274,11 +289,12 @@ def export_backup():
if section.count > 0: if section.count > 0:
sections.append(section[1]) sections.append(section[1])
noJobs = True noJobs = (not runnings_backups())
if "start" in request.args.keys() or not noJobs: if "start" in request.args.keys() or not noJobs:
start=True start=True
if "sections" in request.args.keys(): if "sections" in request.args.keys():
backup_sections = request.args.getlist('sections') backup_sections = request.args.getlist('sections')
else: else:
start=False start=False
@ -294,8 +310,11 @@ def export_backup():
global mindate global mindate
mindate = datetime2isodate(datetime.datetime.now()) mindate = datetime2isodate(datetime.datetime.now())
if not error and start: if not error and start:
run_export_backup.spool(base=backup_base_dir, config_file=tisbackup_config_file, mount_point=mount_point, backup_sections=",".join([str(x) for x in backup_sections])) run_export_backup(base=backup_base_dir, config_file=tisbackup_config_file, mount_point=mount_point, backup_sections=",".join([str(x) for x in backup_sections]))
return render_template("export_backup.html", error=error, start=start, info=info, email=ADMIN_EMAIL, sections=sections) return render_template("export_backup.html", error=error, start=start, info=info, email=ADMIN_EMAIL, sections=sections)
@ -306,7 +325,7 @@ def raise_error(strError, strInfo):
@huey.task() @huey.task()
def run_export_backup(args): def run_export_backup(base, config_file, mount_point, backup_sections):
#Log #Log
logger = logging.getLogger('tisbackup') logger = logging.getLogger('tisbackup')
logger.setLevel(logging.INFO) logger.setLevel(logging.INFO)
@ -318,18 +337,18 @@ def run_export_backup(args):
# Main # Main
logger.info("Running export....") logger.info("Running export....")
if args['backup_sections']: if backup_sections:
backup_sections = args['backup_sections'].split(",") backup_sections = backup_sections.split(",")
else: else:
backup_sections = [] backup_sections = []
backup = tis_backup(dry_run=False,verbose=True,backup_base_dir=base)
backup = tis_backup(dry_run=False,verbose=True,backup_base_dir=args['base']) backup.read_ini_file(config_file)
backup.read_ini_file(args['config_file']) mount_point = mount_point
mount_point = args['mount_point']
backup.export_backups(backup_sections,mount_point) backup.export_backups(backup_sections,mount_point)
os.system("/bin/umount %s" % mount_point) os.system("/bin/umount %s" % mount_point)
os.rmdir(mount_point) os.rmdir(mount_point)
if __name__ == "__main__": if __name__ == "__main__":
@ -337,4 +356,4 @@ if __name__ == "__main__":
from os import environ from os import environ
if 'WINGDB_ACTIVE' in environ: if 'WINGDB_ACTIVE' in environ:
app.debug = False app.debug = False
app.run(use_reloader=True) app.run(use_reloader=True)