use huey to spool exports backups
This commit is contained in:
ssamson-tis
parent
ba022d64d3
commit
4c188d8f94
@ -1,18 +1,25 @@
|
||||
|
||||
{% extends "layout.html" %}
|
||||
{% block content %}
|
||||
<script src="/static/js/bootbox.js"></script>
|
||||
|
||||
<h2 class="title">Hard drive verifications:</h2>
|
||||
{% with messages = get_flashed_messages() %}
|
||||
{% if messages %}
|
||||
{% for message in messages %}
|
||||
<h4>{{ message }} <img src="/static/images/check.png" width="15" height="15"/></h4>
|
||||
<div class="alert alert-success fade in">
|
||||
<a href="#" class="close" data-dismiss="alert">×</a>
|
||||
<strong>Success!</strong> {{ message }}
|
||||
</div>
|
||||
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endwith %}
|
||||
{% if error %}
|
||||
<p>
|
||||
<div class=error><img class="center-img" src='/static/images/important.gif' title="Notice" width="24" height="24"/> <strong>Error:</strong> {{ error }}</div>
|
||||
<div class=info><img class="center-img" src='/static/images/info.gif' title="Notice" width="24" height="24"/> <strong>Notice:</strong> {{ info }}</div>
|
||||
<div class="alert alert-danger fade in"><strong>Error</strong>: {{ error }}</div>
|
||||
<div class="alert alert-warning"><strong>Notice</strong>: {{ info }}</div>
|
||||
|
||||
<h4>Also, you can contact your <a href="mailto:{{ email }}?Subject=TISBACKUP%20Export"> System Administrator</a> for more details </h4>
|
||||
</p>
|
||||
{% elif not start %}
|
||||
@ -39,16 +46,14 @@
|
||||
});
|
||||
</script>
|
||||
<form id="backup" action='/export_backup'>
|
||||
<p> Select backups to save : <br/> <br/>
|
||||
Select all <input type="checkbox" id="selectall" checked="true"><br /><br/>
|
||||
<p> Select backups to save : <br/>
|
||||
<div class="checkbox"><label><input type="checkbox" class="checkbox1" id="selectall" checked>Select all</label></div>
|
||||
{% for entry in sections|sort %}
|
||||
<div class="col-xs-6 col-md-4">
|
||||
<div class="form-group">
|
||||
<div class="checkbox"><label><input type="checkbox" class="checkbox1" value="{{entry}}" checked>{{entry}}</label></div>
|
||||
|
||||
<div class="checkbox"><label><input type="checkbox" name="sections" class="checkbox1" value="{{entry}}" checked>{{entry}}</label></div>
|
||||
</div>
|
||||
</div>
|
||||
<input type="checkbox" name="sections" class="checkbox1" value="{{entry}}" checked="true">{{entry}} <br />
|
||||
{% endfor %}
|
||||
<p>
|
||||
<input type="hidden" name="start" value="true" />
|
||||
@ -57,7 +62,7 @@
|
||||
|
||||
{% else %}
|
||||
<h2 class="title">Backups is running: </h2>
|
||||
<table id="table-design">
|
||||
<table id="table" class='table'>
|
||||
<thead>
|
||||
<th>Server</th>
|
||||
<th>Backup</th>
|
||||
@ -75,9 +80,9 @@ var refresh = 10;
|
||||
var done = false;
|
||||
function status(){
|
||||
$.getJSON("/status.json", function(data) {
|
||||
$("#table-design tbody").remove();
|
||||
$("#table tbody").remove();
|
||||
$.each(data.data, function(key,val){
|
||||
$('#table-design').append('<tr>');
|
||||
$('#table').append('<tr>');
|
||||
$('tbody').append('<td>'+val.server_name+'</td>');
|
||||
$('tbody').append('<td>'+val.backup_name+'</td>');
|
||||
if(val.status != 'OK'){
|
||||
|
||||
113
tisbackup_gui.py
113
tisbackup_gui.py
@ -26,7 +26,7 @@ from shutil import *
|
||||
from iniparse import ConfigParser
|
||||
from libtisbackup.common import *
|
||||
import time
|
||||
from flask import request, Flask, session, g, redirect, url_for, abort, render_template, flash, jsonify, Response
|
||||
from flask import request, Flask, session, g, appcontext_pushed, redirect, url_for, abort, render_template, flash, jsonify, Response
|
||||
from urlparse import urlparse
|
||||
import json
|
||||
import glob
|
||||
@ -39,15 +39,13 @@ import logging
|
||||
import re
|
||||
|
||||
|
||||
cp = ConfigParser()
|
||||
cp.read("/etc/tis/tisbackup_gui.ini")
|
||||
|
||||
CONFIG = cp.get('uwsgi','config_tisbackup').split(",")
|
||||
SECTIONS = cp.get('uwsgi','sections')
|
||||
ADMIN_EMAIL = cp.get('uwsgi','ADMIN_EMAIL')
|
||||
|
||||
"""CONFIG = uwsgi.opt['config_tisbackup'].split(",")
|
||||
SECTIONS = uwsgi.opt['sections']
|
||||
ADMIN_EMAIL = uwsgi.opt.get('ADMIN_EMAIL',uwsgi.opt.get('admin_email'))
|
||||
spooler = uwsgi.opt['spooler']
|
||||
"""
|
||||
CONFIG="/home/homes/ssamson/projects/tisbackup/configtest.ini".split(",")
|
||||
ADMIN_EMAIL="toot@test.fr"
|
||||
SECTIONS=''
|
||||
tisbackup_config_file= CONFIG[0]
|
||||
config_number=0
|
||||
|
||||
@ -63,9 +61,11 @@ app = Flask(__name__)
|
||||
app.secret_key = 'fsiqefiuqsefARZ4Zfesfe34234dfzefzfe'
|
||||
app.config['PROPAGATE_EXCEPTIONS'] = True
|
||||
|
||||
queue = SqliteQueue('tisbackups',os.path.join(tisbackup_root_dir,"tasks.sqlite"))
|
||||
result_store = SqliteDataStore('tisbackups',os.path.join(tisbackup_root_dir,"tasks.sqlite"))
|
||||
huey = Huey(queue,result_store)
|
||||
tasks_db = os.path.join(tisbackup_root_dir,"tasks.sqlite")
|
||||
queue = SqliteQueue('tisbackups',tasks_db)
|
||||
result_store = SqliteDataStore('tisbackups',tasks_db)
|
||||
huey = Huey(queue,result_store,always_eager=False)
|
||||
|
||||
|
||||
def read_config():
|
||||
config_file = CONFIG[config_number]
|
||||
@ -172,9 +172,8 @@ def check_usb_disk():
|
||||
usb_disk_list += [ name ]
|
||||
|
||||
if len(usb_disk_list) == 0:
|
||||
raise_error("cannot find external usb disk", "You should plug the usb hard drive into the server")
|
||||
raise_error("Cannot find any external usb disk", "You should plug the usb hard drive into the server")
|
||||
return ""
|
||||
|
||||
print usb_disk_list
|
||||
|
||||
usb_partition_list = []
|
||||
@ -187,8 +186,13 @@ def check_usb_disk():
|
||||
if '/devices/pci' in output:
|
||||
#flash("partition found: %s1" % usb_disk)
|
||||
usb_partition_list.append(usb_disk + "1")
|
||||
|
||||
print usb_partition_list
|
||||
|
||||
if len(usb_partition_list) ==0:
|
||||
raise_error("The dribe %s has no partition" % (usb_disk_list[0] ), "You should initialize the usb drive and format an ext4 partition with TISBACKUP label")
|
||||
return ""
|
||||
|
||||
tisbackup_partition_list = []
|
||||
for usb_partition in usb_partition_list:
|
||||
if "tisbackup" in os.popen("/sbin/dumpe2fs -h %s 2>&1 |/bin/grep 'volume name'" % usb_partition).read().lower():
|
||||
@ -198,7 +202,7 @@ def check_usb_disk():
|
||||
print tisbackup_partition_list
|
||||
|
||||
if len(tisbackup_partition_list) ==0:
|
||||
raise_error("No tisbackup partition exist on external disk", "You should initialize the usb drive and set TISBACKUP label on ext4 partition")
|
||||
raise_error("No tisbackup partition exist on disk %s" % (usb_disk_list[0] ), "You should initialize the usb drive and format an ext4 partition with TISBACKUP label")
|
||||
return ""
|
||||
|
||||
if len(tisbackup_partition_list) > 1:
|
||||
@ -209,36 +213,37 @@ def check_usb_disk():
|
||||
return tisbackup_partition_list[0]
|
||||
|
||||
|
||||
|
||||
def check_mount_disk(partition_name, refresh):
|
||||
|
||||
flash("check if disk is mounted")
|
||||
|
||||
|
||||
already_mounted=False
|
||||
f = open('/proc/mounts')
|
||||
lines = f.readlines()
|
||||
def check_already_mount(partition_name):
|
||||
with open('/proc/mounts') as f:
|
||||
mount_point = ""
|
||||
for line in lines:
|
||||
for line in f.readlines():
|
||||
if line.startswith(partition_name):
|
||||
already_mounted = True
|
||||
mount_point = line.split(' ')[1]
|
||||
break
|
||||
f.close()
|
||||
|
||||
if not refresh:
|
||||
if already_mounted == True:
|
||||
os.system("/bin/umount %s" % mount_point)
|
||||
run_command("/bin/umount %s" % mount_point)
|
||||
os.rmdir(mount_point)
|
||||
|
||||
mount_point = "/mnt/" + str(time.time())
|
||||
def run_command(cmd, info=""):
|
||||
flash("Executing: %s"% cmd)
|
||||
from subprocess import CalledProcessError, check_output
|
||||
result =""
|
||||
try:
|
||||
result = check_output(cmd, stderr=subprocess.STDOUT,shell=True)
|
||||
except CalledProcessError as e:
|
||||
raise_error(result,info)
|
||||
return result
|
||||
|
||||
def check_mount_disk(partition_name, refresh):
|
||||
if not refresh:
|
||||
check_already_mount(partition_name)
|
||||
|
||||
|
||||
mount_point = "/mnt/TISBACKUP-" +str(time.time())
|
||||
os.mkdir(mount_point)
|
||||
flash("must mount " + partition_name )
|
||||
cmd = "mount %s %s" % (partition_name, mount_point)
|
||||
flash("executing : " + cmd)
|
||||
result = os.popen(cmd+" 2>&1").read()
|
||||
if len(result) > 1:
|
||||
raise_error(result, "You should manualy mount the usb drive")
|
||||
if run_command(cmd,"You should manualy mount the usb drive") != "":
|
||||
flash("Remove directory: %s" % mount_point)
|
||||
os.rmdir(mount_point)
|
||||
return ""
|
||||
|
||||
return mount_point
|
||||
@ -246,7 +251,15 @@ def check_mount_disk(partition_name, refresh):
|
||||
@app.route('/status.json')
|
||||
def export_backup_status():
|
||||
exports = dbstat.query('select * from stats where TYPE="EXPORT" and backup_start>="%s"' % mindate)
|
||||
return jsonify(data=exports,finish=True)
|
||||
return jsonify(data=exports,finish=(not runnings_backups()))
|
||||
|
||||
def runnings_backups():
|
||||
conn = sqlite3.connect(tasks_db)
|
||||
c = conn.cursor()
|
||||
c.execute('SELECT count(*) FROM huey_queue_tisbackups')
|
||||
count = c.fetchall()[0][0]
|
||||
conn.close()
|
||||
return ( int(count) > 0 )
|
||||
|
||||
@app.route('/backups.json')
|
||||
def last_backup_json():
|
||||
@ -264,9 +277,11 @@ def last_backup():
|
||||
|
||||
@app.route('/export_backup')
|
||||
def export_backup():
|
||||
|
||||
raise_error("", "")
|
||||
backup_dict = read_config()
|
||||
sections = []
|
||||
backup_sections = []
|
||||
for backup_types in backup_dict:
|
||||
if backup_types == "null_list":
|
||||
continue
|
||||
@ -274,12 +289,13 @@ def export_backup():
|
||||
if section.count > 0:
|
||||
sections.append(section[1])
|
||||
|
||||
noJobs = True
|
||||
noJobs = (not runnings_backups())
|
||||
if "start" in request.args.keys() or not noJobs:
|
||||
start=True
|
||||
if "sections" in request.args.keys():
|
||||
backup_sections = request.args.getlist('sections')
|
||||
|
||||
|
||||
else:
|
||||
start=False
|
||||
cp.read(tisbackup_config_file)
|
||||
@ -294,7 +310,10 @@ def export_backup():
|
||||
global mindate
|
||||
mindate = datetime2isodate(datetime.datetime.now())
|
||||
if not error and start:
|
||||
run_export_backup.spool(base=backup_base_dir, config_file=tisbackup_config_file, mount_point=mount_point, backup_sections=",".join([str(x) for x in backup_sections]))
|
||||
run_export_backup(base=backup_base_dir, config_file=tisbackup_config_file, mount_point=mount_point, backup_sections=",".join([str(x) for x in backup_sections]))
|
||||
|
||||
|
||||
|
||||
|
||||
return render_template("export_backup.html", error=error, start=start, info=info, email=ADMIN_EMAIL, sections=sections)
|
||||
|
||||
@ -306,7 +325,7 @@ def raise_error(strError, strInfo):
|
||||
|
||||
|
||||
@huey.task()
|
||||
def run_export_backup(args):
|
||||
def run_export_backup(base, config_file, mount_point, backup_sections):
|
||||
#Log
|
||||
logger = logging.getLogger('tisbackup')
|
||||
logger.setLevel(logging.INFO)
|
||||
@ -318,20 +337,20 @@ def run_export_backup(args):
|
||||
# Main
|
||||
logger.info("Running export....")
|
||||
|
||||
if args['backup_sections']:
|
||||
backup_sections = args['backup_sections'].split(",")
|
||||
if backup_sections:
|
||||
backup_sections = backup_sections.split(",")
|
||||
else:
|
||||
backup_sections = []
|
||||
|
||||
backup = tis_backup(dry_run=False,verbose=True,backup_base_dir=args['base'])
|
||||
backup.read_ini_file(args['config_file'])
|
||||
mount_point = args['mount_point']
|
||||
backup = tis_backup(dry_run=False,verbose=True,backup_base_dir=base)
|
||||
backup.read_ini_file(config_file)
|
||||
mount_point = mount_point
|
||||
backup.export_backups(backup_sections,mount_point)
|
||||
|
||||
os.system("/bin/umount %s" % mount_point)
|
||||
os.rmdir(mount_point)
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
read_config()
|
||||
from os import environ
|
||||
|
||||
Loading…
Reference in New Issue
Block a user