k3nny/TISbackup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(drivers): organize backup modules into drivers subfolder
lint / docker (push) Has been cancelled
Details

Browse Source 
- Move all backup_*.py files to libtisbackup/drivers/ subdirectory
- Move XenAPI.py and copy_vm_xcp.py to drivers/ (driver-specific)
- Create drivers/__init__.py with automatic driver imports
- Update tisbackup.py imports to use new structure
- Add pyvmomi>=8.0.0 as mandatory dependency
- Sync requirements.txt with pyproject.toml dependencies
- Add pylint>=3.0.0 and pytest-cov>=6.0.0 to dev dependencies
- Configure pylint and coverage tools in pyproject.toml
- Add conventional commits guidelines to CLAUDE.md
- Enhance .gitignore with comprehensive patterns for Python, IDEs, testing, and secrets
- Update CLAUDE.md documentation with new structure and tooling

Breaking Changes:
- Drivers must now be imported from libtisbackup.drivers instead of libtisbackup
- All backup driver files relocated to drivers/ subdirectory

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
k3nny
2025-10-05 23:54:26 +02:00
parent 38a0d788d4
commit 1cb731cbdb
33 changed files with 2519 additions and 634 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libtisbackup/__init__.py
+74
View File
@@ -15,3 +15,77 @@
# along with TISBackup. If not, see <http://www.gnu.org/licenses/>.
#
# -----------------------------------------------------------------------
"""
TISBackup library - Backup orchestration and driver management.
This package provides a modular backup system with:
- Base driver classes for implementing backup types
- Database management for backup statistics
- SSH and process execution utilities
- Date/time and formatting helpers
"""
# Import from new modular structure
from .base_driver import (
backup_drivers,
backup_generic,
nagiosStateCritical,
nagiosStateOk,
nagiosStateUnknown,
nagiosStateWarning,
register_driver,
)
from .database import BackupStat
from .process import call_external_process, monitor_stdout
from .ssh import load_ssh_private_key, ssh_exec
from .utils import (
check_string,
convert_bytes,
dateof,
datetime2isodate,
fileisodate,
hours_minutes,
html_table,
isodate2datetime,
pp,
splitThousands,
str2bool,
time2display,
)
# Maintain backward compatibility - re-export everything that was in common.py
__all__ = [
# Nagios states
"nagiosStateOk",
"nagiosStateWarning",
"nagiosStateCritical",
"nagiosStateUnknown",
# Driver registry
"backup_drivers",
"register_driver",
# Base classes
"backup_generic",
"BackupStat",
# SSH utilities
"load_ssh_private_key",
"ssh_exec",
# Process utilities
"call_external_process",
"monitor_stdout",
# Date/time utilities
"datetime2isodate",
"isodate2datetime",
"time2display",
"hours_minutes",
"fileisodate",
"dateof",
# Formatting utilities
"splitThousands",
"convert_bytes",
"pp",
"html_table",
# Validation utilities
"check_string",
"str2bool",
]
libtisbackup/common.py → libtisbackup/base_driver.py
+10 -563
View File
@@ -18,588 +18,45 @@
#
# -----------------------------------------------------------------------
"""Base backup driver class and driver registry."""
import datetime
import errno
import logging
import os
import re
import select
import shutil
import sqlite3
import subprocess
import sys
import time
from abc import ABC, abstractmethod
from iniparse import ConfigParser
from .database import BackupStat
from .process import monitor_stdout
from .ssh import load_ssh_private_key
from .utils import dateof, datetime2isodate, isodate2datetime
try:
sys.stderr = open("/dev/null") # Silence silly warnings from paramiko
import paramiko
except ImportError as e:
print(("Error : can not load paramiko library %s" % e))
raise
sys.stderr = sys.__stderr__
# Nagios state constants
nagiosStateOk = 0
nagiosStateWarning = 1
nagiosStateCritical = 2
nagiosStateUnknown = 3
# Global driver registry
backup_drivers = {}
def register_driver(driverclass):
"""Register a backup driver class in the global registry."""
backup_drivers[driverclass.type] = driverclass
def datetime2isodate(adatetime=None):
if not adatetime:
adatetime = datetime.datetime.now()
assert isinstance(adatetime, datetime.datetime)
return adatetime.isoformat()
def isodate2datetime(isodatestr):
# we remove the microseconds part as it is not working for python2.5 strptime
return datetime.datetime.strptime(isodatestr.split(".")[0], "%Y-%m-%dT%H:%M:%S")
def time2display(adatetime):
return adatetime.strftime("%Y-%m-%d %H:%M")
def hours_minutes(hours):
if hours is None:
return None
else:
return "%02i:%02i" % (int(hours), int((hours - int(hours)) * 60.0))
def fileisodate(filename):
return datetime.datetime.fromtimestamp(os.stat(filename).st_mtime).isoformat()
def dateof(adatetime):
return adatetime.replace(hour=0, minute=0, second=0, microsecond=0)
#####################################
# http://code.activestate.com/recipes/498181-add-thousands-separator-commas-to-formatted-number/
# Code from Michael Robellard's comment made 28 Feb 2010
# Modified for leading +, -, space on 1 Mar 2010 by Glenn Linderman
#
# Tail recursion removed and leading garbage handled on March 12 2010, Alessandro Forghieri
def splitThousands(s, tSep=",", dSep="."):
"""Splits a general float on thousands. GIGO on general input"""
if s is None:
return 0
if not isinstance(s, str):
s = str(s)
cnt = 0
numChars = dSep + "0123456789"
ls = len(s)
while cnt < ls and s[cnt] not in numChars:
cnt += 1
lhs = s[0:cnt]
s = s[cnt:]
if dSep == "":
cnt = -1
else:
cnt = s.rfind(dSep)
if cnt > 0:
rhs = dSep + s[cnt + 1 :]
s = s[:cnt]
else:
rhs = ""
splt = ""
while s != "":
splt = s[-3:] + tSep + splt
s = s[:-3]
return lhs + splt[:-1] + rhs
def call_external_process(shell_string):
p = subprocess.call(shell_string, shell=True)
if p != 0:
raise Exception("shell program exited with error code " + str(p), shell_string)
def check_string(test_string):
pattern = r"[^\.A-Za-z0-9\-_]"
if re.search(pattern, test_string):
# Character other then . a-z 0-9 was found
print(("Invalid : %r" % (test_string,)))
def load_ssh_private_key(private_key_path):
"""Load SSH private key with modern algorithm support.
Tries to load the key in order of preference:
1. Ed25519 (most secure, modern)
2. ECDSA (secure, widely supported)
3. RSA (legacy, still secure with sufficient key size)
DSA is not supported as it's deprecated and insecure.
Args:
private_key_path: Path to the private key file
Returns:
paramiko key object
Raises:
paramiko.SSHException: If key cannot be loaded
"""
key_types = [
("Ed25519", paramiko.Ed25519Key),
("ECDSA", paramiko.ECDSAKey),
("RSA", paramiko.RSAKey),
]
last_exception = None
for key_name, key_class in key_types:
try:
return key_class.from_private_key_file(private_key_path)
except paramiko.SSHException as e:
last_exception = e
continue
# If we get here, none of the key types worked
raise paramiko.SSHException(
f"Unable to load private key from {private_key_path}. "
f"Supported formats: Ed25519 (recommended), ECDSA, RSA. "
f"DSA keys are no longer supported. "
f"Last error: {last_exception}"
)
def convert_bytes(bytes):
if bytes is None:
return None
else:
bytes = float(bytes)
if bytes >= 1099511627776:
terabytes = bytes / 1099511627776
size = "%.2fT" % terabytes
elif bytes >= 1073741824:
gigabytes = bytes / 1073741824
size = "%.2fG" % gigabytes
elif bytes >= 1048576:
megabytes = bytes / 1048576
size = "%.2fM" % megabytes
elif bytes >= 1024:
kilobytes = bytes / 1024
size = "%.2fK" % kilobytes
else:
size = "%.2fb" % bytes
return size
## {{{ http://code.activestate.com/recipes/81189/ (r2)
def pp(cursor, data=None, rowlens=0, callback=None):
"""
pretty print a query result as a table
callback is a function called for each field (fieldname,value) to format the output
"""
def defaultcb(fieldname, value):
return value
if not callback:
callback = defaultcb
d = cursor.description
if not d:
return "#### NO RESULTS ###"
names = []
lengths = []
rules = []
if not data:
data = cursor.fetchall()
for dd in d: # iterate over description
l = dd[1]
if not l:
l = 12 # or default arg ...
l = max(l, len(dd[0])) # handle long names
names.append(dd[0])
lengths.append(l)
for col in range(len(lengths)):
if rowlens:
rls = [len(str(callback(d[col][0], row[col]))) for row in data if row[col]]
lengths[col] = max([lengths[col]] + rls)
rules.append("-" * lengths[col])
format = " ".join(["%%-%ss" % l for l in lengths])
result = [format % tuple(names)]
result.append(format % tuple(rules))
for row in data:
row_cb = []
for col in range(len(d)):
row_cb.append(callback(d[col][0], row[col]))
result.append(format % tuple(row_cb))
return "\n".join(result)
## end of http://code.activestate.com/recipes/81189/ }}}
def html_table(cur, callback=None):
"""
cur est un cursor issu d'une requete
callback est une fonction qui prend (rowmap,fieldname,value)
et renvoie une representation texte
"""
def safe_unicode(iso):
if iso is None:
return None
elif isinstance(iso, str):
return iso # .decode()
else:
return iso
def itermap(cur):
for row in cur:
yield dict((cur.description[idx][0], value) for idx, value in enumerate(row))
head = "<tr>" + "".join(["<th>" + c[0] + "</th>" for c in cur.description]) + "</tr>"
lines = ""
if callback:
for r in itermap(cur):
lines = (
lines
+ "<tr>"
+ "".join(["<td>" + str(callback(r, c[0], safe_unicode(r[c[0]]))) + "</td>" for c in cur.description])
+ "</tr>"
)
else:
for r in cur:
lines = lines + "<tr>" + "".join(["<td>" + safe_unicode(c) + "</td>" for c in r]) + "</tr>"
return "<table border=1 cellpadding=2 cellspacing=0>%s%s</table>" % (head, lines)
def monitor_stdout(aprocess, onoutputdata, context):
"""Reads data from stdout and stderr from aprocess and return as a string
on each chunk, call a call back onoutputdata(dataread)
"""
assert isinstance(aprocess, subprocess.Popen)
read_set = []
stdout = []
line = ""
if aprocess.stdout:
read_set.append(aprocess.stdout)
if aprocess.stderr:
read_set.append(aprocess.stderr)
while read_set:
try:
rlist, wlist, xlist = select.select(read_set, [], [])
except select.error as e:
if e.args[0] == errno.EINTR:
continue
raise
# Reads one line from stdout
if aprocess.stdout in rlist:
data = os.read(aprocess.stdout.fileno(), 1)
data = data.decode(errors="ignore")
if data == "":
aprocess.stdout.close()
read_set.remove(aprocess.stdout)
while data and data not in ("\n", "\r"):
line += data
data = os.read(aprocess.stdout.fileno(), 1)
data = data.decode(errors="ignore")
if line or data in ("\n", "\r"):
stdout.append(line)
if onoutputdata:
onoutputdata(line, context)
line = ""
# Reads one line from stderr
if aprocess.stderr in rlist:
data = os.read(aprocess.stderr.fileno(), 1)
data = data.decode(errors="ignore")
if data == "":
aprocess.stderr.close()
read_set.remove(aprocess.stderr)
while data and data not in ("\n", "\r"):
line += data
data = os.read(aprocess.stderr.fileno(), 1)
data = data.decode(errors="ignore")
if line or data in ("\n", "\r"):
stdout.append(line)
if onoutputdata:
onoutputdata(line, context)
line = ""
aprocess.wait()
if line:
stdout.append(line)
if onoutputdata:
onoutputdata(line, context)
return "\n".join(stdout)
def str2bool(val):
if not isinstance(type(val), bool):
return val.lower() in ("yes", "true", "t", "1")
class BackupStat:
dbpath = ""
db = None
logger = logging.getLogger("tisbackup")
def __init__(self, dbpath):
self.dbpath = dbpath
if not os.path.isfile(self.dbpath):
self.db = sqlite3.connect(self.dbpath)
self.initdb()
else:
self.db = sqlite3.connect(self.dbpath, check_same_thread=False)
if "'TYPE'" not in str(self.db.execute("select * from stats").description):
self.updatedb()
def updatedb(self):
self.logger.debug("Update stat database")
self.db.execute("alter table stats add column TYPE TEXT;")
self.db.execute("update stats set TYPE='BACKUP';")
self.db.commit()
def initdb(self):
assert isinstance(self.db, sqlite3.Connection)
self.logger.debug("Initialize stat database")
self.db.execute("""
create table stats (
backup_name TEXT,
server_name TEXT,
description TEXT,
backup_start TEXT,
backup_end TEXT,
backup_duration NUMERIC,
total_files_count INT,
written_files_count INT,
total_bytes INT,
written_bytes INT,
status TEXT,
log TEXT,
backup_location TEXT,
TYPE TEXT)""")
self.db.execute("""
create index idx_stats_backup_name on stats(backup_name);""")
self.db.execute("""
create index idx_stats_backup_location on stats(backup_location);""")
self.db.execute("""
CREATE INDEX idx_stats_backup_name_start on stats(backup_name,backup_start);""")
self.db.commit()
def start(self, backup_name, server_name, TYPE, description="", backup_location=None):
"""Add in stat DB a record for the newly running backup"""
return self.add(
backup_name=backup_name,
server_name=server_name,
description=description,
backup_start=datetime2isodate(),
status="Running",
TYPE=TYPE,
)
def finish(
self,
rowid,
total_files_count=None,
written_files_count=None,
total_bytes=None,
written_bytes=None,
log=None,
status="OK",
backup_end=None,
backup_duration=None,
backup_location=None,
):
"""Update record in stat DB for the finished backup"""
if not backup_end:
backup_end = datetime2isodate()
if backup_duration is None:
try:
# get duration using start of backup datetime
backup_duration = (
isodate2datetime(backup_end)
- isodate2datetime(self.query("select backup_start from stats where rowid=?", (rowid,))[0]["backup_start"])
).seconds / 3600.0
except:
backup_duration = None
# update stat record
self.db.execute(
"""\
update stats set
total_files_count=?,written_files_count=?,total_bytes=?,written_bytes=?,log=?,status=?,backup_end=?,backup_duration=?,backup_location=?
where
rowid = ?
""",
(
total_files_count,
written_files_count,
total_bytes,
written_bytes,
log,
status,
backup_end,
backup_duration,
backup_location,
rowid,
),
)
self.db.commit()
def add(
self,
backup_name="",
server_name="",
description="",
backup_start=None,
backup_end=None,
backup_duration=None,
total_files_count=None,
written_files_count=None,
total_bytes=None,
written_bytes=None,
status="draft",
log="",
TYPE="",
backup_location=None,
):
if not backup_start:
backup_start = datetime2isodate()
if not backup_end:
backup_end = datetime2isodate()
cur = self.db.execute(
"""\
insert into stats (
backup_name,
server_name,
description,
backup_start,
backup_end,
backup_duration,
total_files_count,
written_files_count,
total_bytes,
written_bytes,
status,
log,
backup_location,
TYPE) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?)
""",
(
backup_name,
server_name,
description,
backup_start,
backup_end,
backup_duration,
total_files_count,
written_files_count,
total_bytes,
written_bytes,
status,
log,
backup_location,
TYPE,
),
)
self.db.commit()
return cur.lastrowid
def query(self, query, args=(), one=False):
"""
execute la requete query sur la db et renvoie un tableau de dictionnaires
"""
cur = self.db.execute(query, args)
rv = [dict((cur.description[idx][0], value) for idx, value in enumerate(row)) for row in cur.fetchall()]
return (rv[0] if rv else None) if one else rv
def last_backups(self, backup_name, count=30):
if backup_name:
cur = self.db.execute("select * from stats where backup_name=? order by backup_end desc limit ?", (backup_name, count))
else:
cur = self.db.execute("select * from stats order by backup_end desc limit ?", (count,))
def fcb(fieldname, value):
if fieldname in ("backup_start", "backup_end"):
return time2display(isodate2datetime(value))
elif "bytes" in fieldname:
return convert_bytes(value)
elif "count" in fieldname:
return splitThousands(value, " ", ".")
elif "backup_duration" in fieldname:
return hours_minutes(value)
else:
return value
# for r in self.query('select * from stats where backup_name=? order by backup_end desc limit ?',(backup_name,count)):
print((pp(cur, None, 1, fcb)))
def fcb(self, fields, fieldname, value):
if fieldname in ("backup_start", "backup_end"):
return time2display(isodate2datetime(value))
elif "bytes" in fieldname:
return convert_bytes(value)
elif "count" in fieldname:
return splitThousands(value, " ", ".")
elif "backup_duration" in fieldname:
return hours_minutes(value)
else:
return value
def as_html(self, cur):
if cur:
return html_table(cur, self.fcb)
else:
return html_table(self.db.execute("select * from stats order by backup_start asc"), self.fcb)
def ssh_exec(command, ssh=None, server_name="", remote_user="", private_key="", ssh_port=22):
"""execute command on server_name using the provided ssh connection
or creates a new connection if ssh is not provided.
returns (exit_code,output)
output is the concatenation of stdout and stderr
"""
if not ssh:
assert server_name and remote_user and private_key
mykey = load_ssh_private_key(private_key)
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(server_name, username=remote_user, pkey=mykey, port=ssh_port)
tran = ssh.get_transport()
chan = tran.open_session()
# chan.set_combine_stderr(True)
chan.get_pty()
stdout = chan.makefile()
chan.exec_command(command)
stdout.flush()
output_base = stdout.read()
output = output_base.decode(errors="ignore").replace("'", "")
exit_code = chan.recv_exit_status()
return (exit_code, output)
class backup_generic(ABC):
"""Generic ancestor class for backups, not registered"""
@@ -1067,13 +524,3 @@ class backup_generic(ABC):
backup_location=backup_dest,
)
return stats
if __name__ == "__main__":
logger = logging.getLogger("tisbackup")
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)
dbstat = BackupStat("/backup/data/log/tisbackup.sqlite")
libtisbackup/database.py
+261
View File
@@ -0,0 +1,261 @@
#!/usr/bin/python3
# -*- coding: utf-8 -*-
# -----------------------------------------------------------------------
# This file is part of TISBackup
#
# TISBackup is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# TISBackup is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with TISBackup. If not, see <http://www.gnu.org/licenses/>.
#
# -----------------------------------------------------------------------
"""Database management for backup statistics and history."""
import logging
import os
import sqlite3
from .utils import (
convert_bytes,
datetime2isodate,
hours_minutes,
html_table,
isodate2datetime,
pp,
splitThousands,
time2display,
)
class BackupStat:
"""Manages SQLite database for backup statistics and history."""
dbpath = ""
db = None
logger = logging.getLogger("tisbackup")
def __init__(self, dbpath):
self.dbpath = dbpath
if not os.path.isfile(self.dbpath):
self.db = sqlite3.connect(self.dbpath)
self.initdb()
else:
self.db = sqlite3.connect(self.dbpath, check_same_thread=False)
if "'TYPE'" not in str(self.db.execute("select * from stats").description):
self.updatedb()
def updatedb(self):
"""Update database schema to add TYPE column if missing."""
self.logger.debug("Update stat database")
self.db.execute("alter table stats add column TYPE TEXT;")
self.db.execute("update stats set TYPE='BACKUP';")
self.db.commit()
def initdb(self):
"""Initialize database schema."""
assert isinstance(self.db, sqlite3.Connection)
self.logger.debug("Initialize stat database")
self.db.execute("""
create table stats (
backup_name TEXT,
server_name TEXT,
description TEXT,
backup_start TEXT,
backup_end TEXT,
backup_duration NUMERIC,
total_files_count INT,
written_files_count INT,
total_bytes INT,
written_bytes INT,
status TEXT,
log TEXT,
backup_location TEXT,
TYPE TEXT)""")
self.db.execute("""
create index idx_stats_backup_name on stats(backup_name);""")
self.db.execute("""
create index idx_stats_backup_location on stats(backup_location);""")
self.db.execute("""
CREATE INDEX idx_stats_backup_name_start on stats(backup_name,backup_start);""")
self.db.commit()
def start(self, backup_name, server_name, TYPE, description="", backup_location=None):
"""Add in stat DB a record for the newly running backup"""
return self.add(
backup_name=backup_name,
server_name=server_name,
description=description,
backup_start=datetime2isodate(),
status="Running",
TYPE=TYPE,
)
def finish(
self,
rowid,
total_files_count=None,
written_files_count=None,
total_bytes=None,
written_bytes=None,
log=None,
status="OK",
backup_end=None,
backup_duration=None,
backup_location=None,
):
"""Update record in stat DB for the finished backup"""
if not backup_end:
backup_end = datetime2isodate()
if backup_duration is None:
try:
# get duration using start of backup datetime
backup_duration = (
isodate2datetime(backup_end)
- isodate2datetime(self.query("select backup_start from stats where rowid=?", (rowid,))[0]["backup_start"])
).seconds / 3600.0
except:
backup_duration = None
# update stat record
self.db.execute(
"""\
update stats set
total_files_count=?,written_files_count=?,total_bytes=?,written_bytes=?,log=?,status=?,backup_end=?,backup_duration=?,backup_location=?
where
rowid = ?
""",
(
total_files_count,
written_files_count,
total_bytes,
written_bytes,
log,
status,
backup_end,
backup_duration,
backup_location,
rowid,
),
)
self.db.commit()
def add(
self,
backup_name="",
server_name="",
description="",
backup_start=None,
backup_end=None,
backup_duration=None,
total_files_count=None,
written_files_count=None,
total_bytes=None,
written_bytes=None,
status="draft",
log="",
TYPE="",
backup_location=None,
):
"""Add a new backup record to the database."""
if not backup_start:
backup_start = datetime2isodate()
if not backup_end:
backup_end = datetime2isodate()
cur = self.db.execute(
"""\
insert into stats (
backup_name,
server_name,
description,
backup_start,
backup_end,
backup_duration,
total_files_count,
written_files_count,
total_bytes,
written_bytes,
status,
log,
backup_location,
TYPE) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?)
""",
(
backup_name,
server_name,
description,
backup_start,
backup_end,
backup_duration,
total_files_count,
written_files_count,
total_bytes,
written_bytes,
status,
log,
backup_location,
TYPE,
),
)
self.db.commit()
return cur.lastrowid
def query(self, query, args=(), one=False):
"""
execute la requete query sur la db et renvoie un tableau de dictionnaires
"""
cur = self.db.execute(query, args)
rv = [dict((cur.description[idx][0], value) for idx, value in enumerate(row)) for row in cur.fetchall()]
return (rv[0] if rv else None) if one else rv
def last_backups(self, backup_name, count=30):
"""Display last N backups for a given backup_name."""
if backup_name:
cur = self.db.execute("select * from stats where backup_name=? order by backup_end desc limit ?", (backup_name, count))
else:
cur = self.db.execute("select * from stats order by backup_end desc limit ?", (count,))
def fcb(fieldname, value):
if fieldname in ("backup_start", "backup_end"):
return time2display(isodate2datetime(value))
elif "bytes" in fieldname:
return convert_bytes(value)
elif "count" in fieldname:
return splitThousands(value, " ", ".")
elif "backup_duration" in fieldname:
return hours_minutes(value)
else:
return value
# for r in self.query('select * from stats where backup_name=? order by backup_end desc limit ?',(backup_name,count)):
print((pp(cur, None, 1, fcb)))
def fcb(self, fields, fieldname, value):
"""Format callback for HTML table display."""
if fieldname in ("backup_start", "backup_end"):
return time2display(isodate2datetime(value))
elif "bytes" in fieldname:
return convert_bytes(value)
elif "count" in fieldname:
return splitThousands(value, " ", ".")
elif "backup_duration" in fieldname:
return hours_minutes(value)
else:
return value
def as_html(self, cur):
"""Convert cursor to HTML table."""
if cur:
return html_table(cur, self.fcb)
else:
return html_table(self.db.execute("select * from stats order by backup_start asc"), self.fcb)
libtisbackup/XenAPI.py → libtisbackup/drivers/XenAPI.py
View File
libtisbackup/drivers/__init__.py
+60
View File
@@ -0,0 +1,60 @@
# -----------------------------------------------------------------------
# This file is part of TISBackup
#
# TISBackup is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# TISBackup is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with TISBackup. If not, see <http://www.gnu.org/licenses/>.
#
# -----------------------------------------------------------------------
"""
TISBackup drivers - Pluggable backup driver implementations.
This package contains all backup driver implementations:
- Database drivers (MySQL, PostgreSQL, Oracle, SQL Server)
- File sync drivers (rsync, rsync+btrfs)
- VM backup drivers (XenServer XVA, VMware VMDK)
- Other drivers (Samba4, network switches, etc.)
"""
# Import all drivers to ensure they register themselves
from .backup_mysql import backup_mysql
from .backup_null import backup_null
from .backup_oracle import backup_oracle
from .backup_pgsql import backup_pgsql
from .backup_rsync import backup_rsync, backup_rsync_ssh
from .backup_rsync_btrfs import backup_rsync_btrfs, backup_rsync__btrfs_ssh
from .backup_samba4 import backup_samba4
from .backup_sqlserver import backup_sqlserver
from .backup_switch import backup_switch
from .backup_vmdk import backup_vmdk
from .backup_xcp_metadata import backup_xcp_metadata
from .backup_xva import backup_xva
from .copy_vm_xcp import copy_vm_xcp
__all__ = [
"backup_mysql",
"backup_null",
"backup_oracle",
"backup_pgsql",
"backup_rsync",
"backup_rsync_ssh",
"backup_rsync_btrfs",
"backup_rsync__btrfs_ssh",
"backup_samba4",
"backup_sqlserver",
"backup_switch",
"backup_vmdk",
"backup_xcp_metadata",
"backup_xva",
"copy_vm_xcp",
]
libtisbackup/backup_mysql.py → libtisbackup/drivers/backup_mysql.py
+1 -1
View File
@@ -30,7 +30,7 @@ except ImportError as e:
sys.stderr = sys.__stderr__
from libtisbackup.common import *
from libtisbackup import *
class backup_mysql(backup_generic):
libtisbackup/backup_null.py → libtisbackup/drivers/backup_null.py
Executable → Regular
+1 -1
View File
@@ -21,7 +21,7 @@
import datetime
import os
from .common import *
from libtisbackup import *
class backup_null(backup_generic):
libtisbackup/backup_oracle.py → libtisbackup/drivers/backup_oracle.py
+1 -1
View File
@@ -33,7 +33,7 @@ import datetime
import os
import re
from libtisbackup.common import *
from libtisbackup import *
class backup_oracle(backup_generic):
libtisbackup/backup_pgsql.py → libtisbackup/drivers/backup_pgsql.py
+1 -1
View File
@@ -28,7 +28,7 @@ except ImportError as e:
sys.stderr = sys.__stderr__
from .common import *
from libtisbackup import *
class backup_pgsql(backup_generic):
libtisbackup/backup_rsync.py → libtisbackup/drivers/backup_rsync.py
+1 -1
View File
@@ -25,7 +25,7 @@ import os.path
import re
import time
from libtisbackup.common import *
from libtisbackup import *
class backup_rsync(backup_generic):
libtisbackup/backup_rsync_btrfs.py → libtisbackup/drivers/backup_rsync_btrfs.py
+1 -1
View File
@@ -25,7 +25,7 @@ import os.path
import re
import time
from .common import *
from libtisbackup import *
class backup_rsync_btrfs(backup_generic):
libtisbackup/backup_samba4.py → libtisbackup/drivers/backup_samba4.py
+1 -1
View File
@@ -30,7 +30,7 @@ except ImportError as e:
sys.stderr = sys.__stderr__
from .common import *
from libtisbackup import *
class backup_samba4(backup_generic):
libtisbackup/backup_sqlserver.py → libtisbackup/drivers/backup_sqlserver.py
+1 -1
View File
@@ -34,7 +34,7 @@ import base64
import datetime
import os
from .common import *
from libtisbackup import *
class backup_sqlserver(backup_generic):
libtisbackup/backup_switch.py → libtisbackup/drivers/backup_switch.py
+1 -1
View File
@@ -36,7 +36,7 @@ import pexpect
import requests
from . import XenAPI
from .common import *
from libtisbackup import *
class backup_switch(backup_generic):
libtisbackup/backup_vmdk.py → libtisbackup/drivers/backup_vmdk.py
Executable → Regular
+1 -1
View File
@@ -30,7 +30,7 @@ from pyVmomi import vim, vmodl
# Disable HTTPS verification warnings.
from requests.packages import urllib3
from .common import *
from libtisbackup import *
urllib3.disable_warnings()
import os
libtisbackup/backup_xcp_metadata.py → libtisbackup/drivers/backup_xcp_metadata.py
+1 -1
View File
@@ -21,7 +21,7 @@
import paramiko
from .common import *
from libtisbackup import *
class backup_xcp_metadata(backup_generic):
libtisbackup/backup_xva.py → libtisbackup/drivers/backup_xva.py
Executable → Regular
+1 -1
View File
@@ -35,7 +35,7 @@ from stat import *
import requests
from . import XenAPI
from .common import *
from libtisbackup import *
if hasattr(ssl, "_create_unverified_context"):
ssl._create_default_https_context = ssl._create_unverified_context
libtisbackup/copy_vm_xcp.py → libtisbackup/drivers/copy_vm_xcp.py
Executable → Regular
+1 -1
View File
@@ -34,7 +34,7 @@ import urllib.request
from stat import *
from . import XenAPI
from .common import *
from libtisbackup import *
if hasattr(ssl, "_create_unverified_context"):
ssl._create_default_https_context = ssl._create_unverified_context
libtisbackup/process.py
+97
View File
@@ -0,0 +1,97 @@
#!/usr/bin/python3
# -*- coding: utf-8 -*-
# -----------------------------------------------------------------------
# This file is part of TISBackup
#
# TISBackup is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# TISBackup is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with TISBackup. If not, see <http://www.gnu.org/licenses/>.
#
# -----------------------------------------------------------------------
"""Process execution and monitoring utilities."""
import errno
import os
import select
import subprocess
def call_external_process(shell_string):
"""Execute a shell command and raise exception on non-zero exit code."""
p = subprocess.call(shell_string, shell=True)
if p != 0:
raise Exception("shell program exited with error code " + str(p), shell_string)
def monitor_stdout(aprocess, onoutputdata, context):
"""Reads data from stdout and stderr from aprocess and return as a string
on each chunk, call a call back onoutputdata(dataread)
"""
assert isinstance(aprocess, subprocess.Popen)
read_set = []
stdout = []
line = ""
if aprocess.stdout:
read_set.append(aprocess.stdout)
if aprocess.stderr:
read_set.append(aprocess.stderr)
while read_set:
try:
rlist, wlist, xlist = select.select(read_set, [], [])
except select.error as e:
if e.args[0] == errno.EINTR:
continue
raise
# Reads one line from stdout
if aprocess.stdout in rlist:
data = os.read(aprocess.stdout.fileno(), 1)
data = data.decode(errors="ignore")
if data == "":
aprocess.stdout.close()
read_set.remove(aprocess.stdout)
while data and data not in ("\n", "\r"):
line += data
data = os.read(aprocess.stdout.fileno(), 1)
data = data.decode(errors="ignore")
if line or data in ("\n", "\r"):
stdout.append(line)
if onoutputdata:
onoutputdata(line, context)
line = ""
# Reads one line from stderr
if aprocess.stderr in rlist:
data = os.read(aprocess.stderr.fileno(), 1)
data = data.decode(errors="ignore")
if data == "":
aprocess.stderr.close()
read_set.remove(aprocess.stderr)
while data and data not in ("\n", "\r"):
line += data
data = os.read(aprocess.stderr.fileno(), 1)
data = data.decode(errors="ignore")
if line or data in ("\n", "\r"):
stdout.append(line)
if onoutputdata:
onoutputdata(line, context)
line = ""
aprocess.wait()
if line:
stdout.append(line)
if onoutputdata:
onoutputdata(line, context)
return "\n".join(stdout)
libtisbackup/ssh.py
+104
View File
@@ -0,0 +1,104 @@
#!/usr/bin/python3
# -*- coding: utf-8 -*-
# -----------------------------------------------------------------------
# This file is part of TISBackup
#
# TISBackup is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# TISBackup is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with TISBackup. If not, see <http://www.gnu.org/licenses/>.
#
# -----------------------------------------------------------------------
"""SSH operations and key management utilities."""
import sys
try:
sys.stderr = open("/dev/null") # Silence silly warnings from paramiko
import paramiko
except ImportError as e:
print(("Error : can not load paramiko library %s" % e))
raise
sys.stderr = sys.__stderr__
def load_ssh_private_key(private_key_path):
"""Load SSH private key with modern algorithm support.
Tries to load the key in order of preference:
1. Ed25519 (most secure, modern)
2. ECDSA (secure, widely supported)
3. RSA (legacy, still secure with sufficient key size)
DSA is not supported as it's deprecated and insecure.
Args:
private_key_path: Path to the private key file
Returns:
paramiko key object
Raises:
paramiko.SSHException: If key cannot be loaded
"""
key_types = [
("Ed25519", paramiko.Ed25519Key),
("ECDSA", paramiko.ECDSAKey),
("RSA", paramiko.RSAKey),
]
last_exception = None
for key_name, key_class in key_types:
try:
return key_class.from_private_key_file(private_key_path)
except paramiko.SSHException as e:
last_exception = e
continue
# If we get here, none of the key types worked
raise paramiko.SSHException(
f"Unable to load private key from {private_key_path}. "
f"Supported formats: Ed25519 (recommended), ECDSA, RSA. "
f"DSA keys are no longer supported. "
f"Last error: {last_exception}"
)
def ssh_exec(command, ssh=None, server_name="", remote_user="", private_key="", ssh_port=22):
"""execute command on server_name using the provided ssh connection
or creates a new connection if ssh is not provided.
returns (exit_code,output)
output is the concatenation of stdout and stderr
"""
if not ssh:
assert server_name and remote_user and private_key
mykey = load_ssh_private_key(private_key)
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(server_name, username=remote_user, pkey=mykey, port=ssh_port)
tran = ssh.get_transport()
chan = tran.open_session()
# chan.set_combine_stderr(True)
chan.get_pty()
stdout = chan.makefile()
chan.exec_command(command)
stdout.flush()
output_base = stdout.read()
output = output_base.decode(errors="ignore").replace("'", "")
exit_code = chan.recv_exit_status()
return (exit_code, output)
libtisbackup/utils.py
+222
View File
@@ -0,0 +1,222 @@
#!/usr/bin/python3
# -*- coding: utf-8 -*-
# -----------------------------------------------------------------------
# This file is part of TISBackup
#
# TISBackup is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# TISBackup is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with TISBackup. If not, see <http://www.gnu.org/licenses/>.
#
# -----------------------------------------------------------------------
"""Utility functions for date/time formatting, number formatting, and display helpers."""
import datetime
import os
def datetime2isodate(adatetime=None):
"""Convert datetime to ISO format string."""
if not adatetime:
adatetime = datetime.datetime.now()
assert isinstance(adatetime, datetime.datetime)
return adatetime.isoformat()
def isodate2datetime(isodatestr):
"""Convert ISO format string to datetime."""
# we remove the microseconds part as it is not working for python2.5 strptime
return datetime.datetime.strptime(isodatestr.split(".")[0], "%Y-%m-%dT%H:%M:%S")
def time2display(adatetime):
"""Format datetime for display."""
return adatetime.strftime("%Y-%m-%d %H:%M")
def hours_minutes(hours):
"""Convert decimal hours to HH:MM format."""
if hours is None:
return None
else:
return "%02i:%02i" % (int(hours), int((hours - int(hours)) * 60.0))
def fileisodate(filename):
"""Get file modification time as ISO date string."""
return datetime.datetime.fromtimestamp(os.stat(filename).st_mtime).isoformat()
def dateof(adatetime):
"""Get date part of datetime (midnight)."""
return adatetime.replace(hour=0, minute=0, second=0, microsecond=0)
#####################################
# http://code.activestate.com/recipes/498181-add-thousands-separator-commas-to-formatted-number/
# Code from Michael Robellard's comment made 28 Feb 2010
# Modified for leading +, -, space on 1 Mar 2010 by Glenn Linderman
#
# Tail recursion removed and leading garbage handled on March 12 2010, Alessandro Forghieri
def splitThousands(s, tSep=",", dSep="."):
"""Splits a general float on thousands. GIGO on general input"""
if s is None:
return 0
if not isinstance(s, str):
s = str(s)
cnt = 0
numChars = dSep + "0123456789"
ls = len(s)
while cnt < ls and s[cnt] not in numChars:
cnt += 1
lhs = s[0:cnt]
s = s[cnt:]
if dSep == "":
cnt = -1
else:
cnt = s.rfind(dSep)
if cnt > 0:
rhs = dSep + s[cnt + 1 :]
s = s[:cnt]
else:
rhs = ""
splt = ""
while s != "":
splt = s[-3:] + tSep + splt
s = s[:-3]
return lhs + splt[:-1] + rhs
def convert_bytes(bytes):
"""Convert bytes to human-readable format (T/G/M/K/b)."""
if bytes is None:
return None
else:
bytes = float(bytes)
if bytes >= 1099511627776:
terabytes = bytes / 1099511627776
size = "%.2fT" % terabytes
elif bytes >= 1073741824:
gigabytes = bytes / 1073741824
size = "%.2fG" % gigabytes
elif bytes >= 1048576:
megabytes = bytes / 1048576
size = "%.2fM" % megabytes
elif bytes >= 1024:
kilobytes = bytes / 1024
size = "%.2fK" % kilobytes
else:
size = "%.2fb" % bytes
return size
def check_string(test_string):
"""Check if string contains only alphanumeric characters, dots, dashes, and underscores."""
import re
pattern = r"[^\.A-Za-z0-9\-_]"
if re.search(pattern, test_string):
# Character other then . a-z 0-9 was found
print(("Invalid : %r" % (test_string,)))
def str2bool(val):
"""Convert string to boolean."""
if not isinstance(type(val), bool):
return val.lower() in ("yes", "true", "t", "1")
## {{{ http://code.activestate.com/recipes/81189/ (r2)
def pp(cursor, data=None, rowlens=0, callback=None):
"""
pretty print a query result as a table
callback is a function called for each field (fieldname,value) to format the output
"""
def defaultcb(fieldname, value):
return value
if not callback:
callback = defaultcb
d = cursor.description
if not d:
return "#### NO RESULTS ###"
names = []
lengths = []
rules = []
if not data:
data = cursor.fetchall()
for dd in d: # iterate over description
l = dd[1]
if not l:
l = 12 # or default arg ...
l = max(l, len(dd[0])) # handle long names
names.append(dd[0])
lengths.append(l)
for col in range(len(lengths)):
if rowlens:
rls = [len(str(callback(d[col][0], row[col]))) for row in data if row[col]]
lengths[col] = max([lengths[col]] + rls)
rules.append("-" * lengths[col])
format = " ".join(["%%-%ss" % l for l in lengths])
result = [format % tuple(names)]
result.append(format % tuple(rules))
for row in data:
row_cb = []
for col in range(len(d)):
row_cb.append(callback(d[col][0], row[col]))
result.append(format % tuple(row_cb))
return "\n".join(result)
## end of http://code.activestate.com/recipes/81189/ }}}
def html_table(cur, callback=None):
"""
cur est un cursor issu d'une requete
callback est une fonction qui prend (rowmap,fieldname,value)
et renvoie une representation texte
"""
def safe_unicode(iso):
if iso is None:
return None
elif isinstance(iso, str):
return iso # .decode()
else:
return iso
def itermap(cur):
for row in cur:
yield dict((cur.description[idx][0], value) for idx, value in enumerate(row))
head = "<tr>" + "".join(["<th>" + c[0] + "</th>" for c in cur.description]) + "</tr>"
lines = ""
if callback:
for r in itermap(cur):
lines = (
lines
+ "<tr>"
+ "".join(["<td>" + str(callback(r, c[0], safe_unicode(r[c[0]]))) + "</td>" for c in cur.description])
+ "</tr>"
)
else:
for r in cur:
lines = lines + "<tr>" + "".join(["<td>" + safe_unicode(c) + "</td>" for c in r]) + "</tr>"
return "<table border=1 cellpadding=2 cellspacing=0>%s%s</table>" % (head, lines)