fix iniparse
lint / docker (push) Successful in 9m14s

fix code passing ruff linter
pre-commit ruff
pre-commit ruff format
This commit is contained in:
2024-11-29 23:45:40 +01:00
parent aa8a68aa80
commit 737f9bea38
27 changed files with 2375 additions and 2016 deletions
+184 -149
View File
@@ -30,86 +30,90 @@ from .common import *
class backup_rsync_btrfs(backup_generic):
"""Backup a directory on remote server with rsync and btrfs protocol (requires running remote rsync daemon)"""
type = 'rsync+btrfs'
required_params = backup_generic.required_params + ['remote_user','remote_dir','rsync_module','password_file']
optional_params = backup_generic.optional_params + ['compressionlevel','compression','bwlimit','exclude_list','protect_args','overload_args']
remote_user='root'
remote_dir=''
type = "rsync+btrfs"
required_params = backup_generic.required_params + ["remote_user", "remote_dir", "rsync_module", "password_file"]
optional_params = backup_generic.optional_params + [
"compressionlevel",
"compression",
"bwlimit",
"exclude_list",
"protect_args",
"overload_args",
]
exclude_list=''
rsync_module=''
password_file = ''
compression = ''
remote_user = "root"
remote_dir = ""
exclude_list = ""
rsync_module = ""
password_file = ""
compression = ""
bwlimit = 0
protect_args = '1'
protect_args = "1"
overload_args = None
compressionlevel = 0
def read_config(self, iniconf):
assert isinstance(iniconf, ConfigParser)
backup_generic.read_config(self, iniconf)
if not self.bwlimit and iniconf.has_option("global", "bw_limit"):
self.bwlimit = iniconf.getint("global", "bw_limit")
if not self.compressionlevel and iniconf.has_option("global", "compression_level"):
self.compressionlevel = iniconf.getint("global", "compression_level")
def read_config(self,iniconf):
assert(isinstance(iniconf,ConfigParser))
backup_generic.read_config(self,iniconf)
if not self.bwlimit and iniconf.has_option('global','bw_limit'):
self.bwlimit = iniconf.getint('global','bw_limit')
if not self.compressionlevel and iniconf.has_option('global','compression_level'):
self.compressionlevel = iniconf.getint('global','compression_level')
def do_backup(self,stats):
def do_backup(self, stats):
if not self.set_lock():
self.logger.error("[%s] a lock file is set, a backup maybe already running!!",self.backup_name)
self.logger.error("[%s] a lock file is set, a backup maybe already running!!", self.backup_name)
return False
try:
try:
backup_source = 'undefined'
dest_dir = os.path.join(self.backup_dir,'last_backup')
backup_source = "undefined"
dest_dir = os.path.join(self.backup_dir, "last_backup")
if not os.path.isdir(dest_dir):
if not self.dry_run:
cmd = "/bin/btrfs subvolume create %s"%dest_dir
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
log = monitor_stdout(process,'',self)
cmd = "/bin/btrfs subvolume create %s" % dest_dir
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
log = monitor_stdout(process, "", self)
returncode = process.returncode
if (returncode != 0):
self.logger.error("[" + self.backup_name + "] shell program exited with error code: %s"%log)
if returncode != 0:
self.logger.error("[" + self.backup_name + "] shell program exited with error code: %s" % log)
raise Exception("[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd)
else:
self.logger.info("[" + self.backup_name + "] create btrs volume: %s"%dest_dir)
self.logger.info("[" + self.backup_name + "] create btrs volume: %s" % dest_dir)
else:
print(('btrfs subvolume create "%s"' %dest_dir))
print(('btrfs subvolume create "%s"' % dest_dir))
options = ['-rt','--stats','--delete-excluded','--numeric-ids','--delete-after']
options = ["-rt", "--stats", "--delete-excluded", "--numeric-ids", "--delete-after"]
if self.logger.level:
options.append('-P')
options.append("-P")
if self.dry_run:
options.append('-d')
options.append("-d")
if self.overload_args != None:
if self.overload_args is not None:
options.append(self.overload_args)
elif not "cygdrive" in self.remote_dir:
elif "cygdrive" not in self.remote_dir:
# we don't preserve owner, group, links, hardlinks, perms for windows/cygwin as it is not reliable nor useful
options.append('-lpgoD')
options.append("-lpgoD")
# the protect-args option is not available in all rsync version
if not self.protect_args.lower() in ('false','no','0'):
options.append('--protect-args')
if self.protect_args.lower() not in ("false", "no", "0"):
options.append("--protect-args")
if self.compression.lower() in ('true','yes','1'):
options.append('-z')
if self.compression.lower() in ("true", "yes", "1"):
options.append("-z")
if self.compressionlevel:
options.append('--compress-level=%s' % self.compressionlevel)
options.append("--compress-level=%s" % self.compressionlevel)
if self.bwlimit:
options.append('--bwlimit %s' % self.bwlimit)
options.append("--bwlimit %s" % self.bwlimit)
latest = self.get_latest_backup(self.backup_start_date)
#remove link-dest replace by btrfs
#if latest:
# latest = self.get_latest_backup(self.backup_start_date)
# remove link-dest replace by btrfs
# if latest:
# options.extend(['--link-dest="%s"' % os.path.join('..',b,'') for b in latest])
def strip_quotes(s):
@@ -122,181 +126,203 @@ class backup_rsync_btrfs(backup_generic):
# Add excludes
if "--exclude" in self.exclude_list:
# old settings with exclude_list=--exclude toto --exclude=titi
excludes = [strip_quotes(s).strip() for s in self.exclude_list.replace('--exclude=','').replace('--exclude ','').split()]
excludes = [
strip_quotes(s).strip() for s in self.exclude_list.replace("--exclude=", "").replace("--exclude ", "").split()
]
else:
try:
# newsettings with exclude_list='too','titi', parsed as a str python list content
excludes = eval('[%s]' % self.exclude_list)
excludes = eval("[%s]" % self.exclude_list)
except Exception as e:
raise Exception('Error reading exclude list : value %s, eval error %s (don\'t forget quotes and comma...)' % (self.exclude_list,e))
raise Exception(
"Error reading exclude list : value %s, eval error %s (don't forget quotes and comma...)"
% (self.exclude_list, e)
)
options.extend(['--exclude="%s"' % x for x in excludes])
if (self.rsync_module and not self.password_file):
raise Exception('You must specify a password file if you specify a rsync module')
if self.rsync_module and not self.password_file:
raise Exception("You must specify a password file if you specify a rsync module")
if (not self.rsync_module and not self.private_key):
raise Exception('If you don''t use SSH, you must specify a rsync module')
if not self.rsync_module and not self.private_key:
raise Exception("If you don" "t use SSH, you must specify a rsync module")
#rsync_re = re.compile('(?P<server>[^:]*)::(?P<export>[^/]*)/(?P<path>.*)')
#ssh_re = re.compile('((?P<user>.*)@)?(?P<server>[^:]*):(?P<path>/.*)')
# rsync_re = re.compile('(?P<server>[^:]*)::(?P<export>[^/]*)/(?P<path>.*)')
# ssh_re = re.compile('((?P<user>.*)@)?(?P<server>[^:]*):(?P<path>/.*)')
# Add ssh connection params
if self.rsync_module:
# Case of rsync exports
if self.password_file:
options.append('--password-file="%s"' % self.password_file)
backup_source = '%s@%s::%s%s' % (self.remote_user, self.server_name, self.rsync_module, self.remote_dir)
backup_source = "%s@%s::%s%s" % (self.remote_user, self.server_name, self.rsync_module, self.remote_dir)
else:
# case of rsync + ssh
ssh_params = ['-o StrictHostKeyChecking=no']
ssh_params = ["-o StrictHostKeyChecking=no"]
if self.private_key:
ssh_params.append('-i %s' % self.private_key)
ssh_params.append("-i %s" % self.private_key)
if self.cipher_spec:
ssh_params.append('-c %s' % self.cipher_spec)
ssh_params.append("-c %s" % self.cipher_spec)
if self.ssh_port != 22:
ssh_params.append('-p %i' % self.ssh_port)
ssh_params.append("-p %i" % self.ssh_port)
options.append('-e "/usr/bin/ssh %s"' % (" ".join(ssh_params)))
backup_source = '%s@%s:%s' % (self.remote_user,self.server_name,self.remote_dir)
backup_source = "%s@%s:%s" % (self.remote_user, self.server_name, self.remote_dir)
# ensure there is a slash at end
if backup_source[-1] != '/':
backup_source += '/'
if backup_source[-1] != "/":
backup_source += "/"
options_params = " ".join(options)
cmd = '/usr/bin/rsync %s %s %s 2>&1' % (options_params,backup_source,dest_dir)
self.logger.debug("[%s] rsync : %s",self.backup_name,cmd)
cmd = "/usr/bin/rsync %s %s %s 2>&1" % (options_params, backup_source, dest_dir)
self.logger.debug("[%s] rsync : %s", self.backup_name, cmd)
if not self.dry_run:
self.line = ''
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
def ondata(data,context):
self.line = ""
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
def ondata(data, context):
if context.verbose:
print(data)
context.logger.debug(data)
log = monitor_stdout(process,ondata,self)
log = monitor_stdout(process, ondata, self)
reg_total_files = re.compile('Number of files: (?P<file>\d+)')
reg_transferred_files = re.compile('Number of .*files transferred: (?P<file>\d+)')
reg_total_files = re.compile("Number of files: (?P<file>\d+)")
reg_transferred_files = re.compile("Number of .*files transferred: (?P<file>\d+)")
for l in log.splitlines():
line = l.replace(',','')
line = l.replace(",", "")
m = reg_total_files.match(line)
if m:
stats['total_files_count'] += int(m.groupdict()['file'])
stats["total_files_count"] += int(m.groupdict()["file"])
m = reg_transferred_files.match(line)
if m:
stats['written_files_count'] += int(m.groupdict()['file'])
if line.startswith('Total file size:'):
stats['total_bytes'] += int(line.split(':')[1].split()[0])
if line.startswith('Total transferred file size:'):
stats['written_bytes'] += int(line.split(':')[1].split()[0])
stats["written_files_count"] += int(m.groupdict()["file"])
if line.startswith("Total file size:"):
stats["total_bytes"] += int(line.split(":")[1].split()[0])
if line.startswith("Total transferred file size:"):
stats["written_bytes"] += int(line.split(":")[1].split()[0])
returncode = process.returncode
## deal with exit code 24 (file vanished)
if (returncode == 24):
if returncode == 24:
self.logger.warning("[" + self.backup_name + "] Note: some files vanished before transfer")
elif (returncode == 23):
elif returncode == 23:
self.logger.warning("[" + self.backup_name + "] unable so set uid on some files")
elif (returncode != 0):
elif returncode != 0:
self.logger.error("[" + self.backup_name + "] shell program exited with error code ", str(returncode))
raise Exception("[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd, log[-512:])
raise Exception(
"[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd, log[-512:]
)
else:
print(cmd)
#we take a snapshot of last_backup if everything went well
finaldest = os.path.join(self.backup_dir,self.backup_start_date)
self.logger.debug("[%s] snapshoting last_backup directory from %s to %s" ,self.backup_name,dest_dir,finaldest)
# we take a snapshot of last_backup if everything went well
finaldest = os.path.join(self.backup_dir, self.backup_start_date)
self.logger.debug("[%s] snapshoting last_backup directory from %s to %s", self.backup_name, dest_dir, finaldest)
if not os.path.isdir(finaldest):
if not self.dry_run:
cmd = "/bin/btrfs subvolume snapshot %s %s"%(dest_dir,finaldest)
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
log = monitor_stdout(process,'',self)
cmd = "/bin/btrfs subvolume snapshot %s %s" % (dest_dir, finaldest)
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
log = monitor_stdout(process, "", self)
returncode = process.returncode
if (returncode != 0):
if returncode != 0:
self.logger.error("[" + self.backup_name + "] shell program exited with error code " + str(returncode))
raise Exception("[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd, log[-512:])
raise Exception(
"[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd, log[-512:]
)
else:
self.logger.info("[" + self.backup_name + "] snapshot directory created %s"%finaldest)
self.logger.info("[" + self.backup_name + "] snapshot directory created %s" % finaldest)
else:
print(("btrfs snapshot of %s to %s"%(dest_dir,finaldest)))
print(("btrfs snapshot of %s to %s" % (dest_dir, finaldest)))
else:
raise Exception('snapshot directory already exists : %s' %finaldest)
self.logger.debug("[%s] touching datetime of target directory %s" ,self.backup_name,finaldest)
raise Exception("snapshot directory already exists : %s" % finaldest)
self.logger.debug("[%s] touching datetime of target directory %s", self.backup_name, finaldest)
print((os.popen('touch "%s"' % finaldest).read()))
stats['backup_location'] = finaldest
stats['status']='OK'
stats['log']='ssh+rsync+btrfs backup from %s OK, %d bytes written for %d changed files' % (backup_source,stats['written_bytes'],stats['written_files_count'])
stats["backup_location"] = finaldest
stats["status"] = "OK"
stats["log"] = "ssh+rsync+btrfs backup from %s OK, %d bytes written for %d changed files" % (
backup_source,
stats["written_bytes"],
stats["written_files_count"],
)
except BaseException as e:
stats['status']='ERROR'
stats['log']=str(e)
stats["status"] = "ERROR"
stats["log"] = str(e)
raise
finally:
self.remove_lock()
def get_latest_backup(self,current):
def get_latest_backup(self, current):
result = []
filelist = os.listdir(self.backup_dir)
filelist.sort()
filelist.reverse()
full = ''
r_full = re.compile('^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}$')
r_partial = re.compile('^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}.rsync$')
# full = ''
r_full = re.compile("^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}$")
r_partial = re.compile("^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}.rsync$")
# we take all latest partials younger than the latest full and the latest full
for item in filelist:
if r_partial.match(item) and item<current:
if r_partial.match(item) and item < current:
result.append(item)
elif r_full.match(item) and item<current:
elif r_full.match(item) and item < current:
result.append(item)
break
return result
def register_existingbackups(self):
"""scan backup dir and insert stats in database"""
registered = [b['backup_location'] for b in self.dbstat.query('select distinct backup_location from stats where backup_name=?',(self.backup_name,))]
registered = [
b["backup_location"]
for b in self.dbstat.query("select distinct backup_location from stats where backup_name=?", (self.backup_name,))
]
filelist = os.listdir(self.backup_dir)
filelist.sort()
p = re.compile('^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}$')
p = re.compile("^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}$")
for item in filelist:
if p.match(item):
dir_name = os.path.join(self.backup_dir,item)
if not dir_name in registered:
start = datetime.datetime.strptime(item,'%Y%m%d-%Hh%Mm%S').isoformat()
if fileisodate(dir_name)>start:
dir_name = os.path.join(self.backup_dir, item)
if dir_name not in registered:
start = datetime.datetime.strptime(item, "%Y%m%d-%Hh%Mm%S").isoformat()
if fileisodate(dir_name) > start:
stop = fileisodate(dir_name)
else:
stop = start
self.logger.info('Registering %s started on %s',dir_name,start)
self.logger.debug(' Disk usage %s','du -sb "%s"' % dir_name)
self.logger.info("Registering %s started on %s", dir_name, start)
self.logger.debug(" Disk usage %s", 'du -sb "%s"' % dir_name)
if not self.dry_run:
size_bytes = int(os.popen('du -sb "%s"' % dir_name).read().split('\t')[0])
size_bytes = int(os.popen('du -sb "%s"' % dir_name).read().split("\t")[0])
else:
size_bytes = 0
self.logger.debug(' Size in bytes : %i',size_bytes)
self.logger.debug(" Size in bytes : %i", size_bytes)
if not self.dry_run:
self.dbstat.add(self.backup_name,self.server_name,'',\
backup_start=start,backup_end = stop,status='OK',total_bytes=size_bytes,backup_location=dir_name)
self.dbstat.add(
self.backup_name,
self.server_name,
"",
backup_start=start,
backup_end=stop,
status="OK",
total_bytes=size_bytes,
backup_location=dir_name,
)
else:
self.logger.info('Skipping %s, already registered',dir_name)
self.logger.info("Skipping %s, already registered", dir_name)
def is_pid_still_running(self,lockfile):
def is_pid_still_running(self, lockfile):
f = open(lockfile)
lines = f.readlines()
f.close()
if len(lines)==0 :
if len(lines) == 0:
self.logger.info("[" + self.backup_name + "] empty lock file, removing...")
return False
for line in lines:
if line.startswith('pid='):
pid = line.split('=')[1].strip()
if line.startswith("pid="):
pid = line.split("=")[1].strip()
if os.path.exists("/proc/" + pid):
self.logger.info("[" + self.backup_name + "] process still there")
return True
@@ -307,54 +333,63 @@ class backup_rsync_btrfs(backup_generic):
self.logger.info("[" + self.backup_name + "] incorrrect lock file : no pid line")
return False
def set_lock(self):
self.logger.debug("[" + self.backup_name + "] setting lock")
#TODO: improve for race condition
#TODO: also check if process is really there
if os.path.isfile(self.backup_dir + '/lock'):
self.logger.debug("[" + self.backup_name + "] File " + self.backup_dir + '/lock already exist')
if self.is_pid_still_running(self.backup_dir + '/lock')==False:
self.logger.info("[" + self.backup_name + "] removing lock file " + self.backup_dir + '/lock')
os.unlink(self.backup_dir + '/lock')
# TODO: improve for race condition
# TODO: also check if process is really there
if os.path.isfile(self.backup_dir + "/lock"):
self.logger.debug("[" + self.backup_name + "] File " + self.backup_dir + "/lock already exist")
if not self.is_pid_still_running(self.backup_dir + "/lock"):
self.logger.info("[" + self.backup_name + "] removing lock file " + self.backup_dir + "/lock")
os.unlink(self.backup_dir + "/lock")
else:
return False
lockfile = open(self.backup_dir + '/lock',"w")
lockfile = open(self.backup_dir + "/lock", "w")
# Write all the lines at once:
lockfile.write('pid='+str(os.getpid()))
lockfile.write('\nbackup_time=' + self.backup_start_date)
lockfile.write("pid=" + str(os.getpid()))
lockfile.write("\nbackup_time=" + self.backup_start_date)
lockfile.close()
return True
def remove_lock(self):
self.logger.debug("[%s] removing lock",self.backup_name )
os.unlink(self.backup_dir + '/lock')
self.logger.debug("[%s] removing lock", self.backup_name)
os.unlink(self.backup_dir + "/lock")
class backup_rsync__btrfs_ssh(backup_rsync_btrfs):
"""Backup a directory on remote server with rsync,ssh and btrfs protocol (requires rsync software on remote host)"""
type = 'rsync+btrfs+ssh'
required_params = backup_generic.required_params + ['remote_user','remote_dir','private_key']
optional_params = backup_generic.optional_params + ['compression','bwlimit','ssh_port','exclude_list','protect_args','overload_args','cipher_spec']
cipher_spec = ''
type = "rsync+btrfs+ssh"
required_params = backup_generic.required_params + ["remote_user", "remote_dir", "private_key"]
optional_params = backup_generic.optional_params + [
"compression",
"bwlimit",
"ssh_port",
"exclude_list",
"protect_args",
"overload_args",
"cipher_spec",
]
cipher_spec = ""
register_driver(backup_rsync_btrfs)
register_driver(backup_rsync__btrfs_ssh)
if __name__=='__main__':
logger = logging.getLogger('tisbackup')
if __name__ == "__main__":
logger = logging.getLogger("tisbackup")
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)
cp = ConfigParser()
cp.read('/opt/tisbackup/configtest.ini')
dbstat = BackupStat('/backup/data/log/tisbackup.sqlite')
b = backup_rsync('htouvet','/backup/data/htouvet',dbstat)
cp.read("/opt/tisbackup/configtest.ini")
dbstat = BackupStat("/backup/data/log/tisbackup.sqlite")
b = backup_rsync("htouvet", "/backup/data/htouvet", dbstat)
b.read_config(cp)
b.process_backup()
print((b.checknagios()))