fix iniparse
lint / docker (push) Successful in 9m14s

fix code passing ruff linter
pre-commit ruff
pre-commit ruff format
This commit is contained in:
2024-11-29 23:45:40 +01:00
parent aa8a68aa80
commit 737f9bea38
27 changed files with 2375 additions and 2016 deletions
+80 -69
View File
@@ -20,7 +20,7 @@
import sys
try:
sys.stderr = open('/dev/null') # Silence silly warnings from paramiko
sys.stderr = open("/dev/null") # Silence silly warnings from paramiko
import paramiko
except ImportError as e:
print(("Error : can not load paramiko library %s" % e))
@@ -33,16 +33,17 @@ from .common import *
class backup_pgsql(backup_generic):
"""Backup a postgresql database as gzipped sql file through ssh"""
type = 'pgsql+ssh'
required_params = backup_generic.required_params + ['private_key']
optional_params = backup_generic.optional_params + ['db_name','tmp_dir','encoding']
db_name = ''
tmp_dir = '/tmp'
encoding = 'UTF8'
type = "pgsql+ssh"
required_params = backup_generic.required_params + ["private_key"]
optional_params = backup_generic.optional_params + ["db_name", "tmp_dir", "encoding"]
def do_backup(self,stats):
self.dest_dir = os.path.join(self.backup_dir,self.backup_start_date)
db_name = ""
tmp_dir = "/tmp"
encoding = "UTF8"
def do_backup(self, stats):
self.dest_dir = os.path.join(self.backup_dir, self.backup_start_date)
if not os.path.isdir(self.dest_dir):
if not self.dry_run:
@@ -50,117 +51,127 @@ class backup_pgsql(backup_generic):
else:
print(('mkdir "%s"' % self.dest_dir))
else:
raise Exception('backup destination directory already exists : %s' % self.dest_dir)
raise Exception("backup destination directory already exists : %s" % self.dest_dir)
try:
mykey = paramiko.RSAKey.from_private_key_file(self.private_key)
except paramiko.SSHException:
#mykey = paramiko.DSSKey.from_private_key_file(self.private_key)
# mykey = paramiko.DSSKey.from_private_key_file(self.private_key)
mykey = paramiko.Ed25519Key.from_private_key_file(self.private_key)
self.logger.debug('[%s] Trying to connect to "%s" with username root and key "%s"',self.backup_name,self.server_name,self.private_key)
self.logger.debug(
'[%s] Trying to connect to "%s" with username root and key "%s"', self.backup_name, self.server_name, self.private_key
)
self.ssh = paramiko.SSHClient()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh.connect(self.server_name,username='root',pkey = mykey,port=self.ssh_port)
self.ssh.connect(self.server_name, username="root", pkey=mykey, port=self.ssh_port)
if self.db_name:
stats['log']= "Successfully backup processed to the following database :"
if self.db_name:
stats["log"] = "Successfully backup processed to the following database :"
self.do_pgsqldump(stats)
else:
stats['log']= "Successfully backuping processed to the following databases :"
stats['status']='List'
stats["log"] = "Successfully backuping processed to the following databases :"
stats["status"] = "List"
cmd = """su - postgres -c 'psql -A -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;"' 2> /dev/null"""
self.logger.debug('[%s] List databases: %s',self.backup_name,cmd)
(error_code,output) = ssh_exec(cmd,ssh=self.ssh)
self.logger.debug("[%s] Output of %s :\n%s",self.backup_name,cmd,output)
self.logger.debug("[%s] List databases: %s", self.backup_name, cmd)
(error_code, output) = ssh_exec(cmd, ssh=self.ssh)
self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
if error_code:
raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd))
databases = output.split('\n')
raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
databases = output.split("\n")
for database in databases:
if database.strip() not in ("", "template0", "template1"):
self.db_name = database.strip()
self.do_pgsqldump(stats)
stats["status"] = "OK"
stats['status']='OK'
def do_pgsqldump(self,stats):
def do_pgsqldump(self, stats):
t = datetime.datetime.now()
backup_start_date = t.strftime('%Y%m%d-%Hh%Mm%S')
params = {
'encoding':self.encoding,
'db_name':self.db_name,
'tmp_dir':self.tmp_dir,
'dest_dir':self.dest_dir,
'backup_start_date':backup_start_date}
backup_start_date = t.strftime("%Y%m%d-%Hh%Mm%S")
params = {
"encoding": self.encoding,
"db_name": self.db_name,
"tmp_dir": self.tmp_dir,
"dest_dir": self.dest_dir,
"backup_start_date": backup_start_date,
}
# dump db
filepath = '%(tmp_dir)s/%(db_name)s-%(backup_start_date)s.sql.gz' % params
filepath = "%(tmp_dir)s/%(db_name)s-%(backup_start_date)s.sql.gz" % params
cmd = "su - postgres -c 'pg_dump -E %(encoding)s -Z9 %(db_name)s'" % params
cmd += ' > ' + filepath
self.logger.debug('[%s] %s ',self.backup_name,cmd)
cmd += " > " + filepath
self.logger.debug("[%s] %s ", self.backup_name, cmd)
if not self.dry_run:
(error_code,output) = ssh_exec(cmd,ssh=self.ssh)
self.logger.debug("[%s] Output of %s :\n%s",self.backup_name,cmd,output)
(error_code, output) = ssh_exec(cmd, ssh=self.ssh)
self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
if error_code:
raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd))
raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
# get the file
localpath = '%(dest_dir)s/%(db_name)s-%(backup_start_date)s.sql.gz' % params
self.logger.debug('[%s] get the file using sftp from "%s" to "%s" ',self.backup_name,filepath,localpath)
localpath = "%(dest_dir)s/%(db_name)s-%(backup_start_date)s.sql.gz" % params
self.logger.debug('[%s] get the file using sftp from "%s" to "%s" ', self.backup_name, filepath, localpath)
if not self.dry_run:
transport = self.ssh.get_transport()
transport = self.ssh.get_transport()
sftp = paramiko.SFTPClient.from_transport(transport)
sftp.get(filepath, localpath)
sftp.close()
if not self.dry_run:
stats['total_files_count']=1 + stats.get('total_files_count', 0)
stats['written_files_count']=1 + stats.get('written_files_count', 0)
stats['total_bytes']=os.stat(localpath).st_size + stats.get('total_bytes', 0)
stats['written_bytes']=os.stat(localpath).st_size + stats.get('written_bytes', 0)
stats['log'] = '%s "%s"' % (stats['log'] ,self.db_name)
stats['backup_location'] = self.dest_dir
stats["total_files_count"] = 1 + stats.get("total_files_count", 0)
stats["written_files_count"] = 1 + stats.get("written_files_count", 0)
stats["total_bytes"] = os.stat(localpath).st_size + stats.get("total_bytes", 0)
stats["written_bytes"] = os.stat(localpath).st_size + stats.get("written_bytes", 0)
stats["log"] = '%s "%s"' % (stats["log"], self.db_name)
stats["backup_location"] = self.dest_dir
cmd = 'rm -f %(tmp_dir)s/%(db_name)s-%(backup_start_date)s.sql.gz' % params
self.logger.debug('[%s] %s ',self.backup_name,cmd)
cmd = "rm -f %(tmp_dir)s/%(db_name)s-%(backup_start_date)s.sql.gz" % params
self.logger.debug("[%s] %s ", self.backup_name, cmd)
if not self.dry_run:
(error_code,output) = ssh_exec(cmd,ssh=self.ssh)
self.logger.debug("[%s] Output of %s :\n%s",self.backup_name,cmd,output)
(error_code, output) = ssh_exec(cmd, ssh=self.ssh)
self.logger.debug("[%s] Output of %s :\n%s", self.backup_name, cmd, output)
if error_code:
raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd))
raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code, cmd))
def register_existingbackups(self):
"""scan backup dir and insert stats in database"""
registered = [b['backup_location'] for b in self.dbstat.query('select distinct backup_location from stats where backup_name=?',(self.backup_name,))]
registered = [
b["backup_location"]
for b in self.dbstat.query("select distinct backup_location from stats where backup_name=?", (self.backup_name,))
]
filelist = os.listdir(self.backup_dir)
filelist.sort()
p = re.compile('^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}$')
p = re.compile("^\d{8,8}-\d{2,2}h\d{2,2}m\d{2,2}$")
for item in filelist:
if p.match(item):
dir_name = os.path.join(self.backup_dir,item)
if not dir_name in registered:
start = datetime.datetime.strptime(item,'%Y%m%d-%Hh%Mm%S').isoformat()
if fileisodate(dir_name)>start:
dir_name = os.path.join(self.backup_dir, item)
if dir_name not in registered:
start = datetime.datetime.strptime(item, "%Y%m%d-%Hh%Mm%S").isoformat()
if fileisodate(dir_name) > start:
stop = fileisodate(dir_name)
else:
stop = start
self.logger.info('Registering %s started on %s',dir_name,start)
self.logger.debug(' Disk usage %s','du -sb "%s"' % dir_name)
self.logger.info("Registering %s started on %s", dir_name, start)
self.logger.debug(" Disk usage %s", 'du -sb "%s"' % dir_name)
if not self.dry_run:
size_bytes = int(os.popen('du -sb "%s"' % dir_name).read().split('\t')[0])
size_bytes = int(os.popen('du -sb "%s"' % dir_name).read().split("\t")[0])
else:
size_bytes = 0
self.logger.debug(' Size in bytes : %i',size_bytes)
self.logger.debug(" Size in bytes : %i", size_bytes)
if not self.dry_run:
self.dbstat.add(self.backup_name,self.server_name,'',\
backup_start=start,backup_end = stop,status='OK',total_bytes=size_bytes,backup_location=dir_name)
self.dbstat.add(
self.backup_name,
self.server_name,
"",
backup_start=start,
backup_end=stop,
status="OK",
total_bytes=size_bytes,
backup_location=dir_name,
)
else:
self.logger.info('Skipping %s, already registered',dir_name)
self.logger.info("Skipping %s, already registered", dir_name)
register_driver(backup_pgsql)