From bc4b9811edd5d222352acbf45ad9d318cd6d01b5 Mon Sep 17 00:00:00 2001 From: htouvet Date: Mon, 25 Apr 2022 10:02:43 +0200 Subject: [PATCH] migrate to Python3 (from alejeune) --- .gitignore | 9 +- .vscode/launch.json | 22 + deb/control | 4 +- deb/createdeb.sh | 14 +- deb/postinst | 2 + deb/postrm | 2 + deb/prerm | 3 + .../source/installing_tisbackup.rst | 55 +- docs/_sources/installing_tisbackup.rst.txt | 55 +- lib/huey/__init__.py | 62 --- lib/huey/api.py | 513 ------------------ lib/huey/backends/__init__.py | 0 lib/huey/backends/base.py | 113 ---- lib/huey/backends/dummy.py | 103 ---- lib/huey/backends/rabbitmq_backend.py | 153 ------ lib/huey/backends/redis_backend.py | 153 ------ lib/huey/backends/sqlite_backend.py | 205 ------- lib/huey/bin/__init__.py | 0 lib/huey/bin/huey_consumer.py | 121 ----- lib/huey/consumer.py | 279 ---------- lib/huey/djhuey/__init__.py | 119 ---- lib/huey/djhuey/management/__init__.py | 0 .../djhuey/management/commands/__init__.py | 0 .../djhuey/management/commands/run_huey.py | 126 ----- lib/huey/djhuey/models.py | 0 lib/huey/exceptions.py | 26 - lib/huey/peewee_helpers/__init__.py | 20 - lib/huey/registry.py | 77 --- lib/huey/tests/__init__.py | 10 - lib/huey/tests/backends.py | 170 ------ lib/huey/tests/consumer.py | 441 --------------- lib/huey/tests/crontab.py | 91 ---- lib/huey/tests/peewee_tests.py | 62 --- lib/huey/tests/queue.py | 438 --------------- lib/huey/tests/utils.py | 24 - lib/huey/utils.py | 21 - libtisbackup/XenAPI.py | 152 +++--- libtisbackup/backup_mysql.py | 14 +- libtisbackup/backup_null.py | 4 +- libtisbackup/backup_oracle.py | 14 +- libtisbackup/backup_pgsql.py | 10 +- libtisbackup/backup_rdiff.py | 127 ----- libtisbackup/backup_rsync.py | 26 +- libtisbackup/backup_rsync_btrfs.py | 28 +- libtisbackup/backup_samba4.py | 12 +- libtisbackup/backup_sqlserver.py | 19 +- libtisbackup/backup_switch.py | 13 +- libtisbackup/backup_vmdk.py | 10 +- libtisbackup/backup_xcp_metadata.py | 4 +- libtisbackup/backup_xva.py | 34 +- libtisbackup/common.py | 44 +- libtisbackup/copy_vm_xcp.py | 43 +- .../iniparse}/__init__.py | 20 +- {iniparse => libtisbackup/iniparse}/compat.py | 38 +- {iniparse => libtisbackup/iniparse}/config.py | 22 +- libtisbackup/iniparse/configparser.py | 7 + {iniparse => libtisbackup/iniparse}/ini.py | 95 ++-- {iniparse => libtisbackup/iniparse}/utils.py | 9 +- requirements.txt | 2 + rpm/build.sh | 3 + rpm/tis-tisbackup.spec | 30 +- samples/migrate_rsync_to_btrfs.py | 39 -- tasks.py | 4 +- tisbackup.py | 47 +- tisbackup_gui.py | 95 ++-- 65 files changed, 574 insertions(+), 3884 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 deb/postrm create mode 100644 deb/prerm delete mode 100644 lib/huey/__init__.py delete mode 100644 lib/huey/api.py delete mode 100644 lib/huey/backends/__init__.py delete mode 100644 lib/huey/backends/base.py delete mode 100644 lib/huey/backends/dummy.py delete mode 100644 lib/huey/backends/rabbitmq_backend.py delete mode 100644 lib/huey/backends/redis_backend.py delete mode 100644 lib/huey/backends/sqlite_backend.py delete mode 100644 lib/huey/bin/__init__.py delete mode 100755 lib/huey/bin/huey_consumer.py delete mode 100644 lib/huey/consumer.py delete mode 100644 lib/huey/djhuey/__init__.py delete mode 100644 lib/huey/djhuey/management/__init__.py delete mode 100644 lib/huey/djhuey/management/commands/__init__.py delete mode 100644 lib/huey/djhuey/management/commands/run_huey.py delete mode 100644 lib/huey/djhuey/models.py delete mode 100644 lib/huey/exceptions.py delete mode 100644 lib/huey/peewee_helpers/__init__.py delete mode 100644 lib/huey/registry.py delete mode 100644 lib/huey/tests/__init__.py delete mode 100644 lib/huey/tests/backends.py delete mode 100644 lib/huey/tests/consumer.py delete mode 100644 lib/huey/tests/crontab.py delete mode 100644 lib/huey/tests/peewee_tests.py delete mode 100644 lib/huey/tests/queue.py delete mode 100644 lib/huey/tests/utils.py delete mode 100644 lib/huey/utils.py delete mode 100644 libtisbackup/backup_rdiff.py rename {iniparse => libtisbackup/iniparse}/__init__.py (53%) mode change 100755 => 100644 rename {iniparse => libtisbackup/iniparse}/compat.py (92%) mode change 100755 => 100644 rename {iniparse => libtisbackup/iniparse}/config.py (97%) mode change 100755 => 100644 create mode 100644 libtisbackup/iniparse/configparser.py rename {iniparse => libtisbackup/iniparse}/ini.py (90%) mode change 100755 => 100644 rename {iniparse => libtisbackup/iniparse}/utils.py (88%) mode change 100755 => 100644 create mode 100644 requirements.txt delete mode 100644 samples/migrate_rsync_to_btrfs.py diff --git a/.gitignore b/.gitignore index 4ec3da8..646274c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,11 @@ /tasks.sqlite-wal /srvinstallation /tasks.sqlite-shm -.idea \ No newline at end of file +.idea +/deb/builddir +/deb/*.deb +/lib +/rpm/*.rpm +/rpm/RPMS +/rpm/BUILD +/rpm/__VERSION__ diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..153d4c5 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,22 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Current File", + "type": "python", + "request": "launch", + "program": "${file}", + // "args": [ + // "-ldebug", + // "backup" + // ], + "args": [ + "register_existing" + ], + "console": "integratedTerminal" + } + ] +} \ No newline at end of file diff --git a/deb/control b/deb/control index 518860f..2bcf900 100644 --- a/deb/control +++ b/deb/control @@ -1,9 +1,9 @@ Package: tis-tisbackup -Version: 1:__VERSION__ +Version: 1-__VERSION__ Section: base Priority: optional Architecture: all -Depends: unzip, ssh, rsync, python-paramiko, python-pyvmomi, python-pexpect, python-flask,python-simplejson +Depends: unzip, ssh, rsync, python3-paramiko, python3-pyvmomi, python3-pexpect, python3-flask,python3-simplejson, python3-pip Maintainer: Tranquil-IT Description: TISBackup backup management Homepage: https://www.tranquil.it diff --git a/deb/createdeb.sh b/deb/createdeb.sh index f729b74..8f98da7 100755 --- a/deb/createdeb.sh +++ b/deb/createdeb.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash +VERSION_DEB=$(cat /etc/debian_version | cut -d "." -f 1) VERSION_SHORT=$(cat ../tisbackup.py | grep "__version__" | cut -d "=" -f 2 | sed 's/"//g') GIT_COUNT=`git rev-list HEAD --count` -VERSION="${VERSION_SHORT}.${GIT_COUNT}" +VERSION="${VERSION_SHORT}.${GIT_COUNT}-deb${VERSION_DEB}" rm -f *.deb rm -Rf builddir @@ -10,23 +11,26 @@ mkdir builddir mkdir builddir/DEBIAN cp ./control ./builddir/DEBIAN cp ./postinst ./builddir/DEBIAN +cp ./prerm ./builddir/DEBIAN +cp ./postrm ./builddir/DEBIAN sed "s/__VERSION__/$VERSION/" -i ./builddir/DEBIAN/control -mkdir -p builddir/opt/tisbackup/ +mkdir -p ./builddir/opt/tisbackup/ mkdir -p ./builddir/usr/lib/systemd/system/ mkdir -p ./builddir/etc/tis mkdir -p ./builddir/etc/cron.d/ +pip3 install -r ../requirements.txt -t ./builddir/opt/tisbackup/lib + rsync -aP --exclude "deb/" --exclude "doc/" --exclude "rpm/" --exclude ".git" ../ ./builddir/opt/tisbackup rsync -aP ../scripts/tisbackup_gui.service ./builddir/usr/lib/systemd/system/ rsync -aP ../scripts/tisbackup_huey.service ./builddir/usr/lib/systemd/system/ rsync -aP ../samples/tisbackup_gui.ini ./builddir/etc/tis rsync -aP ../samples/tisbackup-config.ini.sample ./builddir/etc/tis/tisbackup-config.ini.sample -rsync -aP ../lib/huey/bin/huey_consumer.py ./builddir/opt/tisbackup/ -chmod 755 /opt/tisbackup/tisbackup.py +chmod 755 ./builddir/opt/tisbackup/tisbackup.py -dpkg-deb --build builddir tis-tisbackup-1:${VERSION}.deb +dpkg-deb --build builddir tis-tisbackup-1-${VERSION}.deb diff --git a/deb/postinst b/deb/postinst index 8730a7b..592fdae 100755 --- a/deb/postinst +++ b/deb/postinst @@ -10,3 +10,5 @@ if [ ! -f /etc/cron.d/tisbackup ]; then cp /opt/tisbackup/samples/tisbackup.cron /etc/cron.d/tisbackup fi +find /opt/tisbackup -name "*.pyc" -exec rm -rf {} \; +python3 -m compileall /opt/tisbackup/ diff --git a/deb/postrm b/deb/postrm new file mode 100644 index 0000000..02d0482 --- /dev/null +++ b/deb/postrm @@ -0,0 +1,2 @@ +#!/bin/bash +rm -rf /opt/tisbackup diff --git a/deb/prerm b/deb/prerm new file mode 100644 index 0000000..b48174c --- /dev/null +++ b/deb/prerm @@ -0,0 +1,3 @@ +#!/bin/bash + +find /opt/tisbackup/ -name *.pyo -exec rm -f {} \; diff --git a/docs-sphinx-rst/source/installing_tisbackup.rst b/docs-sphinx-rst/source/installing_tisbackup.rst index 0e85f79..b822fd0 100644 --- a/docs-sphinx-rst/source/installing_tisbackup.rst +++ b/docs-sphinx-rst/source/installing_tisbackup.rst @@ -21,9 +21,9 @@ Installing and configuring TISBackup on Debian Setting up the GNU/Linux Debian server -------------------------------------- -In order to install a fresh Debian Linux 10 *Buster* (physical or virtual) +In order to install a fresh Debian Linux 11 *Bullseye* (physical or virtual) without graphical interface, please refer to the -`Debian GNU/Linux Installation Guide `_. +`Debian GNU/Linux Installation Guide `_. Configuring network parameters ++++++++++++++++++++++++++++++ @@ -138,14 +138,51 @@ and :ref:`install TISBackup on your Debian`. .. _install_tisbackup_debian: -Installing the TISBackup server on Debian Linux -+++++++++++++++++++++++++++++++++++++++++++++++ +Installing the TISBackup server ++++++++++++++++++++++++++++++++ + +From Tranquil IT's repository +""""""""""""""""""""""""""""" + +The easiest way is to install the package from Tranquil IT repository : + +.. tabs:: + + .. code-tab:: bash On CentOS8 and derivate + + wget https://srvinstallation.tranquil.it/tisbackup/tis-tisbackup-162-1.el8.x86_64.rpm -O tis-tisbackup.rpm + yum install -y tis-tisbackup.rpm + + .. code-tab:: bash On CentOS7 + + wget https://srvinstallation.tranquil.it/tisbackup/tis-tisbackup-162-1.el7.x86_64.rpm -O tis-tisbackup.rpm + yum install -y tis-tisbackup.rpm + + .. code-tab:: bash On Debian 11 + + wget https://srvinstallation.tranquil.it/tisbackup/tis-tisbackup-1-2.0.163-deb11.deb -O tis-tisbackup.deb + apt install unzip python3-paramiko python3-pyvmomi python3-pexpect python3-flask python3-simplejson python3-pip + dpkg -i tis-tisbackup.deb + + +From sources +"""""""""""" * install the required dependencies: - .. code-block:: bash +.. tabs:: - apt-get install unzip ssh rsync python-paramiko python-pyvmomi python-pexpect + .. code-tab:: bash On CentOS8 and derivate + + unzip, ssh, rsync, python3-paramiko, python3-pyvmomi, python3-pexpect, python3-flask,python3-simplejson, python3-pip + + .. code-tab:: bash On CentOS7 and derivate + + unzip rsync python3-paramiko python3-pyvmomi nfs-utils python3-flask python3-simplejson autofs python3-pexpect + + .. code-tab:: bash on Debian 11 + + unzip rsync python36-paramiko python3-pyvmomi nfs-utils python3-flask python3-simplejson autofs pexpect * retrieve the git sources from https://github.com/tranquilit/TISbackup and place them in the :file:`/opt` folder on your server: @@ -156,6 +193,7 @@ Installing the TISBackup server on Debian Linux wget --no-check-certificate https://github.com/tranquilit/TISbackup/archive/master.zip unzip master.zip mv TISbackup-master tisbackup + pip3 install huey iniparse -t /opt/tisbackup/lib chmod 755 /opt/tisbackup/tisbackup.py ln -sb /opt/tisbackup/tisbackup.py /usr/local/bin/tisbackup @@ -227,11 +265,6 @@ and :ref:`configure the backup jobs for your TISBackup` Setting up the graphical user interface for the TISBackup server ---------------------------------------------------------------- -.. code-block:: bash - - apt-get install python2.7 python-simplejson python-flask python-setuptools sudo - python /usr/lib/python2.7/dist-packages/easy_install.py "huey<=0.4.9" - .. code-block:: bash cp /opt/tisbackup/samples/tisbackup_gui.ini /etc/tis/ diff --git a/docs/_sources/installing_tisbackup.rst.txt b/docs/_sources/installing_tisbackup.rst.txt index 0e85f79..b822fd0 100644 --- a/docs/_sources/installing_tisbackup.rst.txt +++ b/docs/_sources/installing_tisbackup.rst.txt @@ -21,9 +21,9 @@ Installing and configuring TISBackup on Debian Setting up the GNU/Linux Debian server -------------------------------------- -In order to install a fresh Debian Linux 10 *Buster* (physical or virtual) +In order to install a fresh Debian Linux 11 *Bullseye* (physical or virtual) without graphical interface, please refer to the -`Debian GNU/Linux Installation Guide `_. +`Debian GNU/Linux Installation Guide `_. Configuring network parameters ++++++++++++++++++++++++++++++ @@ -138,14 +138,51 @@ and :ref:`install TISBackup on your Debian`. .. _install_tisbackup_debian: -Installing the TISBackup server on Debian Linux -+++++++++++++++++++++++++++++++++++++++++++++++ +Installing the TISBackup server ++++++++++++++++++++++++++++++++ + +From Tranquil IT's repository +""""""""""""""""""""""""""""" + +The easiest way is to install the package from Tranquil IT repository : + +.. tabs:: + + .. code-tab:: bash On CentOS8 and derivate + + wget https://srvinstallation.tranquil.it/tisbackup/tis-tisbackup-162-1.el8.x86_64.rpm -O tis-tisbackup.rpm + yum install -y tis-tisbackup.rpm + + .. code-tab:: bash On CentOS7 + + wget https://srvinstallation.tranquil.it/tisbackup/tis-tisbackup-162-1.el7.x86_64.rpm -O tis-tisbackup.rpm + yum install -y tis-tisbackup.rpm + + .. code-tab:: bash On Debian 11 + + wget https://srvinstallation.tranquil.it/tisbackup/tis-tisbackup-1-2.0.163-deb11.deb -O tis-tisbackup.deb + apt install unzip python3-paramiko python3-pyvmomi python3-pexpect python3-flask python3-simplejson python3-pip + dpkg -i tis-tisbackup.deb + + +From sources +"""""""""""" * install the required dependencies: - .. code-block:: bash +.. tabs:: - apt-get install unzip ssh rsync python-paramiko python-pyvmomi python-pexpect + .. code-tab:: bash On CentOS8 and derivate + + unzip, ssh, rsync, python3-paramiko, python3-pyvmomi, python3-pexpect, python3-flask,python3-simplejson, python3-pip + + .. code-tab:: bash On CentOS7 and derivate + + unzip rsync python3-paramiko python3-pyvmomi nfs-utils python3-flask python3-simplejson autofs python3-pexpect + + .. code-tab:: bash on Debian 11 + + unzip rsync python36-paramiko python3-pyvmomi nfs-utils python3-flask python3-simplejson autofs pexpect * retrieve the git sources from https://github.com/tranquilit/TISbackup and place them in the :file:`/opt` folder on your server: @@ -156,6 +193,7 @@ Installing the TISBackup server on Debian Linux wget --no-check-certificate https://github.com/tranquilit/TISbackup/archive/master.zip unzip master.zip mv TISbackup-master tisbackup + pip3 install huey iniparse -t /opt/tisbackup/lib chmod 755 /opt/tisbackup/tisbackup.py ln -sb /opt/tisbackup/tisbackup.py /usr/local/bin/tisbackup @@ -227,11 +265,6 @@ and :ref:`configure the backup jobs for your TISBackup` Setting up the graphical user interface for the TISBackup server ---------------------------------------------------------------- -.. code-block:: bash - - apt-get install python2.7 python-simplejson python-flask python-setuptools sudo - python /usr/lib/python2.7/dist-packages/easy_install.py "huey<=0.4.9" - .. code-block:: bash cp /opt/tisbackup/samples/tisbackup_gui.ini /etc/tis/ diff --git a/lib/huey/__init__.py b/lib/huey/__init__.py deleted file mode 100644 index 5a4db0a..0000000 --- a/lib/huey/__init__.py +++ /dev/null @@ -1,62 +0,0 @@ -__author__ = 'Charles Leifer' -__license__ = 'MIT' -__version__ = '0.4.9' - -from huey.api import Huey, crontab - -try: - import redis - from huey.backends.redis_backend import RedisBlockingQueue - from huey.backends.redis_backend import RedisDataStore - from huey.backends.redis_backend import RedisEventEmitter - from huey.backends.redis_backend import RedisSchedule - - class RedisHuey(Huey): - def __init__(self, name='huey', store_none=False, always_eager=False, - read_timeout=None, **conn_kwargs): - queue = RedisBlockingQueue( - name, - read_timeout=read_timeout, - **conn_kwargs) - result_store = RedisDataStore(name, **conn_kwargs) - schedule = RedisSchedule(name, **conn_kwargs) - events = RedisEventEmitter(name, **conn_kwargs) - super(RedisHuey, self).__init__( - queue=queue, - result_store=result_store, - schedule=schedule, - events=events, - store_none=store_none, - always_eager=always_eager) - -except ImportError: - class RedisHuey(object): - def __init__(self, *args, **kwargs): - raise RuntimeError('Error, "redis" is not installed. Install ' - 'using pip: "pip install redis"') - -try: - from huey.backends.sqlite_backend import SqliteQueue - from huey.backends.sqlite_backend import SqliteDataStore - from huey.backends.sqlite_backend import SqliteSchedule - - class SqliteHuey(Huey): - def __init__(self, name='huey', store_none=False, always_eager=False, - location=None): - if location is None: - raise ValueError("Please specify a database file with the " - "'location' parameter") - queue = SqliteQueue(name, location) - result_store = SqliteDataStore(name, location) - schedule = SqliteSchedule(name, location) - super(SqliteHuey, self).__init__( - queue=queue, - result_store=result_store, - schedule=schedule, - events=None, - store_none=store_none, - always_eager=always_eager) -except ImportError: - class SqliteHuey(object): - def __init__(self, *args, **kwargs): - raise RuntimeError('Error, "sqlite" is not installed.') diff --git a/lib/huey/api.py b/lib/huey/api.py deleted file mode 100644 index 3cc9490..0000000 --- a/lib/huey/api.py +++ /dev/null @@ -1,513 +0,0 @@ -import datetime -import json -import pickle -import re -import time -import traceback -import uuid -from functools import wraps - -from huey.backends.dummy import DummySchedule -from huey.exceptions import DataStoreGetException -from huey.exceptions import DataStorePutException -from huey.exceptions import DataStoreTimeout -from huey.exceptions import QueueException -from huey.exceptions import QueueReadException -from huey.exceptions import QueueRemoveException -from huey.exceptions import QueueWriteException -from huey.exceptions import ScheduleAddException -from huey.exceptions import ScheduleReadException -from huey.registry import registry -from huey.utils import EmptyData -from huey.utils import local_to_utc -from huey.utils import wrap_exception - - -class Huey(object): - """ - Huey executes tasks by exposing function decorators that cause the function - call to be enqueued for execution by the consumer. - - Typically your application will only need one Huey instance, but you can - have as many as you like -- the only caveat is that one consumer process - must be executed for each Huey instance. - - :param queue: a queue instance, e.g. ``RedisQueue()`` - :param result_store: a place to store results, e.g. ``RedisResultStore()`` - :param schedule: a place to store pending tasks, e.g. ``RedisSchedule()`` - :param events: channel to send events on, e.g. ``RedisEventEmitter()`` - :param store_none: Flag to indicate whether tasks that return ``None`` - should store their results in the result store. - :param always_eager: Useful for testing, this will execute all tasks - immediately, without enqueueing them. - - Example usage:: - - from huey.api import Huey, crontab - from huey.backends.redis_backend import RedisQueue, RedisDataStore, RedisSchedule - - queue = RedisQueue('my-app') - result_store = RedisDataStore('my-app') - schedule = RedisSchedule('my-app') - huey = Huey(queue, result_store, schedule) - - # This is equivalent to the previous 4 lines: - # huey = RedisHuey('my-app', {'host': 'localhost', 'port': 6379}) - - @huey.task() - def slow_function(some_arg): - # ... do something ... - return some_arg - - @huey.periodic_task(crontab(minute='0', hour='3')) - def backup(): - # do a backup every day at 3am - return - """ - def __init__(self, queue, result_store=None, schedule=None, events=None, - store_none=False, always_eager=False): - self.queue = queue - self.result_store = result_store - self.schedule = schedule or DummySchedule(self.queue.name) - self.events = events - self.blocking = self.queue.blocking - self.store_none = store_none - self.always_eager = always_eager - - def task(self, retries=0, retry_delay=0, retries_as_argument=False, - include_task=False, name=None): - def decorator(func): - """ - Decorator to execute a function out-of-band via the consumer. - """ - klass = create_task( - QueueTask, - func, - retries_as_argument, - name, - include_task) - - def schedule(args=None, kwargs=None, eta=None, delay=None, - convert_utc=True, task_id=None): - if delay and eta: - raise ValueError('Both a delay and an eta cannot be ' - 'specified at the same time') - if delay: - eta = (datetime.datetime.now() + - datetime.timedelta(seconds=delay)) - if convert_utc and eta: - eta = local_to_utc(eta) - cmd = klass( - (args or (), kwargs or {}), - execute_time=eta, - retries=retries, - retry_delay=retry_delay, - task_id=task_id) - return self.enqueue(cmd) - - func.schedule = schedule - func.task_class = klass - - @wraps(func) - def inner_run(*args, **kwargs): - cmd = klass( - (args, kwargs), - retries=retries, - retry_delay=retry_delay) - return self.enqueue(cmd) - - inner_run.call_local = func - return inner_run - return decorator - - def periodic_task(self, validate_datetime, name=None): - """ - Decorator to execute a function on a specific schedule. - """ - def decorator(func): - def method_validate(self, dt): - return validate_datetime(dt) - - klass = create_task( - PeriodicQueueTask, - func, - task_name=name, - validate_datetime=method_validate, - ) - - func.task_class = klass - - def _revoke(revoke_until=None, revoke_once=False): - self.revoke(klass(), revoke_until, revoke_once) - func.revoke = _revoke - - def _is_revoked(dt=None, peek=True): - return self.is_revoked(klass(), dt, peek) - func.is_revoked = _is_revoked - - def _restore(): - return self.restore(klass()) - func.restore = _restore - - return func - return decorator - - def _wrapped_operation(exc_class): - def decorator(fn): - def inner(*args, **kwargs): - try: - return fn(*args, **kwargs) - except: - wrap_exception(exc_class) - return inner - return decorator - - @_wrapped_operation(QueueWriteException) - def _write(self, msg): - self.queue.write(msg) - - @_wrapped_operation(QueueReadException) - def _read(self): - return self.queue.read() - - @_wrapped_operation(QueueRemoveException) - def _remove(self, msg): - return self.queue.remove(msg) - - @_wrapped_operation(DataStoreGetException) - def _get(self, key, peek=False): - if peek: - return self.result_store.peek(key) - else: - return self.result_store.get(key) - - @_wrapped_operation(DataStorePutException) - def _put(self, key, value): - return self.result_store.put(key, value) - - @_wrapped_operation(ScheduleAddException) - def _add_schedule(self, data, ts): - if self.schedule is None: - raise AttributeError('Schedule not specified.') - self.schedule.add(data, ts) - - @_wrapped_operation(ScheduleReadException) - def _read_schedule(self, ts): - if self.schedule is None: - raise AttributeError('Schedule not specified.') - return self.schedule.read(ts) - - def emit(self, message): - """Events should always fail silently.""" - try: - self.events.emit(message) - except: - pass - - def enqueue(self, task): - if self.always_eager: - return task.execute() - - self._write(registry.get_message_for_task(task)) - - if self.result_store: - return AsyncData(self, task) - - def dequeue(self): - message = self._read() - if message: - return registry.get_task_for_message(message) - - def _format_time(self, dt): - if dt is None: - return None - return time.mktime(dt.timetuple()) - - def emit_task(self, status, task, error=False): - if self.events: - message_data = {'status': status} - message_data.update({ - 'id': task.task_id, - 'task': type(task).__name__, - 'retries': task.retries, - 'retry_delay': task.retry_delay, - 'execute_time': self._format_time(task.execute_time), - 'error': error}) - if error: - message_data['traceback'] = traceback.format_exc() - self.emit(json.dumps(message_data)) - - def execute(self, task): - if not isinstance(task, QueueTask): - raise TypeError('Unknown object: %s' % task) - - result = task.execute() - - if result is None and not self.store_none: - return - - if self.result_store and not isinstance(task, PeriodicQueueTask): - self._put(task.task_id, pickle.dumps(result)) - - return result - - def revoke(self, task, revoke_until=None, revoke_once=False): - if not self.result_store: - raise QueueException('A DataStore is required to revoke task') - - serialized = pickle.dumps((revoke_until, revoke_once)) - self._put(task.revoke_id, serialized) - - def restore(self, task): - self._get(task.revoke_id) # simply get and delete if there - - def is_revoked(self, task, dt=None, peek=True): - if not self.result_store: - return False - res = self._get(task.revoke_id, peek=True) - if res is EmptyData: - return False - revoke_until, revoke_once = pickle.loads(res) - if revoke_once: - # This task *was* revoked for one run, but now it should be - # restored to normal execution. - if not peek: - self.restore(task) - return True - return revoke_until is None or revoke_until > dt - - def add_schedule(self, task): - msg = registry.get_message_for_task(task) - ex_time = task.execute_time or datetime.datetime.fromtimestamp(0) - self._add_schedule(msg, ex_time) - - def read_schedule(self, ts): - return [ - registry.get_task_for_message(m) for m in self._read_schedule(ts)] - - def flush(self): - self.queue.flush() - - def ready_to_run(self, cmd, dt=None): - dt = dt or datetime.datetime.utcnow() - return cmd.execute_time is None or cmd.execute_time <= dt - - -class AsyncData(object): - def __init__(self, huey, task): - self.huey = huey - self.task = task - - self._result = EmptyData - - def _get(self): - task_id = self.task.task_id - if self._result is EmptyData: - res = self.huey._get(task_id) - - if res is not EmptyData: - self._result = pickle.loads(res) - return self._result - else: - return res - else: - return self._result - - def get(self, blocking=False, timeout=None, backoff=1.15, max_delay=1.0, - revoke_on_timeout=False): - if not blocking: - res = self._get() - if res is not EmptyData: - return res - else: - start = time.time() - delay = .1 - while self._result is EmptyData: - if timeout and time.time() - start >= timeout: - if revoke_on_timeout: - self.revoke() - raise DataStoreTimeout - if delay > max_delay: - delay = max_delay - if self._get() is EmptyData: - time.sleep(delay) - delay *= backoff - - return self._result - - def revoke(self): - self.huey.revoke(self.task) - - def restore(self): - self.huey.restore(self.task) - - -def with_metaclass(meta, base=object): - return meta("NewBase", (base,), {}) - - -class QueueTaskMetaClass(type): - def __init__(cls, name, bases, attrs): - """ - Metaclass to ensure that all task classes are registered - """ - registry.register(cls) - - -class QueueTask(with_metaclass(QueueTaskMetaClass)): - """ - A class that encapsulates the logic necessary to 'do something' given some - arbitrary data. When enqueued with the :class:`Huey`, it will be - stored in a queue for out-of-band execution via the consumer. See also - the :meth:`task` decorator, which can be used to automatically - execute any function out-of-band. - - Example:: - - class SendEmailTask(QueueTask): - def execute(self): - data = self.get_data() - send_email(data['recipient'], data['subject'], data['body']) - - huey.enqueue( - SendEmailTask({ - 'recipient': 'somebody@spam.com', - 'subject': 'look at this awesome website', - 'body': 'http://youtube.com' - }) - ) - """ - - def __init__(self, data=None, task_id=None, execute_time=None, retries=0, - retry_delay=0): - self.set_data(data) - self.task_id = task_id or self.create_id() - self.revoke_id = 'r:%s' % self.task_id - self.execute_time = execute_time - self.retries = retries - self.retry_delay = retry_delay - - def create_id(self): - return str(uuid.uuid4()) - - def get_data(self): - return self.data - - def set_data(self, data): - self.data = data - - def execute(self): - """Execute any arbitary code here""" - raise NotImplementedError - - def __eq__(self, rhs): - return ( - self.task_id == rhs.task_id and - self.execute_time == rhs.execute_time and - type(self) == type(rhs)) - - -class PeriodicQueueTask(QueueTask): - def create_id(self): - return registry.task_to_string(type(self)) - - def validate_datetime(self, dt): - """Validate that the task should execute at the given datetime""" - return False - - -def create_task(task_class, func, retries_as_argument=False, task_name=None, - include_task=False, **kwargs): - def execute(self): - args, kwargs = self.data or ((), {}) - if retries_as_argument: - kwargs['retries'] = self.retries - if include_task: - kwargs['task'] = self - return func(*args, **kwargs) - - attrs = { - 'execute': execute, - '__module__': func.__module__, - '__doc__': func.__doc__ - } - attrs.update(kwargs) - - klass = type( - task_name or 'queuecmd_%s' % (func.__name__), - (task_class,), - attrs - ) - - return klass - -dash_re = re.compile('(\d+)-(\d+)') -every_re = re.compile('\*\/(\d+)') - -def crontab(month='*', day='*', day_of_week='*', hour='*', minute='*'): - """ - Convert a "crontab"-style set of parameters into a test function that will - return True when the given datetime matches the parameters set forth in - the crontab. - - Acceptable inputs: - * = every distinct value - */n = run every "n" times, i.e. hours='*/4' == 0, 4, 8, 12, 16, 20 - m-n = run every time m..n - m,n = run on m and n - """ - validation = ( - ('m', month, range(1, 13)), - ('d', day, range(1, 32)), - ('w', day_of_week, range(7)), - ('H', hour, range(24)), - ('M', minute, range(60)) - ) - cron_settings = [] - - for (date_str, value, acceptable) in validation: - settings = set([]) - - if isinstance(value, int): - value = str(value) - - for piece in value.split(','): - if piece == '*': - settings.update(acceptable) - continue - - if piece.isdigit(): - piece = int(piece) - if piece not in acceptable: - raise ValueError('%d is not a valid input' % piece) - settings.add(piece) - - else: - dash_match = dash_re.match(piece) - if dash_match: - lhs, rhs = map(int, dash_match.groups()) - if lhs not in acceptable or rhs not in acceptable: - raise ValueError('%s is not a valid input' % piece) - settings.update(range(lhs, rhs+1)) - continue - - every_match = every_re.match(piece) - if every_match: - interval = int(every_match.groups()[0]) - settings.update(acceptable[::interval]) - - cron_settings.append(sorted(list(settings))) - - def validate_date(dt): - _, m, d, H, M, _, w, _, _ = dt.timetuple() - - # fix the weekday to be sunday=0 - w = (w + 1) % 7 - - for (date_piece, selection) in zip([m, d, w, H, M], cron_settings): - if date_piece not in selection: - return False - - return True - - return validate_date diff --git a/lib/huey/backends/__init__.py b/lib/huey/backends/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/lib/huey/backends/base.py b/lib/huey/backends/base.py deleted file mode 100644 index ee7da35..0000000 --- a/lib/huey/backends/base.py +++ /dev/null @@ -1,113 +0,0 @@ -class BaseQueue(object): - """ - Base implementation for a Queue, all backends should subclass - """ - - # whether this backend blocks while waiting for new results or should be - # polled by the consumer - blocking = False - - def __init__(self, name, **connection): - """ - Initialize the Queue - this happens once when the module is loaded - - :param name: A string representation of the name for this queue - :param connection: Connection parameters for the queue - """ - self.name = name - self.connection = connection - - def write(self, data): - """ - Push 'data' onto the queue - """ - raise NotImplementedError - - def read(self): - """ - Pop 'data' from the queue, returning None if no data is available -- - an empty queue should not raise an Exception! - """ - raise NotImplementedError - - def remove(self, data): - """ - Remove the given data from the queue - """ - raise NotImplementedError - - def flush(self): - """ - Delete everything from the queue - """ - raise NotImplementedError - - def __len__(self): - """ - Used primarily in tests, but return the number of items in the queue - """ - raise NotImplementedError - - -class BaseSchedule(object): - def __init__(self, name, **connection): - """ - Initialize the Queue - this happens once when the module is loaded - - :param name: A string representation of the name for this queue - :param connection: Connection parameters for the queue - """ - self.name = name - self.connection = connection - - def add(self, data, ts): - """ - Add the timestamped data to the task schedule. - """ - raise NotImplementedError - - def read(self, ts): - """ - Read scheduled items for the given timestamp - """ - raise NotImplementedError - - def flush(self): - """Delete all items in schedule.""" - raise NotImplementedError - - -class BaseDataStore(object): - """ - Base implementation for a data store - """ - def __init__(self, name, **connection): - """ - Initialize the data store - """ - self.name = name - self.connection = connection - - def put(self, key, value): - raise NotImplementedError - - def peek(self, key): - raise NotImplementedError - - def get(self, key): - raise NotImplementedError - - def flush(self): - raise NotImplementedError - - -class BaseEventEmitter(object): - def __init__(self, channel, **connection): - self.channel = channel - self.connection = connection - - def emit(self, message): - raise NotImplementedError - - -Components = (BaseQueue, BaseDataStore, BaseSchedule, BaseEventEmitter) diff --git a/lib/huey/backends/dummy.py b/lib/huey/backends/dummy.py deleted file mode 100644 index 9b1b696..0000000 --- a/lib/huey/backends/dummy.py +++ /dev/null @@ -1,103 +0,0 @@ -""" -Test-only implementations of Queue and DataStore. These will not work for -real applications because they only store tasks/results in memory. -""" -from collections import deque -import heapq - -from huey.backends.base import BaseDataStore -from huey.backends.base import BaseEventEmitter -from huey.backends.base import BaseQueue -from huey.backends.base import BaseSchedule -from huey.utils import EmptyData - - -class DummyQueue(BaseQueue): - def __init__(self, *args, **kwargs): - super(DummyQueue, self).__init__(*args, **kwargs) - self._queue = [] - - def write(self, data): - self._queue.insert(0, data) - - def read(self): - try: - return self._queue.pop() - except IndexError: - return None - - def flush(self): - self._queue = [] - - def remove(self, data): - clone = [] - ct = 0 - for elem in self._queue: - if elem == data: - ct += 1 - else: - clone.append(elem) - self._queue = clone - return ct - - def __len__(self): - return len(self._queue) - - -class DummySchedule(BaseSchedule): - def __init__(self, *args, **kwargs): - super(DummySchedule, self).__init__(*args, **kwargs) - self._schedule = [] - - def add(self, data, ts): - heapq.heappush(self._schedule, (ts, data)) - - def read(self, ts): - res = [] - while len(self._schedule): - sts, data = heapq.heappop(self._schedule) - if sts <= ts: - res.append(data) - else: - self.add(data, sts) - break - return res - - def flush(self): - self._schedule = [] - - -class DummyDataStore(BaseDataStore): - def __init__(self, *args, **kwargs): - super(DummyDataStore, self).__init__(*args, **kwargs) - self._results = {} - - def put(self, key, value): - self._results[key] = value - - def peek(self, key): - return self._results.get(key, EmptyData) - - def get(self, key): - return self._results.pop(key, EmptyData) - - def flush(self): - self._results = {} - - -class DummyEventEmitter(BaseEventEmitter): - def __init__(self, *args, **kwargs): - super(DummyEventEmitter, self).__init__(*args, **kwargs) - self._events = deque() - self.__size = 100 - - def emit(self, message): - self._events.appendleft(message) - num_events = len(self._events) - if num_events > self.__size * 1.5: - while num_events > self.__size: - self._events.popright() - num_events -= 1 - - -Components = (DummyQueue, DummyDataStore, DummySchedule, DummyEventEmitter) diff --git a/lib/huey/backends/rabbitmq_backend.py b/lib/huey/backends/rabbitmq_backend.py deleted file mode 100644 index c23cb4d..0000000 --- a/lib/huey/backends/rabbitmq_backend.py +++ /dev/null @@ -1,153 +0,0 @@ -# -*- coding: utf-8 -*- -__author__ = 'deathowl' - -import datetime -import re -import time -import pika -from pika.exceptions import AMQPConnectionError - -from huey.backends.base import BaseEventEmitter -from huey.backends.base import BaseQueue - - -def clean_name(name): - return re.sub('[^a-z0-9]', '', name) - - -class RabbitQueue(BaseQueue): - """ - A simple Queue that uses the rabbit to store messages - """ - def __init__(self, name, **connection): - """ - connection = { - 'host': 'localhost', - 'port': 5672, - 'username': 'guest', - 'password': 'guest', - 'vhost': '/', - 'ssl': False - } - """ - super(RabbitQueue, self).__init__(name, **connection) - - self.queue_name = 'huey.rabbit.%s' % clean_name(name) - credentials = pika.PlainCredentials( - connection.get('username', 'guest'), - connection.get('password', 'guest')) - connection_params = pika.ConnectionParameters( - host=connection.get('host', 'localhost'), - port=connection.get('port', 5672), - credentials=credentials, - virtual_host=connection.get('vhost', '/'), - ssl=connection.get('ssl', False)) - - self.conn = pika.BlockingConnection(connection_params) - self.channel = self.conn.channel() - self.channel.queue_declare(self.queue_name, durable=True) - - def write(self, data): - self.channel.basic_publish( - exchange='', - routing_key=self.queue_name, - body=data) - - def read(self): - return self.get_data_from_queue(self.queue_name) - - def remove(self, data): - # This is not something you usually do in rabbit, this is the only - # operation, which is not atomic, but this "hack" should do the trick. - amount = 0 - idx = 0 - qlen = len(self) - - for method_frame, _, body in self.channel.consume(self.queue_name): - idx += 1 - if body == data: - self.channel.basic_ack(method_frame.delivery_tag) - amount += 1 - else: - self.channel.basic_nack( - method_frame.delivery_tag, - requeue=True) - - if idx >= qlen: - break - - self.channel.cancel() - return amount - - def flush(self): - self.channel.queue_purge(queue=self.queue_name) - return True - - def __len__(self): - queue = self.channel.queue_declare(self.queue_name, durable=True) - return queue.method.message_count - - def get_data_from_queue(self, queue): - data = None - if len(self) == 0: - return None - - for method_frame, _, body in self.channel.consume(queue): - data = body - self.channel.basic_ack(method_frame.delivery_tag) - break - - self.channel.cancel() - return data - - -class RabbitBlockingQueue(RabbitQueue): - """ - Use the blocking right pop, should result in messages getting - executed close to immediately by the consumer as opposed to - being polled for - """ - blocking = True - - def read(self): - try: - return self.get_data_from_queue(self.queue_name) - except AMQPConnectionError: - return None - - -class RabbitEventEmitter(BaseEventEmitter): - def __init__(self, channel, **connection): - super(RabbitEventEmitter, self).__init__(channel, **connection) - credentials = pika.PlainCredentials( - connection.get('username', 'guest'), - connection.get('password', 'guest')) - connection_params = pika.ConnectionParameters( - host=connection.get('host', 'localhost'), - port=connection.get('port', 5672), - credentials=credentials, - virtual_host=connection.get('vhost', '/'), - ssl=connection.get('ssl', False)) - - self.conn = pika.BlockingConnection(connection_params) - self.channel = self.conn.channel() - self.exchange_name = 'huey.events' - self.channel.exchange_declare( - exchange=self.exchange_name, - type='fanout', - auto_delete=False, - durable=True) - - def emit(self, message): - properties = pika.BasicProperties( - content_type="text/plain", - delivery_mode=2) - - self.channel.basic_publish( - exchange=self.exchange_name, - routing_key='', - body=message, - properties=properties) - - -Components = (RabbitBlockingQueue, None, None, RabbitEventEmitter) diff --git a/lib/huey/backends/redis_backend.py b/lib/huey/backends/redis_backend.py deleted file mode 100644 index e92ab13..0000000 --- a/lib/huey/backends/redis_backend.py +++ /dev/null @@ -1,153 +0,0 @@ -import re -import time - -import redis -from redis.exceptions import ConnectionError - -from huey.backends.base import BaseDataStore -from huey.backends.base import BaseEventEmitter -from huey.backends.base import BaseQueue -from huey.backends.base import BaseSchedule -from huey.utils import EmptyData - - -def clean_name(name): - return re.sub('[^a-z0-9]', '', name) - - -class RedisQueue(BaseQueue): - """ - A simple Queue that uses the redis to store messages - """ - def __init__(self, name, **connection): - """ - connection = { - 'host': 'localhost', - 'port': 6379, - 'db': 0, - } - """ - super(RedisQueue, self).__init__(name, **connection) - - self.queue_name = 'huey.redis.%s' % clean_name(name) - self.conn = redis.Redis(**connection) - - def write(self, data): - self.conn.lpush(self.queue_name, data) - - def read(self): - return self.conn.rpop(self.queue_name) - - def remove(self, data): - return self.conn.lrem(self.queue_name, data) - - def flush(self): - self.conn.delete(self.queue_name) - - def __len__(self): - return self.conn.llen(self.queue_name) - - -class RedisBlockingQueue(RedisQueue): - """ - Use the blocking right pop, should result in messages getting - executed close to immediately by the consumer as opposed to - being polled for - """ - blocking = True - - def __init__(self, name, read_timeout=None, **connection): - """ - connection = { - 'host': 'localhost', - 'port': 6379, - 'db': 0, - } - """ - super(RedisBlockingQueue, self).__init__(name, **connection) - self.read_timeout = read_timeout - - def read(self): - try: - return self.conn.brpop( - self.queue_name, - timeout=self.read_timeout)[1] - except (ConnectionError, TypeError, IndexError): - # unfortunately, there is no way to differentiate a socket timing - # out and a host being unreachable - return None - - -# a custom lua script to pass to redis that will read tasks from the schedule -# and atomically pop them from the sorted set and return them. -# it won't return anything if it isn't able to remove the items it reads. -SCHEDULE_POP_LUA = """ -local key = KEYS[1] -local unix_ts = ARGV[1] -local res = redis.call('zrangebyscore', key, '-inf', unix_ts) -if #res and redis.call('zremrangebyscore', key, '-inf', unix_ts) == #res then - return res -end -""" - -class RedisSchedule(BaseSchedule): - def __init__(self, name, **connection): - super(RedisSchedule, self).__init__(name, **connection) - - self.key = 'huey.schedule.%s' % clean_name(name) - self.conn = redis.Redis(**connection) - self._pop = self.conn.register_script(SCHEDULE_POP_LUA) - - def convert_ts(self, ts): - return time.mktime(ts.timetuple()) - - def add(self, data, ts): - self.conn.zadd(self.key, data, self.convert_ts(ts)) - - def read(self, ts): - unix_ts = self.convert_ts(ts) - # invoke the redis lua script that will atomically pop off - # all the tasks older than the given timestamp - tasks = self._pop(keys=[self.key], args=[unix_ts]) - return [] if tasks is None else tasks - - def flush(self): - self.conn.delete(self.key) - - -class RedisDataStore(BaseDataStore): - def __init__(self, name, **connection): - super(RedisDataStore, self).__init__(name, **connection) - - self.storage_name = 'huey.results.%s' % clean_name(name) - self.conn = redis.Redis(**connection) - - def put(self, key, value): - self.conn.hset(self.storage_name, key, value) - - def peek(self, key): - if self.conn.hexists(self.storage_name, key): - return self.conn.hget(self.storage_name, key) - return EmptyData - - def get(self, key): - val = self.peek(key) - if val is not EmptyData: - self.conn.hdel(self.storage_name, key) - return val - - def flush(self): - self.conn.delete(self.storage_name) - - -class RedisEventEmitter(BaseEventEmitter): - def __init__(self, channel, **connection): - super(RedisEventEmitter, self).__init__(channel, **connection) - self.conn = redis.Redis(**connection) - - def emit(self, message): - self.conn.publish(self.channel, message) - - -Components = (RedisBlockingQueue, RedisDataStore, RedisSchedule, - RedisEventEmitter) diff --git a/lib/huey/backends/sqlite_backend.py b/lib/huey/backends/sqlite_backend.py deleted file mode 100644 index dd801c9..0000000 --- a/lib/huey/backends/sqlite_backend.py +++ /dev/null @@ -1,205 +0,0 @@ -""" SQLite backend for Huey. - -Inspired from a snippet by Thiago Arruda [1] - -[1] http://flask.pocoo.org/snippets/88/ -""" -import json -import sqlite3 -import time -try: - from thread import get_ident -except ImportError: # Python 3 - try: - from threading import get_ident - except ImportError: - from _thread import get_ident - buffer = memoryview - -from huey.backends.base import BaseDataStore -from huey.backends.base import BaseEventEmitter -from huey.backends.base import BaseQueue -from huey.backends.base import BaseSchedule -from huey.utils import EmptyData - - -class _SqliteDatabase(object): - def __init__(self, location): - if location == ':memory:': - raise ValueError("Database location has to be a file path, " - "in-memory databases are not supported.") - self.location = location - self._conn_cache = {} - with self.get_connection() as conn: - # Enable write-ahead logging - conn.execute("PRAGMA journal_mode=WAL;") - # Hand over syncing responsibility to OS - conn.execute("PRAGMA synchronous=OFF;") - # Store temporary tables and indices in memory - conn.execute("PRAGMA temp_store=MEMORY;") - - def get_connection(self, immediate=False): - """ Obtain a sqlite3.Connection instance for the database. - - Connections are cached on a by-thread basis, i.e. every calling thread - will always get the same Connection object back. - """ - if immediate: - return sqlite3.Connection(self.location, timeout=60, - isolation_level="IMMEDIATE") - id = get_ident() - if id not in self._conn_cache: - self._conn_cache[id] = sqlite3.Connection( - self.location, timeout=60) - return self._conn_cache[id] - - -class SqliteQueue(BaseQueue): - """ - A simple Queue that uses SQLite to store messages - """ - _create = """ - CREATE TABLE IF NOT EXISTS {0} - ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - item BLOB - ) - """ - _count = "SELECT COUNT(*) FROM {0}" - _append = "INSERT INTO {0} (item) VALUES (?)" - _get = "SELECT id, item FROM {0} ORDER BY id LIMIT 1" - _remove_by_value = "DELETE FROM {0} WHERE item = ?" - _remove_by_id = "DELETE FROM {0} WHERE id = ?" - _flush = "DELETE FROM {0}" - - def __init__(self, name, location): - super(SqliteQueue, self).__init__(name, location=location) - self.queue_name = 'huey_queue_{0}'.format(name) - self._db = _SqliteDatabase(location) - with self._db.get_connection() as conn: - conn.execute(self._create.format(self.queue_name)) - - def write(self, data): - with self._db.get_connection() as conn: - conn.execute(self._append.format(self.queue_name), (data,)) - - def read(self): - with self._db.get_connection(immediate=True) as conn: - cursor = conn.execute(self._get.format(self.queue_name)) - try: - id, data = next(cursor) - except StopIteration: - return None - if id: - conn.execute(self._remove_by_id.format(self.queue_name), (id,)) - return data - - def remove(self, data): - with self._db.get_connection() as conn: - return conn.execute(self._remove_by_value.format(self.queue_name), - (data,)).rowcount - - def flush(self): - with self._db.get_connection() as conn: - conn.execute(self._flush.format(self.queue_name,)) - - def __len__(self): - with self._db.get_connection() as conn: - return next(conn.execute(self._count.format(self.queue_name)))[0] - - -class SqliteSchedule(BaseSchedule): - _create = """ - CREATE TABLE IF NOT EXISTS {0} - ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - item BLOB, - timestamp INTEGER - ) - """ - _read_items = """ - SELECT item, timestamp FROM {0} WHERE timestamp <= ? - ORDER BY timestamp - """ - _delete_items = "DELETE FROM {0} WHERE timestamp <= ?" - _add_item = "INSERT INTO {0} (item, timestamp) VALUES (?, ?)" - _flush = "DELETE FROM {0}" - - def __init__(self, name, location): - super(SqliteSchedule, self).__init__(name, location=location) - self._db = _SqliteDatabase(location) - self.name = 'huey_schedule_{0}'.format(name) - with self._db.get_connection() as conn: - conn.execute(self._create.format(self.name)) - - def convert_ts(self, ts): - return time.mktime(ts.timetuple()) - - def add(self, data, ts): - with self._db.get_connection() as conn: - conn.execute(self._add_item.format(self.name), - (data, self.convert_ts(ts))) - - def read(self, ts): - with self._db.get_connection() as conn: - results = conn.execute(self._read_items.format(self.name), - (self.convert_ts(ts),)).fetchall() - conn.execute(self._delete_items.format(self.name), - (self.convert_ts(ts),)) - return [data for data, _ in results] - - def flush(self): - with self._db.get_connection() as conn: - conn.execute(self._flush.format(self.name)) - - -class SqliteDataStore(BaseDataStore): - _create = """ - CREATE TABLE IF NOT EXISTS {0} - ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - key TEXT, - result BLOB - ) - """ - _put = "INSERT INTO {0} (key, result) VALUES (?, ?)" - _peek = "SELECT result FROM {0} WHERE key = ?" - _remove = "DELETE FROM {0} WHERE key = ?" - _flush = "DELETE FROM {0}" - - def __init__(self, name, location): - super(SqliteDataStore, self).__init__(name, location=location) - self._db = _SqliteDatabase(location) - self.name = 'huey_results_{0}'.format(name) - with self._db.get_connection() as conn: - conn.execute(self._create.format(self.name)) - - def put(self, key, value): - with self._db.get_connection() as conn: - conn.execute(self._remove.format(self.name), (key,)) - conn.execute(self._put.format(self.name), (key, value)) - - def peek(self, key): - with self._db.get_connection() as conn: - try: - return next(conn.execute(self._peek.format(self.name), - (key,)))[0] - except StopIteration: - return EmptyData - - def get(self, key): - with self._db.get_connection() as conn: - try: - data = next(conn.execute(self._peek.format(self.name), - (key,)))[0] - conn.execute(self._remove.format(self.name), (key,)) - return data - except StopIteration: - return EmptyData - - def flush(self): - with self._db.get_connection() as conn: - conn.execute(self._flush.format(self.name)) - - -Components = (SqliteQueue, SqliteDataStore, SqliteSchedule, None) diff --git a/lib/huey/bin/__init__.py b/lib/huey/bin/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/lib/huey/bin/huey_consumer.py b/lib/huey/bin/huey_consumer.py deleted file mode 100755 index be53a6b..0000000 --- a/lib/huey/bin/huey_consumer.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python - -import logging -import optparse -import os -import sys -from logging.handlers import RotatingFileHandler - -from huey.consumer import Consumer -from huey.utils import load_class - - -def err(s): - sys.stderr.write('\033[91m%s\033[0m\n' % s) - - -def get_loglevel(verbose=None): - if verbose is None: - return logging.INFO - elif verbose: - return logging.DEBUG - return logging.ERROR - - -def setup_logger(loglevel, logfile): - log_format = ('%(threadName)s %(asctime)s %(name)s ' - '%(levelname)s %(message)s') - logging.basicConfig(level=loglevel, format=log_format) - - if logfile: - handler = RotatingFileHandler( - logfile, maxBytes=1024*1024, backupCount=3) - handler.setFormatter(logging.Formatter(log_format)) - logging.getLogger().addHandler(handler) - - -def get_option_parser(): - parser = optparse.OptionParser( - 'Usage: %prog [options] path.to.huey_instance') - parser.add_option('-l', '--logfile', dest='logfile', - help='write logs to FILE', metavar='FILE') - parser.add_option('-v', '--verbose', dest='verbose', - help='verbose logging', action='store_true') - parser.add_option('-q', '--quiet', dest='verbose', - help='log exceptions only', action='store_false') - parser.add_option('-w', '--workers', dest='workers', type='int', - help='worker threads (default=1)', default=1) - parser.add_option('-t', '--threads', dest='workers', type='int', - help='same as "workers"', default=1) - parser.add_option('-p', '--periodic', dest='periodic', default=True, - help='execute periodic tasks (default=True)', - action='store_true') - parser.add_option('-n', '--no-periodic', dest='periodic', - help='do NOT execute periodic tasks', - action='store_false') - parser.add_option('-d', '--delay', dest='initial_delay', type='float', - help='initial delay in seconds (default=0.1)', - default=0.1) - parser.add_option('-m', '--max-delay', dest='max_delay', type='float', - help='maximum time to wait between polling the queue ' - '(default=10)', - default=10) - parser.add_option('-b', '--backoff', dest='backoff', type='float', - help='amount to backoff delay when no results present ' - '(default=1.15)', - default=1.15) - parser.add_option('-P', '--periodic-task-interval', - dest='periodic_task_interval', - type='float', help='Granularity of periodic tasks.', - default=60.0) - parser.add_option('-S', '--scheduler-interval', dest='scheduler_interval', - type='float', help='Granularity of scheduler.', - default=1.0) - parser.add_option('-u', '--utc', dest='utc', action='store_true', - help='use UTC time for all tasks (default=True)', - default=True) - parser.add_option('--localtime', dest='utc', action='store_false', - help='use local time for all tasks') - return parser - - -def load_huey(path): - try: - return load_class(path) - except: - cur_dir = os.getcwd() - if cur_dir not in sys.path: - sys.path.insert(0, cur_dir) - return load_huey(path) - err('Error importing %s' % path) - raise - - -def consumer_main(): - parser = get_option_parser() - options, args = parser.parse_args() - - setup_logger(get_loglevel(options.verbose), options.logfile) - - if len(args) == 0: - err('Error: missing import path to `Huey` instance') - err('Example: huey_consumer.py app.queue.huey_instance') - sys.exit(1) - - huey_instance = load_huey(args[0]) - - consumer = Consumer( - huey_instance, - options.workers, - options.periodic, - options.initial_delay, - options.backoff, - options.max_delay, - options.utc, - options.scheduler_interval, - options.periodic_task_interval) - consumer.run() - - -if __name__ == '__main__': - consumer_main() diff --git a/lib/huey/consumer.py b/lib/huey/consumer.py deleted file mode 100644 index 2bf4b03..0000000 --- a/lib/huey/consumer.py +++ /dev/null @@ -1,279 +0,0 @@ -import datetime -import logging -import signal -import threading -import time - -from huey.exceptions import DataStoreGetException -from huey.exceptions import QueueException -from huey.exceptions import QueueReadException -from huey.exceptions import DataStorePutException -from huey.exceptions import QueueWriteException -from huey.exceptions import ScheduleAddException -from huey.exceptions import ScheduleReadException -from huey.registry import registry - - -class ConsumerThread(threading.Thread): - def __init__(self, huey, utc, shutdown, interval=60): - self.huey = huey - self.utc = utc - self.shutdown = shutdown - self.interval = interval - self._logger = logging.getLogger('huey.consumer.ConsumerThread') - super(ConsumerThread, self).__init__() - - def get_now(self): - if self.utc: - return datetime.datetime.utcnow() - return datetime.datetime.now() - - def on_shutdown(self): - pass - - def loop(self, now): - raise NotImplementedError - - def run(self): - while not self.shutdown.is_set(): - self.loop() - self._logger.debug('Thread shutting down') - self.on_shutdown() - - def enqueue(self, task): - try: - self.huey.enqueue(task) - self.huey.emit_task('enqueued', task) - except QueueWriteException: - self._logger.error('Error enqueueing task: %s' % task) - - def add_schedule(self, task): - try: - self.huey.add_schedule(task) - self.huey.emit_task('scheduled', task) - except ScheduleAddException: - self._logger.error('Error adding task to schedule: %s' % task) - - def is_revoked(self, task, ts): - try: - if self.huey.is_revoked(task, ts, peek=False): - self.huey.emit_task('revoked', task) - return True - return False - except DataStoreGetException: - self._logger.error('Error checking if task is revoked: %s' % task) - return True - - def sleep_for_interval(self, start_ts): - delta = time.time() - start_ts - if delta < self.interval: - time.sleep(self.interval - (time.time() - start_ts)) - - -class PeriodicTaskThread(ConsumerThread): - def loop(self, now=None): - now = now or self.get_now() - self._logger.debug('Checking periodic command registry') - start = time.time() - for task in registry.get_periodic_tasks(): - if task.validate_datetime(now): - self._logger.info('Scheduling %s for execution' % task) - self.enqueue(task) - - self.sleep_for_interval(start) - - -class SchedulerThread(ConsumerThread): - def read_schedule(self, ts): - try: - return self.huey.read_schedule(ts) - except ScheduleReadException: - self._logger.error('Error reading schedule', exc_info=1) - return [] - - def loop(self, now=None): - now = now or self.get_now() - start = time.time() - - for task in self.read_schedule(now): - self._logger.info('Scheduling %s for execution' % task) - self.enqueue(task) - - self.sleep_for_interval(start) - - -class WorkerThread(ConsumerThread): - def __init__(self, huey, default_delay, max_delay, backoff, utc, - shutdown): - self.delay = self.default_delay = default_delay - self.max_delay = max_delay - self.backoff = backoff - self._logger = logging.getLogger('huey.consumer.WorkerThread') - super(WorkerThread, self).__init__(huey, utc, shutdown) - - def loop(self): - self.check_message() - - def check_message(self): - self._logger.debug('Checking for message') - task = exc_raised = None - try: - task = self.huey.dequeue() - except QueueReadException: - self._logger.error('Error reading from queue', exc_info=1) - exc_raised = True - except QueueException: - self._logger.error('Queue exception', exc_info=1) - exc_raised = True - except: - self._logger.error('Unknown exception', exc_info=1) - exc_raised = True - - if task: - self.delay = self.default_delay - self.handle_task(task, self.get_now()) - elif exc_raised or not self.huey.blocking: - self.sleep() - - def sleep(self): - if self.delay > self.max_delay: - self.delay = self.max_delay - - self._logger.debug('No messages, sleeping for: %s' % self.delay) - time.sleep(self.delay) - self.delay *= self.backoff - - def handle_task(self, task, ts): - if not self.huey.ready_to_run(task, ts): - self._logger.info('Adding %s to schedule' % task) - self.add_schedule(task) - elif not self.is_revoked(task, ts): - self.process_task(task, ts) - - def process_task(self, task, ts): - try: - self._logger.info('Executing %s' % task) - self.huey.emit_task('started', task) - self.huey.execute(task) - self.huey.emit_task('finished', task) - except DataStorePutException: - self._logger.warn('Error storing result', exc_info=1) - except: - self._logger.error('Unhandled exception in worker thread', - exc_info=1) - self.huey.emit_task('error', task, error=True) - if task.retries: - self.huey.emit_task('retrying', task) - self.requeue_task(task, self.get_now()) - - def requeue_task(self, task, ts): - task.retries -= 1 - self._logger.info('Re-enqueueing task %s, %s tries left' % - (task.task_id, task.retries)) - if task.retry_delay: - delay = datetime.timedelta(seconds=task.retry_delay) - task.execute_time = ts + delay - self._logger.debug('Execute %s at: %s' % (task, task.execute_time)) - self.add_schedule(task) - else: - self.enqueue(task) - - -class Consumer(object): - def __init__(self, huey, workers=1, periodic=True, initial_delay=0.1, - backoff=1.15, max_delay=10.0, utc=True, scheduler_interval=1, - periodic_task_interval=60): - - self._logger = logging.getLogger('huey.consumer.ConsumerThread') - self.huey = huey - self.workers = workers - self.periodic = periodic - self.default_delay = initial_delay - self.backoff = backoff - self.max_delay = max_delay - self.utc = utc - self.scheduler_interval = scheduler_interval - self.periodic_task_interval = periodic_task_interval - - self.delay = self.default_delay - - self._shutdown = threading.Event() - - def run(self): - try: - self.start() - # it seems that calling self._shutdown.wait() here prevents the - # signal handler from executing - while not self._shutdown.is_set(): - self._shutdown.wait(.1) - except: - self._logger.error('Error', exc_info=1) - self.shutdown() - - def start(self): - self._logger.info('%d worker threads' % self.workers) - - self._set_signal_handler() - self._log_registered_commands() - self._create_threads() - - self._logger.info('Starting scheduler thread') - self.scheduler_t.start() - - self._logger.info('Starting worker threads') - for worker in self.worker_threads: - worker.start() - - if self.periodic: - self._logger.info('Starting periodic task scheduler thread') - self.periodic_t.start() - - def shutdown(self): - self._logger.info('Shutdown initiated') - self._shutdown.set() - - def _handle_signal(self, sig_num, frame): - self._logger.info('Received SIGTERM') - self.shutdown() - - def _set_signal_handler(self): - self._logger.info('Setting signal handler') - signal.signal(signal.SIGTERM, self._handle_signal) - - def _log_registered_commands(self): - msg = ['Huey consumer initialized with following commands'] - for command in registry._registry: - msg.append('+ %s' % command.replace('queuecmd_', '')) - self._logger.info('\n'.join(msg)) - - def _create_threads(self): - self.scheduler_t = SchedulerThread( - self.huey, - self.utc, - self._shutdown, - self.scheduler_interval) - self.scheduler_t.name = 'Scheduler' - - self.worker_threads = [] - for i in range(self.workers): - worker_t = WorkerThread( - self.huey, - self.default_delay, - self.max_delay, - self.backoff, - self.utc, - self._shutdown) - worker_t.daemon = True - worker_t.name = 'Worker %d' % (i + 1) - self.worker_threads.append(worker_t) - - if self.periodic: - self.periodic_t = PeriodicTaskThread( - self.huey, - self.utc, - self._shutdown, - self.periodic_task_interval) - self.periodic_t.daemon = True - self.periodic_t.name = 'Periodic Task' - else: - self.periodic_t = None diff --git a/lib/huey/djhuey/__init__.py b/lib/huey/djhuey/__init__.py deleted file mode 100644 index 9aa3765..0000000 --- a/lib/huey/djhuey/__init__.py +++ /dev/null @@ -1,119 +0,0 @@ -""" -This module contains a lot of cruft to handle instantiating a "Huey" object -using Django settings. Unlike more flexible python apps, the huey django -integration consists of a single global Huey instance configured via the -settings module. -""" -from functools import wraps -import sys - -from django.conf import settings -from django.db import connection - -from huey import crontab -from huey import Huey -from huey.utils import load_class - - -configuration_message = """ -Configuring Huey for use with Django -==================================== - -Huey was designed to be simple to configure in the general case. For that -reason, huey will "just work" with no configuration at all provided you have -Redis installed and running locally. - -On the other hand, you can configure huey manually using the following -setting structure. The following example uses Redis on localhost: - -Simply point to a backend: - -HUEY = { - 'backend': 'huey.backends.redis_backend', - 'name': 'unique name', - 'connection': {'host': 'localhost', 'port': 6379} - - 'consumer_options': {'workers': 4}, -} - -If you would like to configure Huey's logger using Django's integrated logging -settings, the logger used by consumer is named "huey.consumer". - -For more granular control, you can assign HUEY programmatically: - -HUEY = Huey(RedisBlockingQueue('my-queue')) -""" - -def default_queue_name(): - try: - return settings.DATABASE_NAME - except AttributeError: - return settings.DATABASES['default']['NAME'] - except KeyError: - return 'huey' - -def config_error(msg): - print(configuration_message) - print('\n\n') - print(msg) - sys.exit(1) - -def dynamic_import(obj, key, required=False): - try: - path = obj[key] - except KeyError: - if required: - config_error('Missing required configuration: "%s"' % key) - return None - try: - return load_class(path + '.Components') - except ImportError: - config_error('Unable to import %s: "%s"' % (key, path)) - -try: - HUEY = getattr(settings, 'HUEY', None) -except: - config_error('Error encountered reading settings.HUEY') - -if HUEY is None: - try: - from huey import RedisHuey - except ImportError: - config_error('Error: Huey could not import the redis backend. ' - 'Install `redis-py`.') - HUEY = RedisHuey(default_queue_name()) - -if not isinstance(HUEY, Huey): - Queue, DataStore, Schedule, Events = dynamic_import(HUEY, 'backend') - name = HUEY.get('name') or default_queue_name() - conn = HUEY.get('connection', {}) - always_eager = HUEY.get('always_eager', False) - HUEY = Huey( - Queue(name, **conn), - DataStore and DataStore(name, **conn) or None, - Schedule and Schedule(name, **conn) or None, - Events and Events(name, **conn) or None, - always_eager=always_eager) - -task = HUEY.task -periodic_task = HUEY.periodic_task - -def close_db(fn): - """Decorator to be used with tasks that may operate on the database.""" - @wraps(fn) - def inner(*args, **kwargs): - try: - return fn(*args, **kwargs) - finally: - connection.close() - return inner - -def db_task(*args, **kwargs): - def decorator(fn): - return task(*args, **kwargs)(close_db(fn)) - return decorator - -def db_periodic_task(*args, **kwargs): - def decorator(fn): - return periodic_task(*args, **kwargs)(close_db(fn)) - return decorator diff --git a/lib/huey/djhuey/management/__init__.py b/lib/huey/djhuey/management/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/lib/huey/djhuey/management/commands/__init__.py b/lib/huey/djhuey/management/commands/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/lib/huey/djhuey/management/commands/run_huey.py b/lib/huey/djhuey/management/commands/run_huey.py deleted file mode 100644 index 4aafb2d..0000000 --- a/lib/huey/djhuey/management/commands/run_huey.py +++ /dev/null @@ -1,126 +0,0 @@ -import imp -import sys -from optparse import make_option - -from django.conf import settings -from django.core.management.base import BaseCommand -try: - from importlib import import_module -except ImportError: - from django.utils.importlib import import_module - -try: - from django.apps import apps as django_apps - HAS_DJANGO_APPS = True -except ImportError: - # Django 1.6 - HAS_DJANGO_APPS = False - -from huey.consumer import Consumer -from huey.bin.huey_consumer import get_loglevel -from huey.bin.huey_consumer import setup_logger - - -class Command(BaseCommand): - """ - Queue consumer. Example usage:: - - To start the consumer (note you must export the settings module): - - django-admin.py run_huey - """ - help = "Run the queue consumer" - - option_list = BaseCommand.option_list + ( - make_option('--periodic', '-p', - dest='periodic', - action='store_true', - help='Enqueue periodic commands' - ), - make_option('--no-periodic', '-n', - dest='periodic', - action='store_false', - help='Do not enqueue periodic commands' - ), - make_option('--workers', '-w', - dest='workers', - type='int', - help='Number of worker threads' - ), - make_option('--delay', '-d', - dest='initial_delay', - type='float', - help='Delay between polling requests' - ), - make_option('--max_delay', '-m', - dest='max_delay', - type='float', - help='Maximum delay between polling requests' - ), - ) - - def autodiscover_appconfigs(self): - """Use Django app registry to pull out potential apps with tasks.py module.""" - module_name = 'tasks' - for config in django_apps.get_app_configs(): - app_path = config.module.__path__ - try: - fp, path, description = imp.find_module(module_name, app_path) - except ImportError: - continue - else: - import_path = '%s.%s' % (config.name, module_name) - imp.load_module(import_path, fp, path, description) - - def autodiscover_old(self): - # this is to find modules named in a django project's - # installed apps directories - module_name = 'tasks' - - for app in settings.INSTALLED_APPS: - try: - import_module(app) - app_path = sys.modules[app].__path__ - except AttributeError: - continue - try: - imp.find_module(module_name, app_path) - except ImportError: - continue - import_module('%s.%s' % (app, module_name)) - app_path = sys.modules['%s.%s' % (app, module_name)] - - def autodiscover(self): - """Switch between Django 1.7 style and old style app importing.""" - if HAS_DJANGO_APPS: - self.autodiscover_appconfigs() - else: - self.autodiscover_old() - - def handle(self, *args, **options): - from huey.djhuey import HUEY - try: - consumer_options = settings.HUEY['consumer_options'] - except: - consumer_options = {} - - if options['workers'] is not None: - consumer_options['workers'] = options['workers'] - - if options['periodic'] is not None: - consumer_options['periodic'] = options['periodic'] - - if options['initial_delay'] is not None: - consumer_options['initial_delay'] = options['initial_delay'] - - if options['max_delay'] is not None: - consumer_options['max_delay'] = options['max_delay'] - - self.autodiscover() - - loglevel = get_loglevel(consumer_options.pop('loglevel', None)) - logfile = consumer_options.pop('logfile', None) - setup_logger(loglevel, logfile) - - consumer = Consumer(HUEY, **consumer_options) - consumer.run() diff --git a/lib/huey/djhuey/models.py b/lib/huey/djhuey/models.py deleted file mode 100644 index e69de29..0000000 diff --git a/lib/huey/exceptions.py b/lib/huey/exceptions.py deleted file mode 100644 index 16b0340..0000000 --- a/lib/huey/exceptions.py +++ /dev/null @@ -1,26 +0,0 @@ -class QueueException(Exception): - pass - -class QueueWriteException(QueueException): - pass - -class QueueReadException(QueueException): - pass - -class QueueRemoveException(QueueException): - pass - -class DataStoreGetException(QueueException): - pass - -class DataStorePutException(QueueException): - pass - -class DataStoreTimeout(QueueException): - pass - -class ScheduleAddException(QueueException): - pass - -class ScheduleReadException(QueueException): - pass diff --git a/lib/huey/peewee_helpers/__init__.py b/lib/huey/peewee_helpers/__init__.py deleted file mode 100644 index 4da1150..0000000 --- a/lib/huey/peewee_helpers/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -from functools import wraps - - -def _transaction(db, fn): - @wraps(fn) - def inner(*args, **kwargs): - # Execute function in its own connection, in a transaction. - with db.execution_context(with_transaction=True): - return fn(*args, **kwargs) - return inner - -def db_task(huey, db, *args, **kwargs): - def decorator(fn): - return huey.task(*args, **kwargs)(_transaction(db, fn)) - return decorator - -def db_periodic_task(huey, db, *args, **kwargs): - def decorator(fn): - return huey.periodic_task(*args, **kwargs)(_transaction(db, fn)) - return decorator diff --git a/lib/huey/registry.py b/lib/huey/registry.py deleted file mode 100644 index 0c4e520..0000000 --- a/lib/huey/registry.py +++ /dev/null @@ -1,77 +0,0 @@ -import pickle - -from huey.exceptions import QueueException - - -class TaskRegistry(object): - """ - A simple Registry used to track subclasses of :class:`QueueTask` - the - purpose of this registry is to allow translation from queue messages to - task classes, and vice-versa. - """ - _ignore = ['QueueTask', 'PeriodicQueueTask'] - - _registry = {} - _periodic_tasks = [] - - def task_to_string(self, task): - return '%s' % (task.__name__) - - def register(self, task_class): - klass_str = self.task_to_string(task_class) - if klass_str in self._ignore: - return - - if klass_str not in self._registry: - self._registry[klass_str] = task_class - - # store an instance in a separate list of periodic tasks - if hasattr(task_class, 'validate_datetime'): - self._periodic_tasks.append(task_class()) - - def unregister(self, task_class): - klass_str = self.task_to_string(task_class) - - if klass_str in self._registry: - del(self._registry[klass_str]) - - for task in self._periodic_tasks: - if isinstance(task, task_class): - self._periodic_tasks.remove(task) - - def __contains__(self, klass_str): - return klass_str in self._registry - - def get_message_for_task(self, task): - """Convert a task object to a message for storage in the queue""" - return pickle.dumps(( - task.task_id, - self.task_to_string(type(task)), - task.execute_time, - task.retries, - task.retry_delay, - task.get_data(), - )) - - def get_task_class(self, klass_str): - klass = self._registry.get(klass_str) - - if not klass: - raise QueueException('%s not found in TaskRegistry' % klass_str) - - return klass - - def get_task_for_message(self, msg): - """Convert a message from the queue into a task""" - # parse out the pieces from the enqueued message - raw = pickle.loads(msg) - task_id, klass_str, execute_time, retries, delay, data = raw - - klass = self.get_task_class(klass_str) - return klass(data, task_id, execute_time, retries, delay) - - def get_periodic_tasks(self): - return self._periodic_tasks - - -registry = TaskRegistry() diff --git a/lib/huey/tests/__init__.py b/lib/huey/tests/__init__.py deleted file mode 100644 index 118aab6..0000000 --- a/lib/huey/tests/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from huey.tests.backends import * -from huey.tests.consumer import * -from huey.tests.crontab import * -from huey.tests.queue import * -from huey.tests.utils import * -try: - import peewee - from huey.tests.peewee_tests import * -except ImportError: - pass diff --git a/lib/huey/tests/backends.py b/lib/huey/tests/backends.py deleted file mode 100644 index 2423783..0000000 --- a/lib/huey/tests/backends.py +++ /dev/null @@ -1,170 +0,0 @@ -from collections import deque -import datetime -import os -import sys -import tempfile -import unittest - -from huey.api import Huey -from huey.backends.dummy import DummyDataStore -from huey.backends.dummy import DummyEventEmitter -from huey.backends.dummy import DummyQueue -from huey.backends.dummy import DummySchedule -from huey.utils import EmptyData -from huey.backends.sqlite_backend import SqliteDataStore -from huey.backends.sqlite_backend import SqliteQueue -from huey.backends.sqlite_backend import SqliteSchedule -try: - from huey.backends.redis_backend import RedisDataStore - from huey.backends.redis_backend import RedisEventEmitter - from huey.backends.redis_backend import RedisQueue - from huey.backends.redis_backend import RedisSchedule -except ImportError: - RedisQueue = RedisDataStore = RedisSchedule = RedisEventEmitter = None - -try: - from huey.backends.rabbitmq_backend import RabbitQueue, RabbitEventEmitter -except ImportError: - RabbitQueue = RabbitEventEmitter = None - - -if sys.version_info[0] == 2: - redis_kwargs = {} -else: - redis_kwargs = {'decode_responses': True} - - -QUEUES = (DummyQueue, RedisQueue, SqliteQueue, RabbitQueue) -DATA_STORES = (DummyDataStore, RedisDataStore, SqliteDataStore, None) -SCHEDULES = (DummySchedule, RedisSchedule, SqliteSchedule, None) -EVENTS = (DummyEventEmitter, RedisEventEmitter, None, RabbitEventEmitter) - - -class HueyBackendTestCase(unittest.TestCase): - def setUp(self): - self.sqlite_location = tempfile.mkstemp(prefix='hueytest.')[1] - - def tearDown(self): - os.unlink(self.sqlite_location) - - def test_queues(self): - result_store = DummyDataStore('dummy') - for q in QUEUES: - if not q: - continue - if issubclass(q, SqliteQueue): - queue = q('test', location=self.sqlite_location) - elif issubclass(q, RedisQueue): - queue = q('test', **redis_kwargs) - else: - queue = q('test') - queue.flush() - queue.write('a') - queue.write('b') - self.assertEqual(len(queue), 2) - self.assertEqual(queue.read(), 'a') - self.assertEqual(queue.read(), 'b') - self.assertEqual(queue.read(), None) - - queue.write('c') - queue.write('d') - queue.write('c') - queue.write('x') - queue.write('d') - self.assertEqual(len(queue), 5) - self.assertEqual(queue.remove('c'), 2) - self.assertEqual(len(queue), 3) - self.assertEqual(queue.read(), 'd') - self.assertEqual(queue.read(), 'x') - self.assertEqual(queue.read(), 'd') - - queue.flush() - test_huey = Huey(queue, result_store) - - @test_huey.task() - def test_queues_add(k, v): - return k + v - - res = test_queues_add('k', 'v') - self.assertEqual(len(queue), 1) - task = test_huey.dequeue() - test_huey.execute(task) - self.assertEqual(res.get(), 'kv') - - res = test_queues_add('\xce', '\xcf') - task = test_huey.dequeue() - test_huey.execute(task) - self.assertEqual(res.get(), '\xce\xcf') - - def test_data_stores(self): - for d in DATA_STORES: - if not d: - continue - if issubclass(d, SqliteDataStore): - data_store = d('test', location=self.sqlite_location) - elif issubclass(d, RedisDataStore): - data_store = d('test', **redis_kwargs) - else: - data_store = d('test') - data_store.put('k1', 'v1') - data_store.put('k2', 'v2') - data_store.put('k3', 'v3') - self.assertEqual(data_store.peek('k2'), 'v2') - self.assertEqual(data_store.get('k2'), 'v2') - self.assertEqual(data_store.peek('k2'), EmptyData) - self.assertEqual(data_store.get('k2'), EmptyData) - - self.assertEqual(data_store.peek('k3'), 'v3') - data_store.put('k3', 'v3-2') - self.assertEqual(data_store.peek('k3'), 'v3-2') - - def test_schedules(self): - for s in SCHEDULES: - if not s: - continue - if issubclass(s, SqliteSchedule): - schedule = s('test', location=self.sqlite_location) - elif issubclass(s, RedisSchedule): - schedule = s('test', **redis_kwargs) - else: - schedule = s('test') - dt1 = datetime.datetime(2013, 1, 1, 0, 0) - dt2 = datetime.datetime(2013, 1, 2, 0, 0) - dt3 = datetime.datetime(2013, 1, 3, 0, 0) - dt4 = datetime.datetime(2013, 1, 4, 0, 0) - - # Add to schedule out-of-order to ensure sorting is performed by - # the schedule. - schedule.add('s2', dt2) - schedule.add('s1', dt1) - schedule.add('s4', dt4) - schedule.add('s3', dt3) - - # Ensure that asking for a timestamp previous to any item in the - # schedule returns empty list. - self.assertEqual( - schedule.read(dt1 - datetime.timedelta(days=1)), - []) - - # Ensure the upper boundary is inclusive of whatever timestamp - # is passed in. - self.assertEqual(schedule.read(dt3), ['s1', 's2', 's3']) - self.assertEqual(schedule.read(dt3), []) - - # Ensure the schedule is flushed and an empty schedule returns an - # empty list. - self.assertEqual(schedule.read(dt4), ['s4']) - self.assertEqual(schedule.read(dt4), []) - - def test_events(self): - for e in EVENTS: - if not e: - continue - e = e('test') - - messages = ['a', 'b', 'c', 'd'] - for message in messages: - e.emit(message) - - if hasattr(e, '_events'): - self.assertEqual(e._events, deque(['d', 'c', 'b', 'a'])) diff --git a/lib/huey/tests/consumer.py b/lib/huey/tests/consumer.py deleted file mode 100644 index 7f31044..0000000 --- a/lib/huey/tests/consumer.py +++ /dev/null @@ -1,441 +0,0 @@ -from collections import deque -import datetime -import json -import logging -import threading -import time -import unittest - -from huey import crontab -from huey import Huey -from huey.backends.dummy import DummyDataStore -from huey.backends.dummy import DummyEventEmitter -from huey.backends.dummy import DummyQueue -from huey.backends.dummy import DummySchedule -from huey.consumer import Consumer -from huey.consumer import WorkerThread -from huey.registry import registry - -# Logger used by the consumer. -logger = logging.getLogger('huey.consumer') - -# Store some global state. -state = {} - -# Create a queue, result store, schedule and event emitter, then attach them -# to a test-only Huey instance. -test_queue = DummyQueue('test-queue') -test_result_store = DummyDataStore('test-queue') -test_schedule = DummySchedule('test-queue') -test_events = DummyEventEmitter('test-queue') -test_huey = Huey(test_queue, test_result_store, test_schedule, test_events) - -# Create some test tasks. -@test_huey.task() -def modify_state(k, v): - state[k] = v - return v - -@test_huey.task() -def blow_up(): - raise Exception('blowed up') - -@test_huey.task(retries=3) -def retry_command(k, always_fail=True): - if k not in state: - if not always_fail: - state[k] = 'fixed' - raise Exception('fappsk') - return state[k] - -@test_huey.task(retries=3, retry_delay=10) -def retry_command_slow(k, always_fail=True): - if k not in state: - if not always_fail: - state[k] = 'fixed' - raise Exception('fappsk') - return state[k] - -@test_huey.periodic_task(crontab(minute='0')) -def every_hour(): - state['p'] = 'y' - - -# Create a log handler that will track messages generated by the consumer. -class TestLogHandler(logging.Handler): - def __init__(self, *args, **kwargs): - self.messages = [] - logging.Handler.__init__(self, *args, **kwargs) - - def emit(self, record): - self.messages.append(record.getMessage()) - - -class ConsumerTestCase(unittest.TestCase): - def setUp(self): - global state - state = {} - - self.orig_pc = registry._periodic_tasks - registry._periodic_commands = [every_hour.task_class()] - - self.orig_sleep = time.sleep - time.sleep = lambda x: None - - test_huey.queue.flush() - test_huey.result_store.flush() - test_huey.schedule.flush() - test_events._events = deque() - - self.consumer = Consumer(test_huey, workers=2) - self.consumer._create_threads() - - self.handler = TestLogHandler() - logger.addHandler(self.handler) - logger.setLevel(logging.INFO) - - def tearDown(self): - self.consumer.shutdown() - logger.removeHandler(self.handler) - registry._periodic_tasks = self.orig_pc - time.sleep = self.orig_sleep - - def assertStatusTask(self, status_task): - parsed = [] - i = 0 - while i < len(status_task): - event = json.loads(test_events._events[i]) - status, task, extra = status_task[i] - self.assertEqual(event['status'], status) - self.assertEqual(event['id'], task.task_id) - for k, v in extra.items(): - self.assertEqual(event[k], v) - i += 1 - - def spawn(self, func, *args, **kwargs): - t = threading.Thread(target=func, args=args, kwargs=kwargs) - t.start() - return t - - def run_worker(self, task, ts=None): - worker_t = WorkerThread( - test_huey, - self.consumer.default_delay, - self.consumer.max_delay, - self.consumer.backoff, - self.consumer.utc, - self.consumer._shutdown) - ts = ts or datetime.datetime.utcnow() - worker_t.handle_task(task, ts) - - def test_message_processing(self): - self.consumer.worker_threads[0].start() - - self.assertFalse('k' in state) - - res = modify_state('k', 'v') - res.get(blocking=True) - - self.assertTrue('k' in state) - self.assertEqual(res.get(), 'v') - - self.assertEqual(len(test_events._events), 2) - self.assertStatusTask([ - ('finished', res.task, {}), - ('started', res.task, {}), - ]) - - def test_worker(self): - modify_state('k', 'w') - task = test_huey.dequeue() - self.run_worker(task) - self.assertEqual(state, {'k': 'w'}) - - def test_worker_exception(self): - blow_up() - task = test_huey.dequeue() - - self.run_worker(task) - self.assertTrue( - 'Unhandled exception in worker thread' in self.handler.messages) - - self.assertEqual(len(test_events._events), 2) - self.assertStatusTask([ - ('error', task, {'error': True}), - ('started', task, {}), - ]) - - def test_retries_and_logging(self): - # this will continually fail - retry_command('blampf') - - for i in reversed(range(4)): - task = test_huey.dequeue() - self.assertEqual(task.retries, i) - self.run_worker(task) - if i > 0: - self.assertEqual( - self.handler.messages[-1], - 'Re-enqueueing task %s, %s tries left' % ( - task.task_id, i - 1)) - self.assertStatusTask([ - ('enqueued', task, {}), - ('retrying', task, {}), - ('error', task,{}), - ('started', task, {}), - ]) - last_idx = -2 - else: - self.assertStatusTask([ - ('error', task,{}), - ('started', task, {}), - ]) - last_idx = -1 - self.assertEqual(self.handler.messages[last_idx], - 'Unhandled exception in worker thread') - - self.assertEqual(test_huey.dequeue(), None) - - def test_retries_with_success(self): - # this will fail once, then succeed - retry_command('blampf', False) - self.assertFalse('blampf' in state) - - task = test_huey.dequeue() - self.run_worker(task) - self.assertEqual(self.handler.messages, [ - 'Executing %s' % task, - 'Unhandled exception in worker thread', - 'Re-enqueueing task %s, 2 tries left' % task.task_id]) - - task = test_huey.dequeue() - self.assertEqual(task.retries, 2) - self.run_worker(task) - - self.assertEqual(state['blampf'], 'fixed') - self.assertEqual(test_huey.dequeue(), None) - - self.assertStatusTask([ - ('finished', task, {}), - ('started', task, {}), - ('enqueued', task, {'retries': 2}), - ('retrying', task, {'retries': 3}), - ('error', task, {'error': True}), - ('started', task, {}), - ]) - - def test_scheduling(self): - dt = datetime.datetime(2011, 1, 1, 0, 0) - dt2 = datetime.datetime(2037, 1, 1, 0, 0) - ad1 = modify_state.schedule(args=('k', 'v'), eta=dt, convert_utc=False) - ad2 = modify_state.schedule(args=('k2', 'v2'), eta=dt2, convert_utc=False) - - # dequeue the past-timestamped task and run it. - worker = self.consumer.worker_threads[0] - worker.check_message() - - self.assertTrue('k' in state) - - # dequeue the future-timestamped task. - worker.check_message() - - # verify the task got stored in the schedule instead of executing - self.assertFalse('k2' in state) - - self.assertStatusTask([ - ('scheduled', ad2.task, {}), - ('finished', ad1.task, {}), - ('started', ad1.task, {}), - ]) - - # run through an iteration of the scheduler - self.consumer.scheduler_t.loop(dt) - - # our command was not enqueued and no events were emitted. - self.assertEqual(len(test_queue._queue), 0) - self.assertEqual(len(test_events._events), 3) - - # run through an iteration of the scheduler - self.consumer.scheduler_t.loop(dt2) - - # our command was enqueued - self.assertEqual(len(test_queue._queue), 1) - self.assertEqual(len(test_events._events), 4) - self.assertStatusTask([ - ('enqueued', ad2.task, {}), - ]) - - def test_retry_scheduling(self): - # this will continually fail - retry_command_slow('blampf') - cur_time = datetime.datetime.utcnow() - - task = test_huey.dequeue() - self.run_worker(task, ts=cur_time) - self.assertEqual(self.handler.messages, [ - 'Executing %s' % task, - 'Unhandled exception in worker thread', - 'Re-enqueueing task %s, 2 tries left' % task.task_id, - ]) - - in_11 = cur_time + datetime.timedelta(seconds=11) - tasks_from_sched = test_huey.read_schedule(in_11) - self.assertEqual(tasks_from_sched, [task]) - - task = tasks_from_sched[0] - self.assertEqual(task.retries, 2) - exec_time = task.execute_time - - self.assertEqual((exec_time - cur_time).seconds, 10) - self.assertStatusTask([ - ('scheduled', task, { - 'retries': 2, - 'retry_delay': 10, - 'execute_time': time.mktime(exec_time.timetuple())}), - ('retrying', task, { - 'retries': 3, - 'retry_delay': 10, - 'execute_time': None}), - ('error', task, {}), - ('started', task, {}), - ]) - - def test_revoking_normal(self): - # enqueue 2 normal commands - r1 = modify_state('k', 'v') - r2 = modify_state('k2', 'v2') - - # revoke the first *before it has been checked* - r1.revoke() - self.assertTrue(test_huey.is_revoked(r1.task)) - self.assertFalse(test_huey.is_revoked(r2.task)) - - # dequeue a *single* message (r1) - task = test_huey.dequeue() - self.run_worker(task) - - self.assertEqual(len(test_events._events), 1) - self.assertStatusTask([ - ('revoked', r1.task, {}), - ]) - - # no changes and the task was not added to the schedule - self.assertFalse('k' in state) - - # dequeue a *single* message - task = test_huey.dequeue() - self.run_worker(task) - - self.assertTrue('k2' in state) - - def test_revoking_schedule(self): - global state - dt = datetime.datetime(2011, 1, 1) - dt2 = datetime.datetime(2037, 1, 1) - - r1 = modify_state.schedule(args=('k', 'v'), eta=dt, convert_utc=False) - r2 = modify_state.schedule(args=('k2', 'v2'), eta=dt, convert_utc=False) - r3 = modify_state.schedule(args=('k3', 'v3'), eta=dt2, convert_utc=False) - r4 = modify_state.schedule(args=('k4', 'v4'), eta=dt2, convert_utc=False) - - # revoke r1 and r3 - r1.revoke() - r3.revoke() - self.assertTrue(test_huey.is_revoked(r1.task)) - self.assertFalse(test_huey.is_revoked(r2.task)) - self.assertTrue(test_huey.is_revoked(r3.task)) - self.assertFalse(test_huey.is_revoked(r4.task)) - - expected = [ - #state, schedule - ({}, 0), - ({'k2': 'v2'}, 0), - ({'k2': 'v2'}, 1), - ({'k2': 'v2'}, 2), - ] - - for i in range(4): - estate, esc = expected[i] - - # dequeue a *single* message - task = test_huey.dequeue() - self.run_worker(task) - - self.assertEqual(state, estate) - self.assertEqual(len(test_huey.schedule._schedule), esc) - - # lets pretend its 2037 - future = dt2 + datetime.timedelta(seconds=1) - self.consumer.scheduler_t.loop(future) - self.assertEqual(len(test_huey.schedule._schedule), 0) - - # There are two tasks in the queue now (r3 and r4) -- process both. - for i in range(2): - task = test_huey.dequeue() - self.run_worker(task, future) - - self.assertEqual(state, {'k2': 'v2', 'k4': 'v4'}) - - def test_revoking_periodic(self): - global state - def loop_periodic(ts): - self.consumer.periodic_t.loop(ts) - for i in range(len(test_queue._queue)): - task = test_huey.dequeue() - self.run_worker(task, ts) - - # revoke the command once - every_hour.revoke(revoke_once=True) - self.assertTrue(every_hour.is_revoked()) - - # it will be skipped the first go-round - dt = datetime.datetime(2011, 1, 1, 0, 0) - loop_periodic(dt) - - # it has not been run - self.assertEqual(state, {}) - - # the next go-round it will be enqueued - loop_periodic(dt) - - # our command was run - self.assertEqual(state, {'p': 'y'}) - - # reset state - state = {} - - # revoke the command - every_hour.revoke() - self.assertTrue(every_hour.is_revoked()) - - # it will no longer be enqueued - loop_periodic(dt) - loop_periodic(dt) - self.assertEqual(state, {}) - - # restore - every_hour.restore() - self.assertFalse(every_hour.is_revoked()) - - # it will now be enqueued - loop_periodic(dt) - self.assertEqual(state, {'p': 'y'}) - - # reset - state = {} - - # revoke for an hour - td = datetime.timedelta(seconds=3600) - every_hour.revoke(revoke_until=dt + td) - - loop_periodic(dt) - self.assertEqual(state, {}) - - # after an hour it is back - loop_periodic(dt + td) - self.assertEqual(state, {'p': 'y'}) - - # our data store should reflect the delay - task_obj = every_hour.task_class() - self.assertEqual(len(test_huey.result_store._results), 1) - self.assertTrue(task_obj.revoke_id in test_huey.result_store._results) diff --git a/lib/huey/tests/crontab.py b/lib/huey/tests/crontab.py deleted file mode 100644 index 8c603ab..0000000 --- a/lib/huey/tests/crontab.py +++ /dev/null @@ -1,91 +0,0 @@ -import datetime -import unittest - -from huey import crontab - - -class CrontabTestCase(unittest.TestCase): - def test_crontab_month(self): - # validates the following months, 1, 4, 7, 8, 9 - valids = [1, 4, 7, 8, 9] - validate_m = crontab(month='1,4,*/6,8-9') - - for x in range(1, 13): - res = validate_m(datetime.datetime(2011, x, 1)) - self.assertEqual(res, x in valids) - - def test_crontab_day(self): - # validates the following days - valids = [1, 4, 7, 8, 9, 13, 19, 25, 31] - validate_d = crontab(day='*/6,1,4,8-9') - - for x in range(1, 32): - res = validate_d(datetime.datetime(2011, 1, x)) - self.assertEqual(res, x in valids) - - def test_crontab_hour(self): - # validates the following hours - valids = [0, 1, 4, 6, 8, 9, 12, 18] - validate_h = crontab(hour='8-9,*/6,1,4') - - for x in range(24): - res = validate_h(datetime.datetime(2011, 1, 1, x)) - self.assertEqual(res, x in valids) - - edge = crontab(hour=0) - self.assertTrue(edge(datetime.datetime(2011, 1, 1, 0, 0))) - self.assertFalse(edge(datetime.datetime(2011, 1, 1, 12, 0))) - - def test_crontab_minute(self): - # validates the following minutes - valids = [0, 1, 4, 6, 8, 9, 12, 18, 24, 30, 36, 42, 48, 54] - validate_m = crontab(minute='4,8-9,*/6,1') - - for x in range(60): - res = validate_m(datetime.datetime(2011, 1, 1, 1, x)) - self.assertEqual(res, x in valids) - - def test_crontab_day_of_week(self): - # validates the following days of week - # jan, 1, 2011 is a saturday - valids = [2, 4, 9, 11, 16, 18, 23, 25, 30] - validate_dow = crontab(day_of_week='0,2') - - for x in range(1, 32): - res = validate_dow(datetime.datetime(2011, 1, x)) - self.assertEqual(res, x in valids) - - def test_crontab_all_together(self): - # jan 1, 2011 is a saturday - # may 1, 2011 is a sunday - validate = crontab( - month='1,5', - day='1,4,7', - day_of_week='0,6', - hour='*/4', - minute='1-5,10-15,50' - ) - - self.assertTrue(validate(datetime.datetime(2011, 5, 1, 4, 11))) - self.assertTrue(validate(datetime.datetime(2011, 5, 7, 20, 50))) - self.assertTrue(validate(datetime.datetime(2011, 1, 1, 0, 1))) - - # fails validation on month - self.assertFalse(validate(datetime.datetime(2011, 6, 4, 4, 11))) - - # fails validation on day - self.assertFalse(validate(datetime.datetime(2011, 1, 6, 4, 11))) - - # fails validation on day_of_week - self.assertFalse(validate(datetime.datetime(2011, 1, 4, 4, 11))) - - # fails validation on hour - self.assertFalse(validate(datetime.datetime(2011, 1, 1, 1, 11))) - - # fails validation on minute - self.assertFalse(validate(datetime.datetime(2011, 1, 1, 4, 6))) - - def test_invalid_crontabs(self): - # check invalid configurations are detected and reported - self.assertRaises(ValueError, crontab, minute='61') - self.assertRaises(ValueError, crontab, minute='0-61') diff --git a/lib/huey/tests/peewee_tests.py b/lib/huey/tests/peewee_tests.py deleted file mode 100644 index 9ed66c1..0000000 --- a/lib/huey/tests/peewee_tests.py +++ /dev/null @@ -1,62 +0,0 @@ -from contextlib import contextmanager -import unittest - -from huey import Huey -from huey.backends.dummy import DummyDataStore -from huey.backends.dummy import DummyQueue -from huey.backends.dummy import DummySchedule -from huey.peewee_helpers import db_periodic_task -from huey.peewee_helpers import db_task -from peewee import * - - -queue = DummyQueue('test-queue') -schedule = DummySchedule('test-queue') -data_store = DummyDataStore('test-queue') -huey = Huey(queue, data_store, schedule=schedule) - -STATE = [] - -class MockSqliteDatabase(SqliteDatabase): - def record_call(fn): - def inner(*args, **kwargs): - STATE.append(fn.__name__) - return fn(*args, **kwargs) - return inner - connect = record_call(SqliteDatabase.connect) - _close = record_call(SqliteDatabase._close) - transaction = record_call(SqliteDatabase.transaction) - -db = MockSqliteDatabase('test.huey.db') - -class Value(Model): - data = CharField() - - class Meta: - database = db - - @classmethod - def create(cls, *args, **kwargs): - STATE.append('create') - return super(Value, cls).create(*args, **kwargs) - -@db_task(huey, db) -def test_db_task(val): - return Value.create(data=val) - -class TestPeeweeHelpers(unittest.TestCase): - def setUp(self): - global STATE - STATE = [] - queue.flush() - data_store.flush() - schedule.flush() - Value.drop_table(True) - Value.create_table() - - def test_helper(self): - test_db_task('foo') - self.assertEqual(STATE, ['connect']) - huey.execute(huey.dequeue()) - self.assertEqual(STATE, ['connect', 'transaction', 'create', '_close']) - self.assertEqual(Value.select().count(), 1) diff --git a/lib/huey/tests/queue.py b/lib/huey/tests/queue.py deleted file mode 100644 index b66d0ab..0000000 --- a/lib/huey/tests/queue.py +++ /dev/null @@ -1,438 +0,0 @@ -import datetime -import unittest - -from huey import crontab -from huey import exceptions as huey_exceptions -from huey import Huey -from huey.api import QueueTask -from huey.backends.dummy import DummyDataStore -from huey.backends.dummy import DummyQueue -from huey.backends.dummy import DummySchedule -from huey.registry import registry -from huey.utils import EmptyData -from huey.utils import local_to_utc - - -queue_name = 'test-queue' -queue = DummyQueue(queue_name) -schedule = DummySchedule(queue_name) -huey = Huey(queue, schedule=schedule) - -res_queue_name = 'test-queue-2' -res_queue = DummyQueue(res_queue_name) -res_store = DummyDataStore(res_queue_name) - -res_huey = Huey(res_queue, res_store, schedule) -res_huey_nones = Huey(res_queue, res_store, store_none=True) - -# store some global state -state = {} -last_executed_task_class = [] - -# create a decorated queue command -@huey.task() -def add(key, value): - state[key] = value - -@huey.task(include_task=True) -def self_aware(key, value, task=None): - last_executed_task_class.append(task.__class__.__name__) - -# create a periodic queue command -@huey.periodic_task(crontab(minute='0')) -def add_on_the_hour(): - state['periodic'] = 'x' - -# define a command using the class -class AddTask(QueueTask): - def execute(self): - k, v = self.data - state[k] = v - -# create a command that raises an exception -class BampfException(Exception): - pass - -@huey.task() -def throw_error(): - raise BampfException('bampf') - -@res_huey.task() -def add2(a, b): - return a + b - -@res_huey.periodic_task(crontab(minute='0')) -def add_on_the_hour2(): - state['periodic'] = 'x' - -@res_huey.task() -def returns_none(): - return None - -@res_huey_nones.task() -def returns_none2(): - return None - - -class HueyTestCase(unittest.TestCase): - def setUp(self): - global state - global last_executed_task_class - queue.flush() - res_queue.flush() - schedule.flush() - state = {} - last_executed_task_class = [] - - def test_registration(self): - self.assertTrue('queuecmd_add' in registry) - self.assertTrue('queuecmd_add_on_the_hour' in registry) - self.assertTrue('AddTask' in registry) - - def test_enqueue(self): - # sanity check - self.assertEqual(len(queue), 0) - - # initializing the command does not enqueue it - ac = AddTask(('k', 'v')) - self.assertEqual(len(queue), 0) - - # ok, enqueue it, then check that it was enqueued - huey.enqueue(ac) - self.assertEqual(len(queue), 1) - - # it can be enqueued multiple times - huey.enqueue(ac) - self.assertEqual(len(queue), 2) - - # no changes to state - self.assertFalse('k' in state) - - def test_enqueue_decorator(self): - # sanity check - self.assertEqual(len(queue), 0) - - add('k', 'v') - self.assertEqual(len(queue), 1) - - add('k', 'v') - self.assertEqual(len(queue), 2) - - # no changes to state - self.assertFalse('k' in state) - - def test_schedule(self): - dt = datetime.datetime(2011, 1, 1, 0, 1) - add('k', 'v') - self.assertEqual(len(queue), 1) - - task = huey.dequeue() - self.assertEqual(task.execute_time, None) - - add.schedule(args=('k2', 'v2'), eta=dt) - self.assertEqual(len(queue), 1) - task = huey.dequeue() - self.assertEqual(task.execute_time, local_to_utc(dt)) - - add.schedule(args=('k3', 'v3'), eta=dt, convert_utc=False) - self.assertEqual(len(queue), 1) - task = huey.dequeue() - self.assertEqual(task.execute_time, dt) - - def test_error_raised(self): - throw_error() - - # no error - task = huey.dequeue() - - # error - self.assertRaises(BampfException, huey.execute, task) - - def test_internal_error(self): - """ - Verify that exceptions are wrapped with the special "huey" - exception classes. - """ - class SpecialException(Exception): - pass - - class BrokenQueue(DummyQueue): - def read(self): - raise SpecialException('read error') - - def write(self, data): - raise SpecialException('write error') - - class BrokenDataStore(DummyDataStore): - def get(self, key): - raise SpecialException('get error') - - def put(self, key, value): - raise SpecialException('put error') - - class BrokenSchedule(DummySchedule): - def add(self, data, ts): - raise SpecialException('add error') - - def read(self, ts): - raise SpecialException('read error') - - task = AddTask() - huey = Huey( - BrokenQueue('q'), - BrokenDataStore('q'), - BrokenSchedule('q')) - - self.assertRaises( - huey_exceptions.QueueWriteException, - huey.enqueue, - AddTask()) - self.assertRaises( - huey_exceptions.QueueReadException, - huey.dequeue) - self.assertRaises( - huey_exceptions.DataStorePutException, - huey.revoke, - task) - self.assertRaises( - huey_exceptions.DataStoreGetException, - huey.restore, - task) - self.assertRaises( - huey_exceptions.ScheduleAddException, - huey.add_schedule, - task) - self.assertRaises( - huey_exceptions.ScheduleReadException, - huey.read_schedule, - 1) - - def test_dequeueing(self): - res = huey.dequeue() # no error raised if queue is empty - self.assertEqual(res, None) - - add('k', 'v') - task = huey.dequeue() - - self.assertTrue(isinstance(task, QueueTask)) - self.assertEqual(task.get_data(), (('k', 'v'), {})) - - def test_execution(self): - self.assertFalse('k' in state) - add('k', 'v') - - task = huey.dequeue() - self.assertFalse('k' in state) - - huey.execute(task) - self.assertEqual(state['k'], 'v') - - add('k', 'X') - self.assertEqual(state['k'], 'v') - - huey.execute(huey.dequeue()) - self.assertEqual(state['k'], 'X') - - self.assertRaises(TypeError, huey.execute, huey.dequeue()) - - def test_self_awareness(self): - self_aware('k', 'v') - task = huey.dequeue() - huey.execute(task) - self.assertEqual(last_executed_task_class.pop(), "queuecmd_self_aware") - - self_aware('k', 'v') - huey.execute(huey.dequeue()) - self.assertEqual(last_executed_task_class.pop(), "queuecmd_self_aware") - - add('k', 'x') - huey.execute(huey.dequeue()) - self.assertEqual(len(last_executed_task_class), 0) - - def test_call_local(self): - self.assertEqual(len(queue), 0) - self.assertEqual(state, {}) - add.call_local('nugget', 'green') - - self.assertEqual(len(queue), 0) - self.assertEqual(state['nugget'], 'green') - - def test_revoke(self): - ac = AddTask(('k', 'v')) - ac2 = AddTask(('k2', 'v2')) - ac3 = AddTask(('k3', 'v3')) - - res_huey.enqueue(ac) - res_huey.enqueue(ac2) - res_huey.enqueue(ac3) - res_huey.enqueue(ac2) - res_huey.enqueue(ac) - - self.assertEqual(len(res_queue), 5) - res_huey.revoke(ac2) - - while res_queue: - task = res_huey.dequeue() - if not res_huey.is_revoked(task): - res_huey.execute(task) - - self.assertEqual(state, {'k': 'v', 'k3': 'v3'}) - - def test_revoke_periodic(self): - add_on_the_hour2.revoke() - self.assertTrue(add_on_the_hour2.is_revoked()) - - # it is still revoked - self.assertTrue(add_on_the_hour2.is_revoked()) - - add_on_the_hour2.restore() - self.assertFalse(add_on_the_hour2.is_revoked()) - - add_on_the_hour2.revoke(revoke_once=True) - self.assertTrue(add_on_the_hour2.is_revoked()) # it is revoked once, but we are preserving that state - self.assertTrue(add_on_the_hour2.is_revoked(peek=False)) # is revoked once, but clear state - self.assertFalse(add_on_the_hour2.is_revoked()) # no longer revoked - - d = datetime.datetime - add_on_the_hour2.revoke(revoke_until=d(2011, 1, 1, 11, 0)) - self.assertTrue(add_on_the_hour2.is_revoked(dt=d(2011, 1, 1, 10, 0))) - self.assertTrue(add_on_the_hour2.is_revoked(dt=d(2011, 1, 1, 10, 59))) - self.assertFalse(add_on_the_hour2.is_revoked(dt=d(2011, 1, 1, 11, 0))) - - add_on_the_hour2.restore() - self.assertFalse(add_on_the_hour2.is_revoked()) - - def test_result_store(self): - res = add2(1, 2) - res2 = add2(4, 5) - res3 = add2(0, 0) - - # none have been executed as yet - self.assertEqual(res.get(), None) - self.assertEqual(res2.get(), None) - self.assertEqual(res3.get(), None) - - # execute the first task - res_huey.execute(res_huey.dequeue()) - self.assertEqual(res.get(), 3) - self.assertEqual(res2.get(), None) - self.assertEqual(res3.get(), None) - - # execute the second task - res_huey.execute(res_huey.dequeue()) - self.assertEqual(res.get(), 3) - self.assertEqual(res2.get(), 9) - self.assertEqual(res3.get(), None) - - # execute the 3rd, which returns a zero value - res_huey.execute(res_huey.dequeue()) - self.assertEqual(res.get(), 3) - self.assertEqual(res2.get(), 9) - self.assertEqual(res3.get(), 0) - - # check that it returns None when nothing is present - res = returns_none() - self.assertEqual(res.get(), None) - - # execute, it will still return None, but underneath it is an EmptyResult - # indicating its actual result was not persisted - res_huey.execute(res_huey.dequeue()) - self.assertEqual(res.get(), None) - self.assertEqual(res._result, EmptyData) - - # execute again, this time note that we're pointing at the invoker - # that *does* accept None as a store-able result - res = returns_none2() - self.assertEqual(res.get(), None) - - # it stores None - res_huey_nones.execute(res_huey_nones.dequeue()) - self.assertEqual(res.get(), None) - self.assertEqual(res._result, None) - - def test_task_store(self): - dt1 = datetime.datetime(2011, 1, 1, 0, 0) - dt2 = datetime.datetime(2035, 1, 1, 0, 0) - - add2.schedule(args=('k', 'v'), eta=dt1, convert_utc=False) - task1 = res_huey.dequeue() - - add2.schedule(args=('k2', 'v2'), eta=dt2, convert_utc=False) - task2 = res_huey.dequeue() - - add2('k3', 'v3') - task3 = res_huey.dequeue() - - # add the command to the schedule - res_huey.add_schedule(task1) - self.assertEqual(len(res_huey.schedule._schedule), 1) - - # add a future-dated command - res_huey.add_schedule(task2) - self.assertEqual(len(res_huey.schedule._schedule), 2) - - res_huey.add_schedule(task3) - - tasks = res_huey.read_schedule(dt1) - self.assertEqual(tasks, [task3, task1]) - - tasks = res_huey.read_schedule(dt1) - self.assertEqual(tasks, []) - - tasks = res_huey.read_schedule(dt2) - self.assertEqual(tasks, [task2]) - - def test_ready_to_run_method(self): - dt1 = datetime.datetime(2011, 1, 1, 0, 0) - dt2 = datetime.datetime(2035, 1, 1, 0, 0) - - add2.schedule(args=('k', 'v'), eta=dt1) - task1 = res_huey.dequeue() - - add2.schedule(args=('k2', 'v2'), eta=dt2) - task2 = res_huey.dequeue() - - add2('k3', 'v3') - task3 = res_huey.dequeue() - - add2.schedule(args=('k4', 'v4'), task_id='test_task_id') - task4 = res_huey.dequeue() - - # sanity check what should be run - self.assertTrue(res_huey.ready_to_run(task1)) - self.assertFalse(res_huey.ready_to_run(task2)) - self.assertTrue(res_huey.ready_to_run(task3)) - self.assertTrue(res_huey.ready_to_run(task4)) - self.assertEqual('test_task_id', task4.task_id) - - def test_task_delay(self): - curr = datetime.datetime.utcnow() - curr50 = curr + datetime.timedelta(seconds=50) - curr70 = curr + datetime.timedelta(seconds=70) - - add2.schedule(args=('k', 'v'), delay=60) - task1 = res_huey.dequeue() - - add2.schedule(args=('k2', 'v2'), delay=600) - task2 = res_huey.dequeue() - - add2('k3', 'v3') - task3 = res_huey.dequeue() - - # add the command to the schedule - res_huey.add_schedule(task1) - res_huey.add_schedule(task2) - res_huey.add_schedule(task3) - - # sanity check what should be run - self.assertFalse(res_huey.ready_to_run(task1)) - self.assertFalse(res_huey.ready_to_run(task2)) - self.assertTrue(res_huey.ready_to_run(task3)) - - self.assertFalse(res_huey.ready_to_run(task1, curr50)) - self.assertFalse(res_huey.ready_to_run(task2, curr50)) - self.assertTrue(res_huey.ready_to_run(task3, curr50)) - - self.assertTrue(res_huey.ready_to_run(task1, curr70)) - self.assertFalse(res_huey.ready_to_run(task2, curr70)) - self.assertTrue(res_huey.ready_to_run(task3, curr70)) diff --git a/lib/huey/tests/utils.py b/lib/huey/tests/utils.py deleted file mode 100644 index 3489932..0000000 --- a/lib/huey/tests/utils.py +++ /dev/null @@ -1,24 +0,0 @@ -import unittest - -from huey.utils import wrap_exception - - -class MyException(Exception): - pass - - -class TestWrapException(unittest.TestCase): - def test_wrap_exception(self): - def raise_keyerror(): - try: - {}['huey'] - except KeyError as exc: - raise wrap_exception(MyException) - - self.assertRaises(MyException, raise_keyerror) - try: - raise_keyerror() - except MyException as exc: - self.assertEqual(str(exc), "KeyError: 'huey'") - else: - assert False diff --git a/lib/huey/utils.py b/lib/huey/utils.py deleted file mode 100644 index 747d8da..0000000 --- a/lib/huey/utils.py +++ /dev/null @@ -1,21 +0,0 @@ -import datetime -import sys -import time - - -class EmptyData(object): - pass - - -def load_class(s): - path, klass = s.rsplit('.', 1) - __import__(path) - mod = sys.modules[path] - return getattr(mod, klass) - -def wrap_exception(new_exc_class): - exc_class, exc, tb = sys.exc_info() - raise new_exc_class('%s: %s' % (exc_class.__name__, exc)) - -def local_to_utc(dt): - return datetime.datetime(*time.gmtime(time.mktime(dt.timetuple()))[:6]) diff --git a/libtisbackup/XenAPI.py b/libtisbackup/XenAPI.py index 092b4fe..93840ad 100644 --- a/libtisbackup/XenAPI.py +++ b/libtisbackup/XenAPI.py @@ -1,20 +1,30 @@ -#============================================================================ -# This library is free software; you can redistribute it and/or -# modify it under the terms of version 2.1 of the GNU Lesser General Public -# License as published by the Free Software Foundation. +# Copyright (c) Citrix Systems, Inc. +# All rights reserved. # -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: # -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -#============================================================================ -# Copyright (C) 2006-2007 XenSource Inc. -#============================================================================ +# 1) Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. # +# 2) Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# -------------------------------------------------------------------- # Parts of this file are based upon xmlrpclib.py, the XML-RPC client # interface included in the Python distribution. # @@ -45,27 +55,16 @@ # -------------------------------------------------------------------- import gettext -import xmlrpclib -import httplib +import six.moves.xmlrpc_client as xmlrpclib +import six.moves.http_client as httplib import socket +import sys translation = gettext.translation('xen-xm', fallback = True) API_VERSION_1_1 = '1.1' API_VERSION_1_2 = '1.2' -# -# Methods that have different parameters between API versions 1.1 and 1.2, and -# the number of parameters in 1.1. -# -COMPATIBILITY_METHODS_1_1 = [ - ('SR.create' , 8), - ('SR.introduce' , 6), - ('SR.make' , 7), - ('VDI.snapshot' , 1), - ('VDI.clone' , 1), - ] - class Failure(Exception): def __init__(self, details): self.details = details @@ -73,17 +72,18 @@ class Failure(Exception): def __str__(self): try: return str(self.details) - except Exception, exn: - import sys - print >>sys.stderr, exn - return "Xen-API failure: %s" % str(self.details) + except Exception as exn: + msg = "Xen-API failure: %s" % exn + sys.stderr.write(msg) + return msg def _details_map(self): return dict([(str(i), self.details[i]) for i in range(len(self.details))]) -_RECONNECT_AND_RETRY = (lambda _ : ()) +# Just a "constant" that we use to decide whether to retry the RPC +_RECONNECT_AND_RETRY = object() class UDSHTTPConnection(httplib.HTTPConnection): """HTTPConnection subclass to allow HTTP over Unix domain sockets. """ @@ -92,12 +92,26 @@ class UDSHTTPConnection(httplib.HTTPConnection): self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.sock.connect(path) -class UDSHTTP(httplib.HTTP): +class UDSHTTP(httplib.HTTPConnection): _connection_class = UDSHTTPConnection class UDSTransport(xmlrpclib.Transport): + def __init__(self, use_datetime=0): + self._use_datetime = use_datetime + self._extra_headers=[] + self._connection = (None, None) + def add_extra_header(self, key, value): + self._extra_headers += [ (key,value) ] def make_connection(self, host): - return UDSHTTP(host) + # Python 2.4 compatibility + if sys.version_info[0] <= 2 and sys.version_info[1] < 7: + return UDSHTTP(host) + else: + return UDSHTTPConnection(host) + def send_request(self, connection, handler, request_body): + connection.putrequest("POST", handler) + for key, value in self._extra_headers: + connection.putheader(key, value) class Session(xmlrpclib.ServerProxy): """A server proxy and session manager for communicating with xapi using @@ -106,15 +120,26 @@ class Session(xmlrpclib.ServerProxy): Example: session = Session('http://localhost/') - session.login_with_password('me', 'mypassword') + session.login_with_password('me', 'mypassword', '1.0', 'xen-api-scripts-xenapi.py') session.xenapi.VM.start(vm_uuid) session.xenapi.session.logout() """ def __init__(self, uri, transport=None, encoding=None, verbose=0, - allow_none=1): - xmlrpclib.ServerProxy.__init__(self, uri, transport, encoding, - verbose, allow_none) + allow_none=1, ignore_ssl=False): + + # Fix for CA-172901 (+ Python 2.4 compatibility) + # Fix for context=ctx ( < Python 2.7.9 compatibility) + if not (sys.version_info[0] <= 2 and sys.version_info[1] <= 7 and sys.version_info[2] <= 9 ) \ + and ignore_ssl: + import ssl + ctx = ssl._create_unverified_context() + xmlrpclib.ServerProxy.__init__(self, uri, transport, encoding, + verbose, allow_none, context=ctx) + else: + xmlrpclib.ServerProxy.__init__(self, uri, transport, encoding, + verbose, allow_none) + self.transport = transport self._session = None self.last_login_method = None self.last_login_params = None @@ -125,7 +150,7 @@ class Session(xmlrpclib.ServerProxy): if methodname.startswith('login'): self._login(methodname, params) return None - elif methodname == 'logout': + elif methodname == 'logout' or methodname == 'session.logout': self._logout() return None else: @@ -133,7 +158,7 @@ class Session(xmlrpclib.ServerProxy): while retry_count < 3: full_params = (self._session,) + params result = _parse_result(getattr(self, methodname)(*full_params)) - if result == _RECONNECT_AND_RETRY: + if result is _RECONNECT_AND_RETRY: retry_count += 1 if self.last_login_method: self._login(self.last_login_method, @@ -145,21 +170,24 @@ class Session(xmlrpclib.ServerProxy): raise xmlrpclib.Fault( 500, 'Tried 3 times to get a valid session, but failed') - def _login(self, method, params): - result = _parse_result(getattr(self, 'session.%s' % method)(*params)) - if result == _RECONNECT_AND_RETRY: - raise xmlrpclib.Fault( - 500, 'Received SESSION_INVALID when logging in') - self._session = result - self.last_login_method = method - self.last_login_params = params - if method.startswith("slave_local"): - self.API_version = API_VERSION_1_2 - else: + try: + result = _parse_result( + getattr(self, 'session.%s' % method)(*params)) + if result is _RECONNECT_AND_RETRY: + raise xmlrpclib.Fault( + 500, 'Received SESSION_INVALID when logging in') + self._session = result + self.last_login_method = method + self.last_login_params = params self.API_version = self._get_api_version() + except socket.error as e: + if e.errno == socket.errno.ETIMEDOUT: + raise xmlrpclib.Fault(504, 'The connection timed out') + else: + raise e - def logout(self): + def _logout(self): try: if self.last_login_method.startswith("slave_local"): return _parse_result(self.session.local_logout(self._session)) @@ -174,11 +202,9 @@ class Session(xmlrpclib.ServerProxy): def _get_api_version(self): pool = self.xenapi.pool.get_all()[0] host = self.xenapi.pool.get_master(pool) - if (self.xenapi.host.get_API_version_major(host) == "1" and - self.xenapi.host.get_API_version_minor(host) == "2"): - return API_VERSION_1_2 - else: - return API_VERSION_1_1 + major = self.xenapi.host.get_API_version_major(host) + minor = self.xenapi.host.get_API_version_minor(host) + return "%s.%s"%(major,minor) def __getattr__(self, name): if name == 'handle': @@ -187,11 +213,13 @@ class Session(xmlrpclib.ServerProxy): return _Dispatcher(self.API_version, self.xenapi_request, None) elif name.startswith('login') or name.startswith('slave_local'): return lambda *params: self._login(name, params) + elif name == 'logout': + return _Dispatcher(self.API_version, self.xenapi_request, "logout") else: return xmlrpclib.ServerProxy.__getattr__(self, name) def xapi_local(): - return Session("http://_var_xapi_xapi/", transport=UDSTransport()) + return Session("http://_var_lib_xcp_xapi/", transport=UDSTransport()) def _parse_result(result): if type(result) != dict or 'Status' not in result: @@ -233,10 +261,4 @@ class _Dispatcher: return _Dispatcher(self.__API_version, self.__send, "%s.%s" % (self.__name, name)) def __call__(self, *args): - if self.__API_version == API_VERSION_1_1: - for m in COMPATIBILITY_METHODS_1_1: - if self.__name == m[0]: - return self.__send(self.__name, args[0:m[1]]) - return self.__send(self.__name, args) - diff --git a/libtisbackup/backup_mysql.py b/libtisbackup/backup_mysql.py index e45bf5a..69446d0 100644 --- a/libtisbackup/backup_mysql.py +++ b/libtisbackup/backup_mysql.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -*- coding: utf-8 -*- # ----------------------------------------------------------------------- # This file is part of TISBackup @@ -24,13 +24,13 @@ import sys try: sys.stderr = open('/dev/null') # Silence silly warnings from paramiko import paramiko -except ImportError,e: - print "Error : can not load paramiko library %s" % e +except ImportError as e: + print(("Error : can not load paramiko library %s" % e)) raise sys.stderr = sys.__stderr__ -from common import * +from libtisbackup.common import * class backup_mysql(backup_generic): """Backup a mysql database as gzipped sql file through ssh""" @@ -52,7 +52,7 @@ class backup_mysql(backup_generic): if not self.dry_run: os.makedirs(self.dest_dir) else: - print 'mkdir "%s"' % self.dest_dir + print(('mkdir "%s"' % self.dest_dir)) else: raise Exception('backup destination directory already exists : %s' % self.dest_dir) @@ -101,7 +101,7 @@ class backup_mysql(backup_generic): self.logger.debug('[%s] Dump DB : %s',self.backup_name,cmd) if not self.dry_run: (error_code,output) = ssh_exec(cmd,ssh=self.ssh) - print output + print(output) self.logger.debug("[%s] Output of %s :\n%s",self.backup_name,cmd,output) if error_code: raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd)) @@ -176,4 +176,4 @@ class backup_mysql(backup_generic): self.logger.info('Skipping %s, already registered',dir_name) -register_driver(backup_mysql) \ No newline at end of file +register_driver(backup_mysql) diff --git a/libtisbackup/backup_null.py b/libtisbackup/backup_null.py index f4d92b3..15b8a57 100755 --- a/libtisbackup/backup_null.py +++ b/libtisbackup/backup_null.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -*- coding: utf-8 -*- # ----------------------------------------------------------------------- # This file is part of TISBackup @@ -20,7 +20,7 @@ import os import datetime -from common import * +from .common import * class backup_null(backup_generic): diff --git a/libtisbackup/backup_oracle.py b/libtisbackup/backup_oracle.py index 6f1a3e6..cb76b16 100644 --- a/libtisbackup/backup_oracle.py +++ b/libtisbackup/backup_oracle.py @@ -21,8 +21,8 @@ import sys try: sys.stderr = open('/dev/null') # Silence silly warnings from paramiko import paramiko -except ImportError,e: - print "Error : can not load paramiko library %s" % e +except ImportError as e: + print(("Error : can not load paramiko library %s" % e)) raise sys.stderr = sys.__stderr__ @@ -30,7 +30,7 @@ sys.stderr = sys.__stderr__ import datetime import base64 import os -from common import * +from libtisbackup.common import * import re class backup_oracle(backup_generic): @@ -49,8 +49,8 @@ class backup_oracle(backup_generic): try: mykey = paramiko.RSAKey.from_private_key_file(self.private_key) except paramiko.SSHException: - #mykey = paramiko.DSSKey.from_private_key_file(self.private_key) - mykey = paramiko.Ed25519Key.from_private_key_file(self.private_key) + #mykey = paramiko.DSSKey.from_private_key_file(self.private_key) + mykey = paramiko.Ed25519Key.from_private_key_file(self.private_key) self.ssh = paramiko.SSHClient() self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) @@ -66,7 +66,7 @@ class backup_oracle(backup_generic): if not self.dry_run: os.makedirs(self.dest_dir) else: - print 'mkdir "%s"' % self.dest_dir + print(('mkdir "%s"' % self.dest_dir)) else: raise Exception('backup destination directory already exists : %s' % self.dest_dir) # dump db @@ -171,4 +171,4 @@ class backup_oracle(backup_generic): if error_code: raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd)) -register_driver(backup_oracle) \ No newline at end of file +register_driver(backup_oracle) diff --git a/libtisbackup/backup_pgsql.py b/libtisbackup/backup_pgsql.py index 86c443b..41983d3 100644 --- a/libtisbackup/backup_pgsql.py +++ b/libtisbackup/backup_pgsql.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -*- coding: utf-8 -*- # ----------------------------------------------------------------------- # This file is part of TISBackup @@ -21,13 +21,13 @@ import sys try: sys.stderr = open('/dev/null') # Silence silly warnings from paramiko import paramiko -except ImportError,e: - print "Error : can not load paramiko library %s" % e +except ImportError as e: + print(("Error : can not load paramiko library %s" % e)) raise sys.stderr = sys.__stderr__ -from common import * +from .common import * class backup_pgsql(backup_generic): """Backup a postgresql database as gzipped sql file through ssh""" @@ -46,7 +46,7 @@ class backup_pgsql(backup_generic): if not self.dry_run: os.makedirs(self.dest_dir) else: - print 'mkdir "%s"' % self.dest_dir + print(('mkdir "%s"' % self.dest_dir)) else: raise Exception('backup destination directory already exists : %s' % self.dest_dir) diff --git a/libtisbackup/backup_rdiff.py b/libtisbackup/backup_rdiff.py deleted file mode 100644 index ea63663..0000000 --- a/libtisbackup/backup_rdiff.py +++ /dev/null @@ -1,127 +0,0 @@ -# ----------------------------------------------------------------------- -# This file is part of TISBackup -# -# TISBackup is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# TISBackup is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with TISBackup. If not, see . -# -# ----------------------------------------------------------------------- - - -import os -import datetime -from common import * -import time - -class backup_rdiff: - backup_dir='' - backup_start_date=None - backup_name='' - server_name='' - exclude_list='' - ssh_port='22' - remote_user='root' - remote_dir='' - dest_dir='' - verbose = False - dry_run=False - - - - def __init__(self, backup_name, backup_base_dir): - self.backup_dir = backup_base_dir + '/' + backup_name - - if os.path.isdir(self.backup_dir )==False: - os.makedirs(self.backup_dir) - - self.backup_name = backup_name - t = datetime.datetime.now() - self.backup_start_date = t.strftime('%Y%m%d-%Hh%Mm%S') - - def get_latest_backup(self): - filelist = os.listdir(self.backup_dir) - if len(filelist) == 0: - return '' - - filelist.sort() - - return filelist[-1] - - def cleanup_backup(self): - filelist = os.listdir(self.backup_dir) - if len(filelist) == 0: - return '' - - filelist.sort() - for backup_date in filelist: - today = time.time() - print backup_date - datestring = backup_date[0:8] - c = time.strptime(datestring,"%Y%m%d") - # TODO: improve - if today - c < 60 * 60 * 24* 30: - print time.strftime("%Y%m%d",c) + " is to be deleted" - - - def copy_latest_to_new(self): - # TODO check that latest exist - # TODO check that new does not exist - - - last_backup = self.get_latest_backup() - if last_backup=='': - print "*********************************" - print "*first backup for " + self.backup_name - else: - latest_backup_path = self.backup_dir + '/' + last_backup - new_backup_path = self.backup_dir + '/' + self.backup_start_date - print "#cp -al starting" - cmd = 'cp -al ' + latest_backup_path + ' ' + new_backup_path - print cmd - if self.dry_run==False: - call_external_process(cmd) - print "#cp -al finished" - - - def rsync_to_new(self): - - self.dest_dir = self.backup_dir + '/' + self.backup_start_date + '/' - src_server = self.remote_user + '@' + self.server_name + ':"' + self.remote_dir.strip() + '/"' - - print "#starting rsync" - verbose_arg="" - if self.verbose==True: - verbose_arg = "-P " - - cmd = "rdiff-backup " + verbose_arg + ' --compress-level=9 --numeric-ids -az --partial -e "ssh -o StrictHostKeyChecking=no -p ' + self.ssh_port + ' -i ' + self.private_key + '" --stats --delete-after ' + self.exclude_list + ' ' + src_server + ' ' + self.dest_dir - print cmd - - ## deal with exit code 24 (file vanished) - if self.dry_run==False: - p = subprocess.call(cmd, shell=True) - if (p ==24): - print "Note: some files vanished before transfer" - if (p != 0 and p != 24 ): - raise Exception('shell program exited with error code ' + str(p), cmd) - - - print "#finished rsync" - - def process_backup(self): - print "" - print "#========Starting backup item =========" - self.copy_latest_to_new() - - self.rsync_to_new() - print "#========Backup item finished==========" - - diff --git a/libtisbackup/backup_rsync.py b/libtisbackup/backup_rsync.py index 43723ec..72a5cfc 100644 --- a/libtisbackup/backup_rsync.py +++ b/libtisbackup/backup_rsync.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -*- coding: utf-8 -*- # ----------------------------------------------------------------------- # This file is part of TISBackup @@ -20,7 +20,7 @@ import os import datetime -from common import * +from libtisbackup.common import * import time import logging import re @@ -69,7 +69,7 @@ class backup_rsync(backup_generic): if not self.dry_run: os.makedirs(dest_dir) else: - print 'mkdir "%s"' % dest_dir + print(('mkdir "%s"' % dest_dir)) else: raise Exception('backup destination directory already exists : %s' % dest_dir) @@ -80,7 +80,7 @@ class backup_rsync(backup_generic): if self.dry_run: options.append('-d') - if self.overload_args <> None: + if self.overload_args != None: options.append(self.overload_args) elif not "cygdrive" in self.remote_dir: # we don't preserve owner, group, links, hardlinks, perms for windows/cygwin as it is not reliable nor useful @@ -118,7 +118,7 @@ class backup_rsync(backup_generic): try: # newsettings with exclude_list='too','titi', parsed as a str python list content excludes = eval('[%s]' % self.exclude_list) - except Exception,e: + except Exception as e: raise Exception('Error reading exclude list : value %s, eval error %s (don\'t forget quotes and comma...)' % (self.exclude_list,e)) options.extend(['--exclude="%s"' % x for x in excludes]) @@ -146,13 +146,13 @@ class backup_rsync(backup_generic): ssh_params.append('-i %s' % self.private_key) if self.cipher_spec: ssh_params.append('-c %s' % self.cipher_spec) - if self.ssh_port <> 22: + if self.ssh_port != 22: ssh_params.append('-p %i' % self.ssh_port) options.append('-e "/usr/bin/ssh %s"' % (" ".join(ssh_params))) backup_source = '%s@%s:%s' % (self.remote_user,self.server_name,self.remote_dir) # ensure there is a slash at end - if backup_source[-1] <> '/': + if backup_source[-1] != '/': backup_source += '/' options_params = " ".join(options) @@ -165,7 +165,7 @@ class backup_rsync(backup_generic): process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) def ondata(data,context): if context.verbose: - print data + print(data) context.logger.debug(data) log = monitor_stdout(process,ondata,self) @@ -195,7 +195,7 @@ class backup_rsync(backup_generic): self.logger.error("[" + self.backup_name + "] shell program exited with error code " + str(returncode)) raise Exception("[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd, log[-512:]) else: - print cmd + print(cmd) #we suppress the .rsync suffix if everything went well finaldest = os.path.join(self.backup_dir,self.backup_start_date) @@ -203,14 +203,14 @@ class backup_rsync(backup_generic): if not self.dry_run: os.rename(dest_dir, finaldest) self.logger.debug("[%s] touching datetime of target directory %s" ,self.backup_name,finaldest) - print os.popen('touch "%s"' % finaldest).read() + print((os.popen('touch "%s"' % finaldest).read())) else: - print "mv" ,dest_dir,finaldest + print(("mv" ,dest_dir,finaldest)) stats['backup_location'] = finaldest stats['status']='OK' stats['log']='ssh+rsync backup from %s OK, %d bytes written for %d changed files' % (backup_source,stats['written_bytes'],stats['written_files_count']) - except BaseException , e: + except BaseException as e: stats['status']='ERROR' stats['log']=str(e) raise @@ -340,5 +340,5 @@ if __name__=='__main__': b = backup_rsync('htouvet','/backup/data/htouvet',dbstat) b.read_config(cp) b.process_backup() - print b.checknagios() + print((b.checknagios())) diff --git a/libtisbackup/backup_rsync_btrfs.py b/libtisbackup/backup_rsync_btrfs.py index a978c6d..8bd3a27 100644 --- a/libtisbackup/backup_rsync_btrfs.py +++ b/libtisbackup/backup_rsync_btrfs.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -*- coding: utf-8 -*- # ----------------------------------------------------------------------- # This file is part of TISBackup @@ -20,13 +20,13 @@ import os import datetime -from common import * +from .common import * import time import logging import re import os.path import datetime -from common import * +from .common import * class backup_rsync_btrfs(backup_generic): @@ -78,7 +78,7 @@ class backup_rsync_btrfs(backup_generic): else: self.logger.info("[" + self.backup_name + "] create btrs volume: %s"%dest_dir) else: - print 'btrfs subvolume create "%s"' %dest_dir + print(('btrfs subvolume create "%s"' %dest_dir)) @@ -89,7 +89,7 @@ class backup_rsync_btrfs(backup_generic): if self.dry_run: options.append('-d') - if self.overload_args <> None: + if self.overload_args != None: options.append(self.overload_args) elif not "cygdrive" in self.remote_dir: # we don't preserve owner, group, links, hardlinks, perms for windows/cygwin as it is not reliable nor useful @@ -128,7 +128,7 @@ class backup_rsync_btrfs(backup_generic): try: # newsettings with exclude_list='too','titi', parsed as a str python list content excludes = eval('[%s]' % self.exclude_list) - except Exception,e: + except Exception as e: raise Exception('Error reading exclude list : value %s, eval error %s (don\'t forget quotes and comma...)' % (self.exclude_list,e)) options.extend(['--exclude="%s"' % x for x in excludes]) @@ -154,13 +154,13 @@ class backup_rsync_btrfs(backup_generic): ssh_params.append('-i %s' % self.private_key) if self.cipher_spec: ssh_params.append('-c %s' % self.cipher_spec) - if self.ssh_port <> 22: + if self.ssh_port != 22: ssh_params.append('-p %i' % self.ssh_port) options.append('-e "/usr/bin/ssh %s"' % (" ".join(ssh_params))) backup_source = '%s@%s:%s' % (self.remote_user,self.server_name,self.remote_dir) # ensure there is a slash at end - if backup_source[-1] <> '/': + if backup_source[-1] != '/': backup_source += '/' options_params = " ".join(options) @@ -173,7 +173,7 @@ class backup_rsync_btrfs(backup_generic): process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) def ondata(data,context): if context.verbose: - print data + print(data) context.logger.debug(data) log = monitor_stdout(process,ondata,self) @@ -203,7 +203,7 @@ class backup_rsync_btrfs(backup_generic): self.logger.error("[" + self.backup_name + "] shell program exited with error code ", str(returncode)) raise Exception("[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd, log[-512:]) else: - print cmd + print(cmd) #we take a snapshot of last_backup if everything went well finaldest = os.path.join(self.backup_dir,self.backup_start_date) @@ -220,16 +220,16 @@ class backup_rsync_btrfs(backup_generic): else: self.logger.info("[" + self.backup_name + "] snapshot directory created %s"%finaldest) else: - print "btrfs snapshot of %s to %s"%(dest_dir,finaldest) + print(("btrfs snapshot of %s to %s"%(dest_dir,finaldest))) else: raise Exception('snapshot directory already exists : %s' %finaldest) self.logger.debug("[%s] touching datetime of target directory %s" ,self.backup_name,finaldest) - print os.popen('touch "%s"' % finaldest).read() + print((os.popen('touch "%s"' % finaldest).read())) stats['backup_location'] = finaldest stats['status']='OK' stats['log']='ssh+rsync+btrfs backup from %s OK, %d bytes written for %d changed files' % (backup_source,stats['written_bytes'],stats['written_files_count']) - except BaseException , e: + except BaseException as e: stats['status']='ERROR' stats['log']=str(e) raise @@ -358,5 +358,5 @@ if __name__=='__main__': b = backup_rsync('htouvet','/backup/data/htouvet',dbstat) b.read_config(cp) b.process_backup() - print b.checknagios() + print((b.checknagios())) diff --git a/libtisbackup/backup_samba4.py b/libtisbackup/backup_samba4.py index 4ce284b..d716868 100644 --- a/libtisbackup/backup_samba4.py +++ b/libtisbackup/backup_samba4.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -*- coding: utf-8 -*- # ----------------------------------------------------------------------- # This file is part of TISBackup @@ -24,13 +24,13 @@ import sys try: sys.stderr = open('/dev/null') # Silence silly warnings from paramiko import paramiko -except ImportError,e: - print "Error : can not load paramiko library %s" % e +except ImportError as e: + print("Error : can not load paramiko library %s" % e) raise sys.stderr = sys.__stderr__ -from common import * +from .common import * class backup_samba4(backup_generic): """Backup a samba4 databases as gzipped tdbs file through ssh""" @@ -47,7 +47,7 @@ class backup_samba4(backup_generic): if not self.dry_run: os.makedirs(self.dest_dir) else: - print 'mkdir "%s"' % self.dest_dir + print('mkdir "%s"' % self.dest_dir) else: raise Exception('backup destination directory already exists : %s' % self.dest_dir) @@ -88,7 +88,7 @@ class backup_samba4(backup_generic): self.logger.debug('[%s] Dump DB : %s',self.backup_name,cmd) if not self.dry_run: (error_code,output) = ssh_exec(cmd,ssh=self.ssh) - print output + print(output) self.logger.debug("[%s] Output of %s :\n%s",self.backup_name,cmd,output) if error_code: raise Exception('Aborting, Not null exit code (%i) for "%s"' % (error_code,cmd)) diff --git a/libtisbackup/backup_sqlserver.py b/libtisbackup/backup_sqlserver.py index e155e13..9c1cb9d 100644 --- a/libtisbackup/backup_sqlserver.py +++ b/libtisbackup/backup_sqlserver.py @@ -24,8 +24,8 @@ import sys try: sys.stderr = open('/dev/null') # Silence silly warnings from paramiko import paramiko -except ImportError,e: - print "Error : can not load paramiko library %s" % e +except ImportError as e: + print("Error : can not load paramiko library %s" % e) raise sys.stderr = sys.__stderr__ @@ -33,7 +33,7 @@ sys.stderr = sys.__stderr__ import datetime import base64 import os -from common import * +from .common import * class backup_sqlserver(backup_generic): """Backup a SQLSERVER database as gzipped sql file through ssh""" @@ -67,18 +67,17 @@ class backup_sqlserver(backup_generic): backup_start_date = t.strftime('%Y%m%d-%Hh%Mm%S') backup_file = self.remote_backup_dir + '/' + self.db_name + '-' + backup_start_date + '.bak' - if not self.db_user == '': - self.userdb = '-U %s -P %s' % ( self.db_user, self.db_password ) + if not self.db_user == '': + self.userdb = '-U %s -P %s' % ( self.db_user, self.db_password ) # dump db stats['status']='Dumping' - if self.sqlserver_before_2005: - cmd = """osql -E -Q "BACKUP DATABASE [%s] + if self.sqlserver_before_2005: + cmd = """osql -E -Q "BACKUP DATABASE [%s] TO DISK='%s' WITH FORMAT" """ % ( self.db_name, backup_file ) - - else: - cmd = """sqlcmd %s -S "%s" -d master -Q "BACKUP DATABASE [%s] + else: + cmd = """sqlcmd %s -S "%s" -d master -Q "BACKUP DATABASE [%s] TO DISK = N'%s' WITH INIT, NOUNLOAD , NAME = N'Backup %s', NOSKIP ,STATS = 10, NOFORMAT" """ % (self.userdb, self.db_server_name, self.db_name, backup_file ,self.db_name ) diff --git a/libtisbackup/backup_switch.py b/libtisbackup/backup_switch.py index 5a25bac..a5d11a1 100644 --- a/libtisbackup/backup_switch.py +++ b/libtisbackup/backup_switch.py @@ -20,15 +20,15 @@ import os import datetime -from common import * -import XenAPI +from .common import * +from . import XenAPI import time import logging import re import os.path import datetime import select -import urllib2, urllib +import urllib.request, urllib.error, urllib.parse, urllib.request, urllib.parse, urllib.error import base64 import socket import requests @@ -149,8 +149,9 @@ class backup_switch(backup_generic): else: child.sendline(self.switch_user) child.expect(".*#") - child.sendline( "terminal datadump") - child.expect("#") + + child.sendline( "terminal datadump") + child.expect("#") child.sendline( "show startup-config") child.expect("#") lines = child.before @@ -237,7 +238,7 @@ class backup_switch(backup_generic): stats['log']='Switch backup from %s OK, %d bytes written' % (self.server_name,stats['written_bytes']) - except BaseException , e: + except BaseException as e: stats['status']='ERROR' stats['log']=str(e) raise diff --git a/libtisbackup/backup_vmdk.py b/libtisbackup/backup_vmdk.py index 6719060..434d66d 100755 --- a/libtisbackup/backup_vmdk.py +++ b/libtisbackup/backup_vmdk.py @@ -17,8 +17,8 @@ # along with TISBackup. If not, see . # # ----------------------------------------------------------------------- -from __future__ import with_statement -from common import * + +from .common import * import pyVmomi from pyVmomi import vim from pyVmomi import vmodl @@ -101,7 +101,7 @@ class backup_vmdk(backup_generic): ovfDescParams = vim.OvfManager.CreateDescriptorParams() ovf = si.content.ovfManager.CreateDescriptor(vm, ovfDescParams) root = ET.fromstring(ovf.ovfDescriptor) - new_id = root[0][1].attrib.values()[0][1:3] + new_id = list(root[0][1].attrib.values())[0][1:3] ovfFiles = [] for vmdk in vmdks: old_id = vmdk['id'][1:3] @@ -211,7 +211,7 @@ class backup_vmdk(backup_generic): if not self.dry_run: os.makedirs(dest_dir) else: - print 'mkdir "%s"' % dest_dir + print('mkdir "%s"' % dest_dir) else: raise Exception('backup destination directory already exists : %s' % dest_dir) os.chdir(dest_dir) @@ -271,7 +271,7 @@ class backup_vmdk(backup_generic): stats['status']='OK' - except BaseException , e: + except BaseException as e: stats['status']='ERROR' stats['log']=str(e) raise diff --git a/libtisbackup/backup_xcp_metadata.py b/libtisbackup/backup_xcp_metadata.py index 204f363..da125ee 100644 --- a/libtisbackup/backup_xcp_metadata.py +++ b/libtisbackup/backup_xcp_metadata.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -*- coding: utf-8 -*- # ----------------------------------------------------------------------- # This file is part of TISBackup @@ -20,7 +20,7 @@ -from common import * +from .common import * import paramiko class backup_xcp_metadata(backup_generic): diff --git a/libtisbackup/backup_xva.py b/libtisbackup/backup_xva.py index cd4e683..d91634d 100755 --- a/libtisbackup/backup_xva.py +++ b/libtisbackup/backup_xva.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -*- coding: utf-8 -*- # ----------------------------------------------------------------------- # This file is part of TISBackup @@ -17,20 +17,21 @@ # along with TISBackup. If not, see . # # ----------------------------------------------------------------------- -from __future__ import with_statement + import logging import re import os import datetime -import urllib +import urllib.request, urllib.parse, urllib.error import socket import tarfile import hashlib from stat import * import ssl +import requests -from common import * -import XenAPI +from .common import * +from . import XenAPI if hasattr(ssl, '_create_unverified_context'): ssl._create_default_https_context = ssl._create_unverified_context @@ -72,7 +73,7 @@ class backup_xva(backup_generic): session = XenAPI.Session('https://'+self.xcphost) try: session.login_with_password(user_xen,password_xen) - except XenAPI.Failure, error: + except XenAPI.Failure as error: msg,ip = error.details if msg == 'HOST_IS_SLAVE': @@ -117,7 +118,7 @@ class backup_xva(backup_generic): if not 'NULL' in vdi: session.xenapi.VDI.destroy(vdi) session.xenapi.VM.destroy(old_snapshot) - except XenAPI.Failure, error: + except XenAPI.Failure as error: return("error when destroy snapshot %s"%(error)) now = datetime.datetime.now() @@ -125,7 +126,7 @@ class backup_xva(backup_generic): try: snapshot = session.xenapi.VM.snapshot(vm,"tisbackup-%s"%(vdi_name)) self.logger.debug("[%s] got snapshot %s", vdi_name, snapshot) - except XenAPI.Failure, error: + except XenAPI.Failure as error: return("error when snapshot %s"%(error)) #get snapshot opaqueRef vm = session.xenapi.VM.get_by_name_label("tisbackup-%s"%(vdi_name))[0] @@ -135,7 +136,7 @@ class backup_xva(backup_generic): if status_vm == "Running": self.logger.debug("[%s] Shudown in progress",self.backup_name) if dry_run: - print "session.xenapi.VM.clean_shutdown(vm)" + print("session.xenapi.VM.clean_shutdown(vm)") else: session.xenapi.VM.clean_shutdown(vm) try: @@ -150,8 +151,13 @@ class backup_xva(backup_generic): scheme = "https://" url = scheme+user_xen+":"+password_xen+"@"+self.xcphost+"/export?use_compression="+self.use_compression+"&uuid="+session.xenapi.VM.get_uuid(vm) - urllib.urlretrieve(url, filename_temp) - urllib.urlcleanup() + + + + top_level_url = scheme+self.xcphost+"/export?use_compression="+self.use_compression+"&uuid="+session.xenapi.VM.get_uuid(vm) + r = requests.get(top_level_url, auth=(user_xen, password_xen)) + open(filename_temp, 'wb').write(r.content) + except Exception as e: self.logger.error("[%s] error when fetching snap: %s", "tisbackup-%s"%(vdi_name), e) @@ -171,13 +177,13 @@ class backup_xva(backup_generic): if not 'NULL' in vdi: session.xenapi.VDI.destroy(vdi) session.xenapi.VM.destroy(snapshot) - except XenAPI.Failure, error: + except XenAPI.Failure as error: return("error when destroy snapshot %s"%(error)) elif status_vm == "Running": self.logger.debug("[%s] Starting in progress",self.backup_name) if dry_run: - print "session.xenapi.Async.VM.start(vm,False,True)" + print("session.xenapi.Async.VM.start(vm,False,True)") else: session.xenapi.Async.VM.start(vm,False,True) @@ -219,7 +225,7 @@ class backup_xva(backup_generic): else: raise Exception(cmd) - except BaseException , e: + except BaseException as e: stats['status']='ERROR' stats['log']=str(e) raise diff --git a/libtisbackup/common.py b/libtisbackup/common.py index 0357505..39db683 100644 --- a/libtisbackup/common.py +++ b/libtisbackup/common.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -*- coding: utf-8 -*- # ----------------------------------------------------------------------- # This file is part of TISBackup @@ -18,6 +18,7 @@ # # ----------------------------------------------------------------------- +from abc import ABC, abstractmethod import os import subprocess import re @@ -34,8 +35,8 @@ import sys try: sys.stderr = open('/dev/null') # Silence silly warnings from paramiko import paramiko -except ImportError,e: - print "Error : can not load paramiko library %s" % e +except ImportError as e: + print(("Error : can not load paramiko library %s" % e)) raise sys.stderr = sys.__stderr__ @@ -121,7 +122,7 @@ def check_string(test_string): pattern = r'[^\.A-Za-z0-9\-_]' if re.search(pattern, test_string): #Character other then . a-z 0-9 was found - print 'Invalid : %r' % (test_string,) + print(('Invalid : %r' % (test_string,))) def convert_bytes(bytes): if bytes is None: @@ -207,7 +208,7 @@ def html_table(cur,callback=None): yield dict((cur.description[idx][0], value) for idx, value in enumerate(row)) - head=u""+"".join([""+c[0]+"" for c in cur.description])+"" + head=""+"".join([""+c[0]+"" for c in cur.description])+"" lines="" if callback: for r in itermap(cur): @@ -237,7 +238,7 @@ def monitor_stdout(aprocess, onoutputdata,context): while read_set: try: rlist, wlist, xlist = select.select(read_set, [], []) - except select.error, e: + except select.error as e: if e.args[0] == errno.EINTR: continue raise @@ -245,12 +246,14 @@ def monitor_stdout(aprocess, onoutputdata,context): # Reads one line from stdout if aprocess.stdout in rlist: data = os.read(aprocess.stdout.fileno(), 1) + data = data.decode('utf-8') if data == "": aprocess.stdout.close() read_set.remove(aprocess.stdout) while data and not data in ('\n','\r'): line += data data = os.read(aprocess.stdout.fileno(), 1) + data = data.decode('utf-8') if line or data in ('\n','\r'): stdout.append(line) if onoutputdata: @@ -260,12 +263,14 @@ def monitor_stdout(aprocess, onoutputdata,context): # Reads one line from stderr if aprocess.stderr in rlist: data = os.read(aprocess.stderr.fileno(), 1) + data = data.decode('utf-8') if data == "": aprocess.stderr.close() read_set.remove(aprocess.stderr) while data and not data in ('\n','\r'): line += data data = os.read(aprocess.stderr.fileno(), 1) + data = data.decode('utf-8') if line or data in ('\n','\r'): stdout.append(line) if onoutputdata: @@ -442,7 +447,7 @@ CREATE INDEX idx_stats_backup_name_start on stats(backup_name,backup_start);""") return value #for r in self.query('select * from stats where backup_name=? order by backup_end desc limit ?',(backup_name,count)): - print pp(cur,None,1,fcb) + print((pp(cur,None,1,fcb))) def fcb(self,fields,fieldname,value): @@ -492,12 +497,13 @@ def ssh_exec(command,ssh=None,server_name='',remote_user='',private_key='',ssh_p chan.exec_command(command) stdout.flush() - output = stdout.read() + output_base = stdout.read() + output = output_base.decode(encoding='UTF-8').replace("'","") exit_code = chan.recv_exit_status() return (exit_code,output) -class backup_generic: +class backup_generic(ABC): """Generic ancestor class for backups, not registered""" type = 'generic' required_params = ['type','backup_name','backup_dir','server_name','backup_retention_time','maximum_backup_age'] @@ -696,7 +702,7 @@ class backup_generic: self.logger.info('[%s] ######### Backup finished : %s',self.backup_name,stats['log']) return stats - except BaseException, e: + except BaseException as e: stats['status']='ERROR' stats['log']=str(e) endtime = time.time() @@ -798,7 +804,7 @@ class backup_generic: if not self.dry_run: self.dbstat.db.execute('update stats set TYPE="CLEAN" where backup_name=? and backup_location=?',(self.backup_name,oldbackup_location)) self.dbstat.db.commit() - except BaseException,e: + except BaseException as e: self.logger.error('cleanup_backup : Unable to remove directory/file "%s". Error %s', oldbackup_location,e) removed.append((self.backup_name,oldbackup_location)) else: @@ -809,10 +815,12 @@ class backup_generic: self.logger.info('[%s] Cleanup finished : removed : %s' , self.backup_name,','.join([('[%s]-"%s"') % r for r in removed]) or 'Nothing') return removed + @abstractmethod def register_existingbackups(self): - """scan existing backups and insert stats in database""" - registered = [b['backup_location'] for b in self.dbstat.query('select distinct backup_location from stats where backup_name=?',[self.backup_name])] - raise Exception('Abstract method') + pass + # """scan existing backups and insert stats in database""" + # registered = [b['backup_location'] for b in self.dbstat.query('select distinct backup_location from stats where backup_name=?',[self.backup_name])] + # raise Exception('Abstract method') def export_latestbackup(self,destdir): """Copy (rsync) latest OK backup to external storage located at locally mounted "destdir" @@ -846,9 +854,9 @@ class backup_generic: raise Exception('Backup source %s doesn\'t exists' % backup_source) # ensure there is a slash at end - if os.path.isdir(backup_source) and backup_source[-1] <> '/': + if os.path.isdir(backup_source) and backup_source[-1] != '/': backup_source += '/' - if backup_dest[-1] <> '/': + if backup_dest[-1] != '/': backup_dest += '/' if not os.path.isdir(backup_dest): @@ -874,7 +882,7 @@ class backup_generic: process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) def ondata(data,context): if context.verbose: - print data + print(data) context.logger.debug(data) log = monitor_stdout(process,ondata,self) @@ -898,7 +906,7 @@ class backup_generic: self.logger.error("[" + self.backup_name + "] shell program exited with error code ") raise Exception("[" + self.backup_name + "] shell program exited with error code " + str(returncode), cmd) else: - print cmd + print(cmd) stats['status']='OK' self.logger.info('export backup from %s to %s OK, %d bytes written for %d changed files' % (backup_source,backup_dest,stats['written_bytes'],stats['written_files_count'])) diff --git a/libtisbackup/copy_vm_xcp.py b/libtisbackup/copy_vm_xcp.py index 93bdd45..64acc17 100755 --- a/libtisbackup/copy_vm_xcp.py +++ b/libtisbackup/copy_vm_xcp.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -*- coding: utf-8 -*- # ----------------------------------------------------------------------- # This file is part of TISBackup @@ -20,8 +20,8 @@ import os import datetime -from common import * -import XenAPI +from .common import * +from . import XenAPI import time import logging import re @@ -29,7 +29,7 @@ import os.path import os import datetime import select -import urllib2 +import urllib.request, urllib.error, urllib.parse import base64 import socket from stat import * @@ -66,7 +66,7 @@ class copy_vm_xcp(backup_generic): session = XenAPI.Session('https://'+self.server_name) try: session.login_with_password(user_xen,password_xen) - except XenAPI.Failure, error: + except XenAPI.Failure as error: msg,ip = error.details if msg == 'HOST_IS_SLAVE': @@ -81,7 +81,7 @@ class copy_vm_xcp(backup_generic): #get storage opaqueRef try: storage = session.xenapi.SR.get_by_name_label(storage_name)[0] - except IndexError,error: + except IndexError as error: result = (1,"error get SR opaqueref %s"%(error)) return result @@ -89,14 +89,14 @@ class copy_vm_xcp(backup_generic): #get vm to copy opaqueRef try: vm = session.xenapi.VM.get_by_name_label(vm_name)[0] - except IndexError,error: + except IndexError as error: result = (1,"error get VM opaqueref %s"%(error)) return result # get vm backup network opaqueRef try: networkRef = session.xenapi.network.get_by_name_label(self.network_name)[0] - except IndexError, error: + except IndexError as error: result = (1, "error get VM network opaqueref %s" % (error)) return result @@ -104,9 +104,9 @@ class copy_vm_xcp(backup_generic): status_vm = session.xenapi.VM.get_power_state(vm) self.logger.debug("[%s] Status of VM: %s",self.backup_name,status_vm) if status_vm == "Running": - self.logger.debug("[%s] Shudown in progress",self.backup_name) + self.logger.debug("[%s] Shutdown in progress",self.backup_name) if dry_run: - print "session.xenapi.VM.clean_shutdown(vm)" + print("session.xenapi.VM.clean_shutdown(vm)") else: session.xenapi.VM.clean_shutdown(vm) snapshot = vm @@ -115,7 +115,7 @@ class copy_vm_xcp(backup_generic): self.logger.debug("[%s] Snapshot in progress",self.backup_name) try: snapshot = session.xenapi.VM.snapshot(vm,"tisbackup-%s"%(vm_name)) - except XenAPI.Failure, error: + except XenAPI.Failure as error: result = (1,"error when snapshot %s"%(error)) return result @@ -165,7 +165,7 @@ class copy_vm_xcp(backup_generic): session.xenapi.VDI.destroy(vdi) session.xenapi.VM.destroy(oldest_backup_vm) - except XenAPI.Failure, error: + except XenAPI.Failure as error: result = (1,"error when destroy old backup vm %s"%(error)) return result @@ -173,7 +173,7 @@ class copy_vm_xcp(backup_generic): self.logger.debug("[%s] Copy %s in progress on %s",self.backup_name,vm_name,storage_name) try: backup_vm = session.xenapi.VM.copy(snapshot,vm_backup_name+now.strftime("%Y-%m-%d %H:%M"),storage) - except XenAPI.Failure, error: + except XenAPI.Failure as error: result = (1,"error when copy %s"%(error)) return result @@ -184,7 +184,7 @@ class copy_vm_xcp(backup_generic): #change the network of the new VM try: vifDestroy = session.xenapi.VM.get_VIFs(backup_vm) - except IndexError,error: + except IndexError as error: result = (1,"error get VIF opaqueref %s"%(error)) return result @@ -213,7 +213,7 @@ class copy_vm_xcp(backup_generic): } try: session.xenapi.VIF.create(data) - except Exception, error: + except Exception as error: result = (1,error) return result @@ -237,7 +237,7 @@ class copy_vm_xcp(backup_generic): return result #Disable automatic boot - if session.xenapi.VM.get_other_config(backup_vm).has_key('auto_poweron'): + if 'auto_poweron' in session.xenapi.VM.get_other_config(backup_vm): session.xenapi.VM.remove_from_other_config(backup_vm, "auto_poweron") if not str2bool(self.halt_vm): @@ -251,14 +251,14 @@ class copy_vm_xcp(backup_generic): if not 'NULL' in vdi: session.xenapi.VDI.destroy(vdi) session.xenapi.VM.destroy(snapshot) - except XenAPI.Failure, error: + except XenAPI.Failure as error: result = (1,"error when destroy snapshot %s"%(error)) return result else: if status_vm == "Running": self.logger.debug("[%s] Starting in progress",self.backup_name) if dry_run: - print "session.xenapi.VM.start(vm,False,True)" + print("session.xenapi.VM.start(vm,False,True)") else: session.xenapi.VM.start(vm,False,True) @@ -282,9 +282,14 @@ class copy_vm_xcp(backup_generic): stats['status']='ERROR' stats['log']=cmd[1] - except BaseException,e: + except BaseException as e: stats['status']='ERROR' stats['log']=str(e) raise + def register_existingbackups(self): + """scan backup dir and insert stats in database""" + #This backup is on target server, no data available on this server + pass + register_driver(copy_vm_xcp) diff --git a/iniparse/__init__.py b/libtisbackup/iniparse/__init__.py old mode 100755 new mode 100644 similarity index 53% rename from iniparse/__init__.py rename to libtisbackup/iniparse/__init__.py index 8de756f..7193f92 --- a/iniparse/__init__.py +++ b/libtisbackup/iniparse/__init__.py @@ -3,17 +3,17 @@ # Copyright (c) 2007 Tim Lauridsen # All Rights Reserved. See LICENSE-PSF & LICENSE for details. -from ini import INIConfig, change_comment_syntax -from config import BasicConfig, ConfigNamespace -from compat import RawConfigParser, ConfigParser, SafeConfigParser -from utils import tidy +from .ini import INIConfig, change_comment_syntax +from .config import BasicConfig, ConfigNamespace +from .compat import RawConfigParser, ConfigParser, SafeConfigParser +from .utils import tidy -from ConfigParser import DuplicateSectionError, \ - NoSectionError, NoOptionError, \ - InterpolationMissingOptionError, \ - InterpolationDepthError, \ - InterpolationSyntaxError, \ - DEFAULTSECT, MAX_INTERPOLATION_DEPTH +from .configparser import DuplicateSectionError, \ + NoSectionError, NoOptionError, \ + InterpolationMissingOptionError, \ + InterpolationDepthError, \ + InterpolationSyntaxError, \ + DEFAULTSECT, MAX_INTERPOLATION_DEPTH __all__ = [ 'BasicConfig', 'ConfigNamespace', diff --git a/iniparse/compat.py b/libtisbackup/iniparse/compat.py old mode 100755 new mode 100644 similarity index 92% rename from iniparse/compat.py rename to libtisbackup/iniparse/compat.py index db89ed8..76eb1ea --- a/iniparse/compat.py +++ b/libtisbackup/iniparse/compat.py @@ -12,19 +12,22 @@ The underlying INIConfig object can be accessed as cfg.data """ import re -from ConfigParser import DuplicateSectionError, \ - NoSectionError, NoOptionError, \ - InterpolationMissingOptionError, \ - InterpolationDepthError, \ - InterpolationSyntaxError, \ - DEFAULTSECT, MAX_INTERPOLATION_DEPTH +from .configparser import DuplicateSectionError, \ + NoSectionError, NoOptionError, \ + InterpolationMissingOptionError, \ + InterpolationDepthError, \ + InterpolationSyntaxError, \ + DEFAULTSECT, MAX_INTERPOLATION_DEPTH # These are imported only for compatiability. # The code below does not reference them directly. -from ConfigParser import Error, InterpolationError, \ - MissingSectionHeaderError, ParsingError +from .configparser import Error, InterpolationError, \ + MissingSectionHeaderError, ParsingError + +import six + +from . import ini -import ini class RawConfigParser(object): def __init__(self, defaults=None, dict_type=dict): @@ -56,7 +59,7 @@ class RawConfigParser(object): # The default section is the only one that gets the case-insensitive # treatment - so it is special-cased here. if section.lower() == "default": - raise ValueError, 'Invalid section name: %s' % section + raise ValueError('Invalid section name: %s' % section) if self.has_section(section): raise DuplicateSectionError(section) @@ -68,7 +71,7 @@ class RawConfigParser(object): The DEFAULT section is not acknowledged. """ - return (section in self.data) + return section in self.data def options(self, section): """Return a list of option names for the given section name.""" @@ -88,7 +91,7 @@ class RawConfigParser(object): filename may also be given. """ files_read = [] - if isinstance(filenames, basestring): + if isinstance(filenames, six.string_types): filenames = [filenames] for filename in filenames: try: @@ -113,8 +116,6 @@ class RawConfigParser(object): def get(self, section, option, vars=None): if not self.has_section(section): raise NoSectionError(section) - if vars is not None and option in vars: - value = vars[option] sec = self.data[section] if option in sec: @@ -143,7 +144,7 @@ class RawConfigParser(object): def getboolean(self, section, option): v = self.get(section, option) if v.lower() not in self._boolean_states: - raise ValueError, 'Not a boolean: %s' % v + raise ValueError('Not a boolean: %s' % v) return self._boolean_states[v.lower()] def has_option(self, section, option): @@ -234,7 +235,7 @@ class ConfigParser(RawConfigParser): if "%(" in value: try: value = value % vars - except KeyError, e: + except KeyError as e: raise InterpolationMissingOptionError( option, section, rawval, e.args[0]) else: @@ -283,7 +284,7 @@ class SafeConfigParser(ConfigParser): _badpercent_re = re.compile(r"%[^%]|%$") def set(self, section, option, value): - if not isinstance(value, basestring): + if not isinstance(value, six.string_types): raise TypeError("option values must be strings") # check for bad percent signs: # first, replace all "good" interpolations @@ -323,8 +324,7 @@ class SafeConfigParser(ConfigParser): elif c == "(": m = self._interpvar_match(rest) if m is None: - raise InterpolationSyntaxError(option, section, - "bad interpolation variable reference %r" % rest) + raise InterpolationSyntaxError(option, section, "bad interpolation variable reference %r" % rest) var = m.group(1) rest = rest[m.end():] try: diff --git a/iniparse/config.py b/libtisbackup/iniparse/config.py old mode 100755 new mode 100644 similarity index 97% rename from iniparse/config.py rename to libtisbackup/iniparse/config.py index 5cfa2ea..d0b8423 --- a/iniparse/config.py +++ b/libtisbackup/iniparse/config.py @@ -86,6 +86,7 @@ class ConfigNamespace(object): def __setstate__(self, state): self.__dict__.update(state) + class Undefined(object): """Helper class used to hold undefined names until assignment. @@ -143,16 +144,16 @@ class BasicConfig(ConfigNamespace): >>> n.aaa = 42 >>> del n.x - >>> print n + >>> print(n) aaa = 42 name.first = paramjit name.last = oberoi - Nested namepsaces are also namespaces: + Nested namespaces are also namespaces: >>> isinstance(n.name, ConfigNamespace) True - >>> print n.name + >>> print(n.name) first = paramjit last = oberoi >>> sorted(list(n.name)) @@ -160,7 +161,7 @@ class BasicConfig(ConfigNamespace): Finally, values can be read from a file as follows: - >>> from StringIO import StringIO + >>> from six import StringIO >>> sio = StringIO(''' ... # comment ... ui.height = 100 @@ -171,7 +172,7 @@ class BasicConfig(ConfigNamespace): ... ''') >>> n = BasicConfig() >>> n._readfp(sio) - >>> print n + >>> print(n) complexity = medium data.secret.password = goodness=gracious me have_python @@ -199,7 +200,7 @@ class BasicConfig(ConfigNamespace): def __str__(self, prefix=''): lines = [] - keys = self._data.keys() + keys = list(self._data.keys()) keys.sort() for name in keys: value = self._data[name] @@ -258,7 +259,7 @@ def update_config(target, source): >>> n.ui.display_clock = True >>> n.ui.display_qlength = True >>> n.ui.width = 150 - >>> print n + >>> print(n) playlist.expand_playlist = True ui.display_clock = True ui.display_qlength = True @@ -267,7 +268,7 @@ def update_config(target, source): >>> from iniparse import ini >>> i = ini.INIConfig() >>> update_config(i, n) - >>> print i + >>> print(i) [playlist] expand_playlist = True @@ -277,7 +278,7 @@ def update_config(target, source): width = 150 """ - for name in source: + for name in sorted(source): value = source[name] if isinstance(value, ConfigNamespace): if name in target: @@ -289,6 +290,3 @@ def update_config(target, source): update_config(myns, value) else: target[name] = value - - - diff --git a/libtisbackup/iniparse/configparser.py b/libtisbackup/iniparse/configparser.py new file mode 100644 index 0000000..c543d50 --- /dev/null +++ b/libtisbackup/iniparse/configparser.py @@ -0,0 +1,7 @@ +try: + from ConfigParser import * + # not all objects get imported with __all__ + from ConfigParser import Error, InterpolationMissingOptionError +except ImportError: + from configparser import * + from configparser import Error, InterpolationMissingOptionError diff --git a/iniparse/ini.py b/libtisbackup/iniparse/ini.py old mode 100755 new mode 100644 similarity index 90% rename from iniparse/ini.py rename to libtisbackup/iniparse/ini.py index 68dd65c..dc47621 --- a/iniparse/ini.py +++ b/libtisbackup/iniparse/ini.py @@ -7,7 +7,7 @@ Example: - >>> from StringIO import StringIO + >>> from six import StringIO >>> sio = StringIO('''# configure foo-application ... [foo] ... bar1 = qualia @@ -16,14 +16,14 @@ Example: ... special = 1''') >>> cfg = INIConfig(sio) - >>> print cfg.foo.bar1 + >>> print(cfg.foo.bar1) qualia - >>> print cfg['foo-ext'].special + >>> print(cfg['foo-ext'].special) 1 >>> cfg.foo.newopt = 'hi!' >>> cfg.baz.enabled = 0 - >>> print cfg + >>> print(cfg) # configure foo-application [foo] bar1 = qualia @@ -42,9 +42,12 @@ Example: # Backward-compatiable with ConfigParser import re -from ConfigParser import DEFAULTSECT, ParsingError, MissingSectionHeaderError +from .configparser import DEFAULTSECT, ParsingError, MissingSectionHeaderError + +import six + +from . import config -import config class LineType(object): line = None @@ -73,10 +76,10 @@ class LineType(object): class SectionLine(LineType): - regex = re.compile(r'^\[' - r'(?P[^]]+)' - r'\]\s*' - r'((?P;|#)(?P.*))?$') + regex = re.compile(r'^\[' + r'(?P[^]]+)' + r'\]\s*' + r'((?P;|#)(?P.*))?$') def __init__(self, name, comment=None, comment_separator=None, comment_offset=-1, line=None): @@ -170,8 +173,9 @@ def change_comment_syntax(comment_chars='%;#', allow_rem=False): regex += r')(?P.*)$' CommentLine.regex = re.compile(regex) + class CommentLine(LineType): - regex = re.compile(r'^(?P[;#]|[rR][eE][mM] +)' + regex = re.compile(r'^(?P[;#])' r'(?P.*)$') def __init__(self, comment='', separator='#', line=None): @@ -187,6 +191,7 @@ class CommentLine(LineType): if m is None: return None return cls(m.group('comment'), m.group('csep'), line) + parse = classmethod(parse) @@ -195,11 +200,13 @@ class EmptyLine(LineType): def to_string(self): return '' - value = property(lambda _: '') + value = property(lambda self: '') def parse(cls, line): - if line.strip(): return None + if line.strip(): + return None return cls(line) + parse = classmethod(parse) @@ -221,6 +228,7 @@ class ContinuationLine(LineType): if m is None: return None return cls(m.group('value'), m.start('value'), line) + parse = classmethod(parse) @@ -275,6 +283,7 @@ class LineContainer(object): self.add(EmptyLine()) name = property(get_name, set_name) + value = property(get_value, set_value) def __str__(self): @@ -322,8 +331,8 @@ class INISection(config.ConfigNamespace): _optionxformvalue = None _optionxformsource = None _compat_skip_empty_lines = set() - def __init__(self, lineobj, defaults = None, - optionxformvalue=None, optionxformsource=None): + + def __init__(self, lineobj, defaults=None, optionxformvalue=None, optionxformsource=None): self._lines = [lineobj] self._defaults = defaults self._optionxformvalue = optionxformvalue @@ -453,6 +462,7 @@ class INIConfig(config.ConfigNamespace): _sectionxformsource = None _parse_exc = None _bom = False + def __init__(self, fp=None, defaults=None, parse_exc=True, optionxformvalue=lower, optionxformsource=None, sectionxformvalue=None, sectionxformsource=None): @@ -465,7 +475,7 @@ class INIConfig(config.ConfigNamespace): self._sections = {} if defaults is None: defaults = {} self._defaults = INISection(LineContainer(), optionxformsource=self) - for name, value in defaults.iteritems(): + for name, value in defaults.items(): self._defaults[name] = value if fp is not None: self._readfp(fp) @@ -545,34 +555,34 @@ class INIConfig(config.ConfigNamespace): fname = fp.name except AttributeError: fname = '' - linecount = 0 + line_count = 0 exc = None line = None for line in readline_iterator(fp): # Check for BOM on first line - if linecount == 0 and isinstance(line, unicode): + if line_count == 0 and isinstance(line, six.text_type): if line[0] == u'\ufeff': line = line[1:] self._bom = True - lineobj = self._parse(line) - linecount += 1 + line_obj = self._parse(line) + line_count += 1 - if not cur_section and not isinstance(lineobj, - (CommentLine, EmptyLine, SectionLine)): + if not cur_section and not isinstance(line_obj, (CommentLine, EmptyLine, SectionLine)): if self._parse_exc: - raise MissingSectionHeaderError(fname, linecount, line) + raise MissingSectionHeaderError(fname, line_count, line) else: - lineobj = make_comment(line) + line_obj = make_comment(line) - if lineobj is None: + if line_obj is None: if self._parse_exc: - if exc is None: exc = ParsingError(fname) - exc.append(linecount, line) - lineobj = make_comment(line) + if exc is None: + exc = ParsingError(fname) + exc.append(line_count, line) + line_obj = make_comment(line) - if isinstance(lineobj, ContinuationLine): + if isinstance(line_obj, ContinuationLine): if cur_option: if pending_lines: cur_option.extend(pending_lines) @@ -580,20 +590,21 @@ class INIConfig(config.ConfigNamespace): if pending_empty_lines: optobj._compat_skip_empty_lines.add(cur_option_name) pending_empty_lines = False - cur_option.add(lineobj) + cur_option.add(line_obj) else: # illegal continuation line - convert to comment if self._parse_exc: - if exc is None: exc = ParsingError(fname) - exc.append(linecount, line) - lineobj = make_comment(line) + if exc is None: + exc = ParsingError(fname) + exc.append(line_count, line) + line_obj = make_comment(line) - if isinstance(lineobj, OptionLine): + if isinstance(line_obj, OptionLine): if pending_lines: cur_section.extend(pending_lines) pending_lines = [] pending_empty_lines = False - cur_option = LineContainer(lineobj) + cur_option = LineContainer(line_obj) cur_section.add(cur_option) if self._optionxform: cur_option_name = self._optionxform(cur_option.name) @@ -605,11 +616,11 @@ class INIConfig(config.ConfigNamespace): optobj = self._sections[cur_section_name] optobj._options[cur_option_name] = cur_option - if isinstance(lineobj, SectionLine): + if isinstance(line_obj, SectionLine): self._data.extend(pending_lines) pending_lines = [] pending_empty_lines = False - cur_section = LineContainer(lineobj) + cur_section = LineContainer(line_obj) self._data.add(cur_section) cur_option = None cur_option_name = None @@ -628,16 +639,14 @@ class INIConfig(config.ConfigNamespace): else: self._sections[cur_section_name]._lines.append(cur_section) - if isinstance(lineobj, (CommentLine, EmptyLine)): - pending_lines.append(lineobj) - if isinstance(lineobj, EmptyLine): + if isinstance(line_obj, (CommentLine, EmptyLine)): + pending_lines.append(line_obj) + if isinstance(line_obj, EmptyLine): pending_empty_lines = True self._data.extend(pending_lines) - if line and line[-1]=='\n': + if line and line[-1] == '\n': self._data.add(EmptyLine()) if exc: raise exc - - diff --git a/iniparse/utils.py b/libtisbackup/iniparse/utils.py old mode 100755 new mode 100644 similarity index 88% rename from iniparse/utils.py rename to libtisbackup/iniparse/utils.py index 829fc28..ee5e40d --- a/iniparse/utils.py +++ b/libtisbackup/iniparse/utils.py @@ -1,5 +1,6 @@ -import compat -from ini import LineContainer, EmptyLine +from . import compat +from .ini import LineContainer, EmptyLine + def tidy(cfg): """Clean up blank lines. @@ -32,12 +33,12 @@ def tidy(cfg): if cont and not isinstance(cont[-1], EmptyLine): cont.append(EmptyLine()) + def tidy_section(lc): cont = lc.contents i = 1 while i < len(cont): - if (isinstance(cont[i-1], EmptyLine) and - isinstance(cont[i], EmptyLine)): + if isinstance(cont[i-1], EmptyLine) and isinstance(cont[i], EmptyLine): del cont[i] else: i += 1 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d237e6d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +huey==2.4.3 +iniparse diff --git a/rpm/build.sh b/rpm/build.sh index 2d226de..5677023 100755 --- a/rpm/build.sh +++ b/rpm/build.sh @@ -4,6 +4,9 @@ set -ex rm -rf ./builddir/ ./BUILD *.rpm ./RPMS mkdir -p BUILD RPMS + + + VERSION=`git rev-list HEAD --count` echo $VERSION > __VERSION__ diff --git a/rpm/tis-tisbackup.spec b/rpm/tis-tisbackup.spec index 620b19b..75d1bfa 100644 --- a/rpm/tis-tisbackup.spec +++ b/rpm/tis-tisbackup.spec @@ -14,28 +14,32 @@ URL: http://dev.tranquil.it Source0: ../ Prefix: / -Requires: unzip rsync python-paramiko python-pyvmomi nfs-utils python-flask python-simplejson autofs pexpect - -# Turn off the brp-python-bytecompile script -#%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g') +%if "%{rhel}" == "8" +Requires: unzip rsync python3-paramiko python3-pyvmomi nfs-utils python3-flask python3-simplejson autofs python3-pexpect +%endif +%if "%{rhel}" == "7" +Requires: unzip rsync python36-paramiko python3-pyvmomi nfs-utils python3-flask python3-simplejson autofs pexpect +%endif %description %install set -ex -mkdir -p %{buildroot}/opt/tisbackup/ +mkdir -p %{buildroot}/opt/tisbackup/lib mkdir -p %{buildroot}/usr/lib/systemd/system/ mkdir -p %{buildroot}/etc/cron.d/ mkdir -p %{buildroot}/etc/tis mkdir -p %{buildroot}/usr/bin/ -rsync --exclude "deb/" --exclude "doc/" --exclude "rpm/" --exclude ".git" -aP ../../../tisbackup/ %{buildroot}/opt/tisbackup/ -rsync -aP ../../../tisbackup/scripts/tisbackup_gui.service %{buildroot}/usr/lib/systemd/system/ -rsync -aP ../../../tisbackup/scripts/tisbackup_huey.service %{buildroot}/usr/lib/systemd/system/ -rsync -aP ../../../tisbackup/samples/tisbackup.cron %{buildroot}/etc/cron.d/tisbackup -rsync -aP ../../../tisbackup/samples/tisbackup_gui.ini %{buildroot}/etc/tis -rsync -aP ../../../tisbackup/samples/tisbackup-config.ini.sample %{buildroot}/etc/tis/tisbackup-config.ini.sample +pip3 install -r ../../requirements.txt -t %{buildroot}/opt/tisbackup/lib + +rsync --exclude "deb/" --exclude "doc/" --exclude "rpm/" --exclude ".git" -aP ../../ %{buildroot}/opt/tisbackup/ +rsync -aP ../../scripts/tisbackup_gui.service %{buildroot}/usr/lib/systemd/system/ +rsync -aP ../../scripts/tisbackup_huey.service %{buildroot}/usr/lib/systemd/system/ +rsync -aP ../../samples/tisbackup.cron %{buildroot}/etc/cron.d/tisbackup +rsync -aP ../../samples/tisbackup_gui.ini %{buildroot}/etc/tis +rsync -aP ../../samples/tisbackup-config.ini.sample %{buildroot}/etc/tis/tisbackup-config.ini.sample ln -s /opt/tisbackup/tisbackup.py %{buildroot}/usr/bin/tisbackup %files @@ -51,4 +55,8 @@ ln -s /opt/tisbackup/tisbackup.py %{buildroot}/usr/bin/tisbackup %post +python3 -m compileall /opt/tisbackup/ +find /opt/tisbackup -name "*.pyc" -exec rm -rf {} \; +%postun +rm -rf /opt/tisbackup diff --git a/samples/migrate_rsync_to_btrfs.py b/samples/migrate_rsync_to_btrfs.py deleted file mode 100644 index ee4a24c..0000000 --- a/samples/migrate_rsync_to_btrfs.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -import subprocess -import os - -backups = [ "ns3-test-etc-bind"] -backup_base_dir = "/backup/data/" -backup_retention_time=60 - - - -if not os.path.isdir("/backup/data/empty/"): - os.mkdir("/backup/data/empty/") -for backup in backups: - base_dir = os.path.join(backup_base_dir,backup) - dest_dir = os.path.join(base_dir, 'last_backup') - if not os.path.isdir(dest_dir): - cmd = "/bin/btrfs subvolume create %s"%dest_dir - print 'btrfs subvolume create "%s"' %dest_dir - print subprocess.check_output(cmd, shell=True) - - if len(os.listdir(dest_dir)) == 0: - list_backups = sorted([os.path.join(base_dir, f) for f in os.listdir(base_dir)], key=os.path.getctime) - recent_backup = list_backups[-2] - print "The most recent backup : " + recent_backup - print "Initial copy" - #cmd = 'rsync -rt --stats --delete-excluded --numeric-ids -P -lpgoD --protect-args "%s"/ "%s"' % ( recent_backup, dest_dir) - cmd = 'cp -v -a --reflink=always "%s"/* "%s"' % ( recent_backup, dest_dir) - print "Runinig %s " % cmd - print subprocess.check_output(cmd, shell=True) - if len(os.listdir(base_dir)) > backup_retention_time: - for folder in sorted([os.path.join(base_dir, f) for f in os.listdir(base_dir)], key=os.path.getctime)[0:len(os.listdir(base_dir)) - (backup_retention_time )]: - #cmd = 'rsync --dry-run -av --del /backup/data/empty/ "%s/"' % folder - cmd = 'rsync -av --del /backup/data/empty/ "%s/"' % folder - print "Runinig %s " % cmd - print subprocess.check_output(cmd, shell=True) - os.rmdir(folder) - - diff --git a/tasks.py b/tasks.py index b9010f2..c55e1bf 100644 --- a/tasks.py +++ b/tasks.py @@ -1,9 +1,9 @@ -from config import huey +from huey import RedisHuey import os import logging from tisbackup import tis_backup - +huey = RedisHuey('tisbackup', host='localhost') @huey.task() def run_export_backup(base, config_file, mount_point, backup_sections): try: diff --git a/tisbackup.py b/tisbackup.py index 8f6077d..79273bf 100755 --- a/tisbackup.py +++ b/tisbackup.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -*- coding: utf-8 -*- # ----------------------------------------------------------------------- # This file is part of TISBackup @@ -17,14 +17,17 @@ # along with TISBackup. If not, see . # # ----------------------------------------------------------------------- -import os import datetime import subprocess -from iniparse import ConfigParser -from optparse import OptionParser +import os,sys +from os.path import isfile, join +tisbackup_root_dir = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0,os.path.join(tisbackup_root_dir,'lib')) + +from iniparse import ini,ConfigParser +from optparse import OptionParser import re -import sys import getopt import os.path import logging @@ -33,20 +36,20 @@ from libtisbackup.common import * from libtisbackup.backup_mysql import backup_mysql from libtisbackup.backup_rsync import backup_rsync from libtisbackup.backup_rsync import backup_rsync_ssh -from libtisbackup.backup_oracle import backup_oracle +#from libtisbackup.backup_oracle import backup_oracle from libtisbackup.backup_rsync_btrfs import backup_rsync_btrfs from libtisbackup.backup_rsync_btrfs import backup_rsync__btrfs_ssh from libtisbackup.backup_pgsql import backup_pgsql from libtisbackup.backup_xva import backup_xva -from libtisbackup.backup_vmdk import backup_vmdk -from libtisbackup.backup_switch import backup_switch +#from libtisbackup.backup_vmdk import backup_vmdk +#from libtisbackup.backup_switch import backup_switch from libtisbackup.backup_null import backup_null from libtisbackup.backup_xcp_metadata import backup_xcp_metadata from libtisbackup.copy_vm_xcp import copy_vm_xcp -from libtisbackup.backup_sqlserver import backup_sqlserver +#from libtisbackup.backup_sqlserver import backup_sqlserver from libtisbackup.backup_samba4 import backup_samba4 -__version__="1.1" +__version__="2.0" usage="""\ %prog -c configfile action @@ -88,6 +91,7 @@ class tis_backup: self.verbose=False def read_ini_file(self,filename): + ini.change_comment_syntax() cp = ConfigParser() cp.read(filename) @@ -179,15 +183,15 @@ class tis_backup: nagiosoutput = 'ALL backups OK %s' % (','.join(sections)) - except BaseException,e: + except BaseException as e: worst_nagiosstatus = nagiosStateCritical nagiosoutput = 'EXCEPTION',"Critical : %s" % str(e) raise finally: self.logger.debug('worst nagios status :"%i"',worst_nagiosstatus) - print '%s (tisbackup V%s)' %(nagiosoutput,version) - print '\n'.join(["[%s]:%s" % (l[0],l[1]) for l in globallog]) + print('%s (tisbackup V%s)' %(nagiosoutput,version)) + print('\n'.join(["[%s]:%s" % (l[0],l[1]) for l in globallog])) sys.exit(worst_nagiosstatus) def process_backup(self,sections=[]): @@ -204,7 +208,7 @@ class tis_backup: self.logger.info('Processing [%s]',(backup_item.backup_name)) stats = backup_item.process_backup() processed.append((backup_item.backup_name,stats)) - except BaseException,e: + except BaseException as e: self.logger.critical('Backup [%s] processed with error : %s',backup_item.backup_name,e) errors.append((backup_item.backup_name,str(e))) if not processed and not errors: @@ -230,7 +234,7 @@ class tis_backup: self.logger.info('Processing [%s]',(backup_item.backup_name)) stats = backup_item.export_latestbackup(destdir=exportdir) processed.append((backup_item.backup_name,stats)) - except BaseException,e: + except BaseException as e: self.logger.critical('Export Backup [%s] processed with error : %s',backup_item.backup_name,e) errors.append((backup_item.backup_name,str(e))) if not processed and not errors: @@ -252,7 +256,8 @@ class tis_backup: from stats where status="OK" and backup_start>=?""",(mindate,)) - defined_backups = map(lambda f:f.backup_name, [ x for x in self.backup_list if not isinstance(x, backup_null) ]) + + defined_backups = list(map(lambda f:f.backup_name, [ x for x in self.backup_list if not isinstance(x, backup_null) ])) failed_backups_names = set(defined_backups) - set([b['bname'] for b in failed_backups if b['bname'] in defined_backups]) @@ -265,7 +270,7 @@ class tis_backup: self.logger.info('Processing [%s]',(backup_item.backup_name)) stats = backup_item.process_backup() processed.append((backup_item.backup_name,stats)) - except BaseException,e: + except BaseException as e: self.logger.critical('Backup [%s] not processed, error : %s',backup_item.backup_name,e) errors.append((backup_item.backup_name,str(e))) if not processed and not errors: @@ -293,7 +298,7 @@ class tis_backup: self.logger.info('Processing cleanup of [%s]',(backup_item.backup_name)) backup_item.cleanup_backup() processed = True - except BaseException,e: + except BaseException as e: self.logger.critical('Cleanup of [%s] not processed, error : %s',backup_item.backup_name,e) if not processed: self.logger.critical('No cleanup properly finished or processed') @@ -325,7 +330,7 @@ def main(): (options,args)=parser.parse_args() if len(args) != 1: - print "ERROR : You must provide one action to perform" + print("ERROR : You must provide one action to perform") parser.print_usage() sys.exit(2) @@ -335,7 +340,7 @@ def main(): action = args[0] if action == "listdrivers": for t in backup_drivers: - print backup_drivers[t].get_help() + print(backup_drivers[t].get_help()) sys.exit(0) config_file =options.config @@ -376,7 +381,7 @@ def main(): hdlr = logging.FileHandler(os.path.join(log_dir,'tisbackup_%s.log' % (backup_start_date))) hdlr.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) logger.addHandler(hdlr) - except IOError, e: + except IOError as e: if action == 'cleanup' and e.errno == errno.ENOSPC: logger.warning("No space left on device, disabling file logging.") else: diff --git a/tisbackup_gui.py b/tisbackup_gui.py index c2f7770..2163f7a 100755 --- a/tisbackup_gui.py +++ b/tisbackup_gui.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -*- coding: utf-8 -*- # ----------------------------------------------------------------------- # This file is part of TISBackup @@ -28,12 +28,12 @@ from iniparse import ConfigParser,RawConfigParser from libtisbackup.common import * import time from flask import request, Flask, session, g, appcontext_pushed, redirect, url_for, abort, render_template, flash, jsonify, Response -from urlparse import urlparse +from urllib.parse import urlparse import json import glob import time -from config import huey +from huey import * from tasks import run_export_backup, get_task, set_task from tisbackup import tis_backup @@ -87,11 +87,11 @@ def read_all_configs(base_dir): backup_dict['null_list'] = [] backup_dict['pgsql_list'] = [] backup_dict['mysql_list'] = [] - backup_dict['sqlserver_list'] = [] + #backup_dict['sqlserver_list'] = [] backup_dict['xva_list'] = [] backup_dict['metadata_list'] = [] - backup_dict['switch_list'] = [] - backup_dict['oracle_list'] = [] + #backup_dict['switch_list'] = [] + #backup_dict['oracle_list'] = [] result = [] cp = ConfigParser() @@ -153,20 +153,20 @@ def read_all_configs(base_dir): db_name = row['db_name'] if len(row['db_name']) > 0 else '*' backup_dict['mysql_list'].append( [server_name, backup_name, backup_type, db_name]) - if backup_type == "sqlserver+ssh": - db_name = row['db_name'] - backup_dict['sqlserver_list'].append( - [server_name, backup_name, backup_type, db_name]) - if backup_type == "oracle+ssh": - db_name = row['db_name'] - backup_dict['oracle_list'].append( - [server_name, backup_name, backup_type, db_name]) + # if backup_type == "sqlserver+ssh": + # db_name = row['db_name'] + # backup_dict['sqlserver_list'].append( + # [server_name, backup_name, backup_type, db_name]) + # if backup_type == "oracle+ssh": + # db_name = row['db_name'] + # backup_dict['oracle_list'].append( + # [server_name, backup_name, backup_type, db_name]) if backup_type == "xen-xva": backup_dict['xva_list'].append( [server_name, backup_name, backup_type, ""]) - if backup_type == "switch": - backup_dict['switch_list'].append( - [server_name, backup_name, backup_type, ""]) + # if backup_type == "switch": + # backup_dict['switch_list'].append( + # [server_name, backup_name, backup_type, ""]) return backup_dict @@ -209,11 +209,11 @@ def read_config(): backup_dict['null_list'] = [] backup_dict['pgsql_list'] = [] backup_dict['mysql_list'] = [] - backup_dict['sqlserver_list'] = [] + #backup_dict['sqlserver_list'] = [] backup_dict['xva_list'] = [] backup_dict['metadata_list'] = [] - backup_dict['switch_list'] = [] - backup_dict['oracle_list'] = [] + #backup_dict['switch_list'] = [] + #backup_dict['oracle_list'] = [] for row in result: backup_name = row['backup_name'] server_name = row['server_name'] @@ -237,16 +237,16 @@ def read_config(): if backup_type == "mysql+ssh": db_name = row['db_name'] if len(row['db_name']) > 0 else '*' backup_dict['mysql_list'].append([server_name, backup_name, backup_type, db_name]) - if backup_type == "sqlserver+ssh": - db_name = row['db_name'] - backup_dict['sqlserver_list'].append([server_name, backup_name, backup_type, db_name]) - if backup_type == "oracle+ssh": - db_name = row['db_name'] - backup_dict['oracle_list'].append([server_name, backup_name, backup_type, db_name]) + # if backup_type == "sqlserver+ssh": + # db_name = row['db_name'] + # backup_dict['sqlserver_list'].append([server_name, backup_name, backup_type, db_name]) + # if backup_type == "oracle+ssh": + # db_name = row['db_name'] + # backup_dict['oracle_list'].append([server_name, backup_name, backup_type, db_name]) if backup_type == "xen-xva": backup_dict['xva_list'].append([server_name, backup_name, backup_type, ""]) - if backup_type == "switch": - backup_dict['switch_list'].append([server_name, backup_name, backup_type, ""]) + # if backup_type == "switch": + # backup_dict['switch_list'].append([server_name, backup_name, backup_type, ""]) return backup_dict @app.route('/') @@ -262,19 +262,21 @@ def set_config_number(id=None): global config_number config_number=id read_config() - return jsonify(configs=CONFIG,config_number=config_number) + return jsonify(configs=CONFIG,config_number=config_number) @app.route('/all_json') def backup_all_json(): backup_dict = read_all_configs(BASE_DIR) - return json.dumps(backup_dict['rsync_list']+backup_dict['sqlserver_list']+backup_dict['rsync_btrfs_list']+backup_dict['rsync_ssh_list']+backup_dict['pgsql_list']+backup_dict['mysql_list']+backup_dict['xva_list']+backup_dict['null_list']+backup_dict['metadata_list']+ backup_dict['switch_list']) + return json.dumps(backup_dict['rsync_list']+backup_dict['rsync_btrfs_list']+backup_dict['rsync_ssh_list']+backup_dict['pgsql_list']+backup_dict['mysql_list']+backup_dict['xva_list']+backup_dict['null_list']+backup_dict['metadata_list']) + #+ backup_dict['switch_list'])+backup_dict['sqlserver_list'] @app.route('/json') def backup_json(): backup_dict = read_config() - return json.dumps(backup_dict['rsync_list']+backup_dict['sqlserver_list']+backup_dict['rsync_btrfs_list']+backup_dict['rsync_ssh_list']+backup_dict['pgsql_list']+backup_dict['mysql_list']+backup_dict['xva_list']+backup_dict['null_list']+backup_dict['metadata_list']+ backup_dict['switch_list']) + return json.dumps(backup_dict['rsync_list']+backup_dict['rsync_btrfs_list']+backup_dict['rsync_ssh_list']+backup_dict['pgsql_list']+backup_dict['mysql_list']+backup_dict['xva_list']+backup_dict['null_list']+backup_dict['metadata_list']) + #+ backup_dict['switch_list'])+backup_dict['sqlserver_list'] def check_usb_disk(): @@ -289,23 +291,23 @@ def check_usb_disk(): if len(usb_disk_list) == 0: raise_error("Cannot find any external usb disk", "You should plug the usb hard drive into the server") return "" - print usb_disk_list + print(usb_disk_list) usb_partition_list = [] for usb_disk in usb_disk_list: cmd = "udevadm info -q path -n %s" % usb_disk + '1' output = os.popen(cmd).read() - print "cmd : " + cmd - print "output : " + output + print("cmd : " + cmd) + print("output : " + output) if '/devices/pci' in output: #flash("partition found: %s1" % usb_disk) usb_partition_list.append(usb_disk + "1") - print usb_partition_list + print(usb_partition_list) if len(usb_partition_list) ==0: - raise_error("The dribe %s has no partition" % (usb_disk_list[0] ), "You should initialize the usb drive and format an ext4 partition with TISBACKUP label") + raise_error("The drive %s has no partition" % (usb_disk_list[0] ), "You should initialize the usb drive and format an ext4 partition with TISBACKUP label") return "" tisbackup_partition_list = [] @@ -314,7 +316,7 @@ def check_usb_disk(): flash("tisbackup backup partition found: %s" % usb_partition) tisbackup_partition_list.append(usb_partition) - print tisbackup_partition_list + print(tisbackup_partition_list) if len(tisbackup_partition_list) ==0: raise_error("No tisbackup partition exist on disk %s" % (usb_disk_list[0] ), "You should initialize the usb drive and format an ext4 partition with TISBACKUP label") @@ -334,9 +336,9 @@ def check_already_mount(partition_name,refresh): for line in f.readlines(): if line.startswith(partition_name): mount_point = line.split(' ')[1] - if not refresh: - run_command("/bin/umount %s" % mount_point) - os.rmdir(mount_point) + if not refresh: + run_command("/bin/umount %s" % mount_point) + os.rmdir(mount_point) return mount_point def run_command(cmd, info=""): @@ -411,13 +413,14 @@ def export_backup(): if backup_types == "null_list": continue for section in backup_dict[backup_types]: - if section.count > 0: + #if section.count > 0: + if len(section) > 0: sections.append(section[1]) noJobs = (not runnings_backups()) - if "start" in request.args.keys() or not noJobs: + if "start" in list(request.args.keys()) or not noJobs: start=True - if "sections" in request.args.keys(): + if "sections" in list(request.args.keys()): backup_sections = request.args.getlist('sections') @@ -435,9 +438,9 @@ def export_backup(): global mindate mindate = datetime2isodate(datetime.datetime.now()) if not error and start: - print tisbackup_config_file - task = run_export_backup(base=backup_base_dir, config_file=CONFIG[config_number], mount_point=mount_point, backup_sections=",".join([str(x) for x in backup_sections])) - set_task(task) + print(tisbackup_config_file) + task = run_export_backup(base=backup_base_dir, config_file=CONFIG[config_number], mount_point=mount_point, backup_sections=",".join([str(x) for x in backup_sections])) + set_task(task) return render_template("export_backup.html", error=error, start=start, info=info, email=ADMIN_EMAIL, sections=sections)