Commit 56abb0d5 authored by Oleg Borisenko's avatar Oleg Borisenko
Browse files

restore daemon

parent f6571283
import argparse
import asyncio
import logging
import psutil
import redis
import signal
import sys
import time
from tapebackup import models
from tapebackup import utils
from pyramid.paster import bootstrap, setup_logging
from sqlalchemy.exc import OperationalError, SQLAlchemyError
log = logging.getLogger("tapebackup_eventloop")
red = redis.Redis()
def restore_copy_process(mountpoint, f, dbsession, loop):
src = mountpoint + "/" + f.file_to_restore.relative_path
if f.file_to_restore.kind == models.DataKind.wgs or f.file_to_restore.kind == models.DataKind.vcf:
dest = f.restore_target.fullpath + "/" + f.file_to_restore.relative_path
else:
dest = f.restore_target.fullpath + "/" + f.restore_target.unique_label + "/" + f.file_to_restore.relative_path
copy_result, checksum = utils.secure_copy2(src, dest, loop)
if checksum == f.file_to_restore.checksum and copy_result == 0:
log.info("File %s restored; checksum matched with database", f.file_to_restore.relative_path)
f.status = True
elif copy_result != 0:
log.warning("File % restore failed", f.file_to_restore.relative_path)
else:
log.warning("File % restored; checksum failed", f.file_to_restore.relative_path)
return
def restore_copy_queue(dbsession, loop):
dbsession.begin_nested()
manager = utils.tapemanager.TapeManager(dbsession)
current_batch = dbsession.query(models.RestoreHistory).join(models.RestoreHistory.file_to_restore)\
.filter(models.RestoreHistory.status == False).order_by(models.FileToBackup.tape_label)
prev_tape_label = ""
for f in current_batch:
target = f.restore_target
cur_tape_label = f.file_to_restore.tape_label
if cur_tape_label != prev_tape_label:
tape = manager.find_tape_by_label(cur_tape_label)
# here we are changing the tape
if manager.get_drive_state() is utils.tapemanager.DriveState.empty:
manager.insert_into_drive(tape.last_seen_slot)
manager.use_tape(restoremode=True)
elif manager.identify_tape() != tape:
manager.eject_current_tape_to_empty_magazine_slot()
manager.insert_into_drive(tape.last_seen_slot)
manager.use_tape(restoremode=True)
else:
manager.use_tape(restoremode=True)
restore_copy_process(manager.mountpoint, f, dbsession, loop)
prev_tape_label = cur_tape_label
if manager.get_drive_state() == utils.tapemanager.DriveState.data_tape_inside:
manager.eject_current_tape_to_empty_magazine_slot()
return
def parse_args(argv):
parser = argparse.ArgumentParser()
......@@ -54,7 +110,9 @@ def main(argv=sys.argv):
while True:
with env['request'].tm:
# returns restore queue by tape and serves the right tape
restore_copy_queue()
dbsession = env['request'].dbsession
restore_copy_queue(dbsession, loop)
time.sleep(5)
except SQLAlchemyError as e:
log.error(str(e))
......
......@@ -76,10 +76,10 @@
{% endfor %}
</select>
{% if request.GET.get('labnum') %}
<input type="hidden" name="labnum" value="request.GET.get('labnum')"/>
<input type="hidden" name="labnum" value="{{ request.GET.get('labnum') }}"/>
{% endif %}
{% if request.GET.get('backup_target') %}
<input type="hidden" name="labnum" value="request.GET.get('backup_target')"/>
<input type="hidden" name="backup_target" value="{{ request.GET.get('backup_target') }}"/>
{% endif %}
<button type="submit" class="button">Восстановить попадающее под фильтр на выбранную цель восстановления</button>
</div>
......
......@@ -29,7 +29,7 @@ async def cmd(command):
return exit_code, stdout
def secure_copy2(src, dst, loop):
subprocess.run(["mkdir", "-p", "-v", os.path.dirname(dst)], capture_output=True, shell=False)
mkdir = subprocess.run(["mkdir", "-p", "-v", os.path.dirname(dst)], capture_output=True, shell=False)
md5sum_proc = loop.create_task(cmd("md5sum %s" % src))
rsync_proc = loop.create_task(cmd("rsync --inplace --append-verify --partial -a %s %s" % (src, dst)))
results = loop.run_until_complete(asyncio.gather(md5sum_proc, rsync_proc))
......
......@@ -305,6 +305,17 @@ class TapeManager:
return tapes[0]
return None
def find_tape_by_label(self, tape_label):
self.scan()
# NOTE: there is a trick: drives have lowest slot numbers in MSL3040 so we actually start from active drive
tapes = self.dbsession.query(models.Tape).filter(models.Tape.label == tape_label).all()
if tapes:
# check that it's really in slot
for i in self.drives + self.magazine + self.mailslot:
if tapes[0].label == i[self.PRIMARY_VOLUME_TAG]:
return tapes[0]
return None
def insert_into_drive(self, slot):
self.dbsession.begin_nested()
self.scan()
......@@ -386,10 +397,15 @@ class TapeManager:
else:
raise Exception("Unexpected tape drive state")
def use_tape(self):
def use_tape(self, restoremode=False):
tape = self.identify_tape()
if tape.state == models.TapeState.finalized:
if tape.state == models.TapeState.finalized and not restoremode:
raise Exception("The tape %s is finalized; you can not use it for backup", tape.label)
elif tape.state == models.TapeState.finalized and restoremode:
if self.mount():
return tape
else:
raise Exception("Can't mount tape for restore")
elif tape.state == models.TapeState.inuse:
# checking consistency of TapeState in database: it should be mounted or mountable
if self.mount():
......@@ -397,7 +413,8 @@ class TapeManager:
else:
raise Exception("The tape is marked as in-use but it is not mounted and failed to mount")
elif tape.state == models.TapeState.unknown or tape.state == models.TapeState.new:
# try to mount; if failed - then format
if restoremode:
raise Exception("We can't get into this state since we are restoring")
if self.mount():
# this means that db info about tape is outdated; we mount it and mark it ready to use
tape.state = models.TapeState.inuse
......@@ -406,6 +423,8 @@ class TapeManager:
tape.state = models.TapeState.formatted
if tape.state == models.TapeState.formatted:
if restoremode:
raise Exception("We can't get into this state since we are restoring")
self.mount()
tape.state = models.TapeState.inuse
return tape
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment