Last fixes for version 1.0

tapebackup/routes.py

@@ -19,7 +19,7 @@ def includeme(config):
     config.add_route('import_tape', '/import_tape')
     config.add_route('list_backup_targets', '/list_backup_targets')
     config.add_route('list_restore_targets', '/list_restore_targets')
-    config.add_route('test_behavior', '/test_behavior')
+    config.add_route('cleanup_missing', '/cleanup_missing')
     config.add_route('copy_status', '/copy_status')
     config.add_route('batches', '/batches')
     config.add_route('batch_info', '/batch/{batch_id}')

tapebackup/scripts/restore_daemon.py

 import argparse
 import asyncio
+import datetime
 import logging
 import psutil
 import redis
@@ -17,6 +18,7 @@ log = logging.getLogger("tapebackup_eventloop")
 
 red = redis.Redis()
 
+
 def restore_copy_process(mountpoint, f, dbsession, loop):
     src = mountpoint + "/" + f.file_to_restore.relative_path
     if f.file_to_restore.kind == models.DataKind.wgs or f.file_to_restore.kind == models.DataKind.vcf:
@@ -28,6 +30,7 @@ def restore_copy_process(mountpoint, f, dbsession, loop):
     if checksum == f.file_to_restore.checksum and copy_result == 0:
         log.info("File %s restored; checksum matched with database", f.file_to_restore.relative_path)
         f.status = True
+        f.restored_at_time = datetime.datetime.now()
     elif copy_result != 0:
         log.warning("File % restore failed", f.file_to_restore.relative_path)
     else:
@@ -35,6 +38,7 @@ def restore_copy_process(mountpoint, f, dbsession, loop):
 
     return
 
+
 def restore_copy_queue(dbsession, loop):
     dbsession.begin_nested()
     manager = utils.tapemanager.TapeManager(dbsession)
@@ -42,6 +46,7 @@ def restore_copy_queue(dbsession, loop):
         .filter(models.RestoreHistory.status == False).order_by(models.FileToBackup.tape_label)
     prev_tape_label = ""
     for f in current_batch:
+        dbsession.begin_nested()
         target = f.restore_target
         cur_tape_label = f.file_to_restore.tape_label
         if cur_tape_label != prev_tape_label:

tapebackup/templates/queue_and_log.jinja2

 {% extends "base.jinja2" %}
 
 {% block content %}
-
-<p>Формируется очередей: {{ new_len }}</p>
-<p>Очередей в процессе бэкапа: {{ in_progress_len }}</p>
-<p>Файлов в текущей очереди бэкапа: {{ in_progress_remaining_files }}</p>
-<p>Завершенных очередей бэкапа: {{ finished_len }}</p>
-{#    нужно вставить кнопку для очистки стертых файлов#}
-{#    нужно вставить отфильтрованный лог и поменять уровни логирования#}
+<div class="row">
+    <div class="cell" style="flex-basis: 30%">
+        <p>Формируется очередей: {{ new_len }}</p>
+        <p>Очередей в процессе бэкапа: {{ in_progress_len }}</p>
+        <p>Файлов в текущей очереди бэкапа: {{ in_progress_remaining_files }}</p>
+        <p>Завершенных очередей бэкапа: {{ finished_len }}</p>
+        <form action="/cleanup_missing">
+            <button type="submit">Удалить из очереди бэкапа файлы,<br> отсутствующие в файловой системе</button>
+        </form>
+      </div>
+    <div class="cell" style="flex-basis: 70%">
+        <div class="pagination">
+            {% if page != 0 %}
+                <a href="{{ request.current_route_path(_query=dict(request.params.dicts[0], page = 0)) }}">к началу</a>
+                <a>...</a>
+            {% endif %}
+            {% if page + pagination_pages_to_show < pages_total %}
+                <a class="active" href="{{ request.current_route_path(_query=dict(request.params.dicts[0])) }}">{{ page + 1 }}</a>
+                {% for i in range(page + 2, page + pagination_pages_to_show + 1) %}
+                    <a href="{{ request.current_route_path(_query=dict(request.params.dicts[0], page = i-1)) }}">{{ i }}</a>
+                {% endfor %}
+                <a>...</a>
+                <a href="{{ request.current_route_path(_query=dict(request.params.dicts[0], page = pages_total-1)) }}">{{ pages_total }}</a>
+            {% else %}
+                <a class="active" href="{{ request.current_route_path(_query=dict(request.params.dicts[0])) }}">{{ page + 1 }}</a>
+                {% for i in range(page + 2, pages_total + 1) %}
+                    <a href="{{ request.current_route_path(_query=dict(request.params.dicts[0], page = i-1)) }}">{{ i }}</a>
+                {% endfor %}
+            {% endif %}
+        </div>
+        <table class="files">
+            <thead>
+                <th>Файл для восстановления</th>
+                <th>Цель восстановления</th>
+                <th>Восстановлен?</th>
+                <th>Время восстановления</th>
+            </thead>
+            <tbody>
+                {% for f in files_to_restore %}
+                <tr>
+                    <td>{{ f.file_to_restore.relative_path }}</td>
+                    <td>{{ f.restore_target.unique_label }}</td>
+                    <td>{{ f.status }}</td>
+                    <td>{{ f.restored_at_time }}</td>
+                </tr>
+                {% endfor %}
+            </tbody>
+        </table>
+    </div>
+</div>
+{% for line in log_tail %}
+    <p style="font-family: monospace">{{ line }}</p>
+{% endfor %}
 {% endblock %}
\ No newline at end of file

tapebackup/templates/stats.jinja2

@@ -9,7 +9,7 @@
                 <li>файлов - {{ copy_status['total_files'] }}</li>
                 <li>суммарный объем - {{ copy_status['total_gb'] }} Гбайт</li>
                 <li>из них wgs - {{ copy_status['total_wgs_gb'] }} Гбайт</li>
-                <li>из них известны лаборатные номера - {{ copy_status['total_labnums'] }}</li>
+                <li>из них известны лабораторные номера - {{ copy_status['total_labnums'] }}</li>
             </ul>
         <p class="paragraph">Данных скопировано на кассеты:</p>
             <ul>

tapebackup/views/default.py

@@ -291,11 +291,11 @@ def add_to_restore_queue(request):
             raise HTTPException("No restore target selected")
         restore_target_object = request.dbsession.query(models.RestoreTarget).filter(models.RestoreTarget.unique_label
                                                                                      == restore_target).one()
-        files = file_filter(request)
+        files, tapes_status = file_filter(request)
         backup_target = request.GET.get('backup_target')
         if backup_target:
             files = files.filter(models.FileToBackup.target_unique_label == backup_target)
-        files, tapes_status = files.all()
+        files = files.all()
 
         for f in files:
             restore_queue_object = models.RestoreHistory(
@@ -314,10 +314,45 @@ def add_to_restore_queue(request):
                                              ('color', 'red')))
         return HTTPFound(location=next_url)
 
+
+def tail(f, lines=20):
+    total_lines_wanted = lines
+
+    BLOCK_SIZE = 1024
+    f.seek(0, 2)
+    block_end_byte = f.tell()
+    lines_to_go = total_lines_wanted
+    block_number = -1
+    blocks = []
+    while lines_to_go > 0 and block_end_byte > 0:
+        if (block_end_byte - BLOCK_SIZE > 0):
+            f.seek(block_number*BLOCK_SIZE, 2)
+            blocks.append(f.read(BLOCK_SIZE))
+        else:
+            f.seek(0,0)
+            blocks.append(f.read(block_end_byte))
+        lines_found = blocks[-1].count(b'\n')
+        lines_to_go -= lines_found
+        block_end_byte -= BLOCK_SIZE
+        block_number -= 1
+    all_read_text = b''.join(reversed(blocks))
+    return b'\n'.join(all_read_text.splitlines()[-total_lines_wanted:])
+
+
 @view_config(route_name="queue_and_log", renderer='tapebackup:templates/queue_and_log.jinja2')
 def queue_and_log(request):
     try:
         backup_queues = request.dbsession.query(models.Batch).all()
+
+        limit = request.GET.get('limit') or 100
+        limit = int(limit)
+        page = request.GET.get('page') or 0
+        page = int(page)
+        pagination_pages_to_show = 10
+
+        files_to_restore = request.dbsession.query(models.RestoreHistory).all()
+        pages_total = ceil(len(files_to_restore) / limit)
+
         new = []
         in_progress = []
         paused = []  # need this state for clearer model state and debug
@@ -325,6 +360,10 @@ def queue_and_log(request):
         cancelled = []
         finalized = []
 
+        with open('/var/log/tapebackup.log', 'rb') as f:
+            log_tail = tail(f, 50).decode("utf-8").split("\n")
+        log_tail.reverse()
+
         for i in backup_queues:
             if i.status == models.BatchStatus.new:
                 new.append(i)
@@ -347,11 +386,17 @@ def queue_and_log(request):
                 "finished_len": len(finished),
                 "cancelled_len": len(cancelled),
                 "finalized_len": len(finalized),
-                "in_progress_remaining_files": in_progress_remaining_files
+                "in_progress_remaining_files": in_progress_remaining_files,
+                "log_tail": log_tail,
+                "files_to_restore": files_to_restore,
+                "limit": limit,
+                "page": page,
+                "pages_total": pages_total,
+                "pagination_pages_to_show": pagination_pages_to_show
                 }
 
     except HTTPException as e:
-        next_url = request.route_url('files',
+        next_url = request.route_url('queue_and_log',
                                      _query=(('result', e.detail),
                                              ('color', 'red')))
-    return HTTPFound(location=next_url)
\ No newline at end of file
+        return next_url

tapebackup/views/locate_files.py

+import logging
+import os.path
 
-def locate_file_list_on_tapes():
-    return
\ No newline at end of file
+from pyramid.view import view_config
+from pyramid.response import Response
+from pyramid.httpexceptions import HTTPException, HTTPConflict, HTTPFound
+from sqlalchemy.exc import SQLAlchemyError
+
+from .. import models
+
+log = logging.getLogger(__name__)
+
+
+@view_config(route_name='cleanup_missing', renderer='json')
+def cleanup_missing(request):
+    try:
+        files = request.dbsession.query(models.FileToBackup, models.BackupTarget).\
+            filter(models.FileToBackup.tape_label == None).filter(models.FileToBackup.is_file == True).\
+            filter(models.FileToBackup.target_unique_label == models.BackupTarget.unique_label)
+        missing_list = []
+        present_count = 0
+        for i in files:
+            if os.path.exists(i[1].fullpath + '/' + i[0].relative_path):
+                present_count += 1
+                continue
+            else:
+                missing_list.append(i[0].id)
+        log.info('Present: %d, Missing: %d' % (present_count, len(missing_list)) )
+        if len(missing_list):
+            log.warning("There are missing files on local FS; cleaning up only first 30000")
+        shrinked_list = missing_list[:30000]
+
+        missing_obj = request.dbsession.query(models.FileToBackup).filter(models.FileToBackup.id.in_(shrinked_list))
+        for i in missing_obj:
+            request.dbsession.delete(i)
+
+    except SQLAlchemyError as e:
+        return Response(json_body={"error": e._message()}, content_type='application/json', status=500)
+    except HTTPException as e:
+        return Response(json_body={"error": e.detail}, content_type='application/json', status=e.status)
+    next_url = request.route_url('queue_and_log')
+    return HTTPFound(location=next_url)

tapebackup/views/test_behaviour.py

@@ -15,43 +15,6 @@ from ..models.meta import DataKind
 log = logging.getLogger(__name__)
 
 
-@view_config(route_name='test_behavior', renderer='json')
-def test_behavior(request):
-    try:
-        # manager = utils.tapemanager.TapeManager(request.dbsession)
-        # library = manager.scan()
-        # tape = manager.find_tape_to_use()
-        # if not tape:
-        #     raise HTTPConflict("No usable tape found!")
-        # log.info("Going to move tape from %d to drive" % tape.last_seen_slot)
-        # manager.insert_into_drive(tape.last_seen_slot)
-        # log.info("Moved tape from slot %d to drive" % tape.last_seen_slot)
-        import os.path
-        files = request.dbsession.query(models.FileToBackup, models.BackupTarget).\
-            filter(models.FileToBackup.tape_label == None).filter(models.FileToBackup.is_file == True).\
-            filter(models.FileToBackup.target_unique_label == models.BackupTarget.unique_label)
-        missing_list = []
-        present_count = 0
-        for i in files:
-            if os.path.exists(i[1].fullpath + '/' + i[0].relative_path):
-                present_count += 1
-                continue
-            else:
-                missing_list.append(i[0].id)
-        print('Present: ', present_count, 'Missing: ', len(missing_list))
-        shrinked_list = missing_list[:30000]
-
-        missing_obj = request.dbsession.query(models.FileToBackup).filter(models.FileToBackup.id.in_(shrinked_list))
-        for i in missing_obj:
-            request.dbsession.delete(i)
-
-    except SQLAlchemyError as e:
-        return Response(json_body={"error": e._message()}, content_type='application/json', status=500)
-    except HTTPException as e:
-        return Response(json_body={"error": e.detail}, content_type='application/json', status=e.status)
-    return {}
-
-
 def tape_changer_test(request):
     try:
         manager = utils.tapemanager.TapeManager(request.dbsession)