tools.win_pg_dump_controller

Yohn Y. 2022-02-11 Parent:be791d354d2a Child:a38a008ce3e8

1:a22dd63ba19e Go to Latest

tools.win_pg_dump_controller/win_pg_dump_controller/store_controller.py

* Исправление ошибки в логике работы с резервными копиями второго класса . Добавлено больше информации о состоянии резервных копий второго класса . Не пишем в логи лишние строки во время операций чистки старых логов

Download raw file
View source Diff to previous Annotate
History
awgur@0 1 # coding: utf-8
awgur@0 2
awgur@0 3 import json
awgur@0 4 from datetime import datetime
awgur@0 5 from os.path import exists, join as p_join, basename
awgur@0 6 from os import remove as file_remove
awgur@0 7 from typing import Dict, List
awgur@0 8
awgur@0 9 from .error import Error
awgur@0 10 from .config import DBTask
awgur@0 11
awgur@0 12
awgur@0 13 class StoreError(Error):
awgur@0 14 pass
awgur@0 15
awgur@0 16
awgur@0 17 class StoreControllerBase(object):
awgur@0 18 def get_filename(self, filename: str) -> str:
awgur@0 19 raise NotImplemented()
awgur@0 20
awgur@0 21
awgur@0 22 class IndexItem(object):
awgur@0 23 def __init__(self, controller: StoreControllerBase, time: int, filename: str):
awgur@0 24 self._controller = controller
awgur@0 25 self.filename = filename
awgur@0 26 self.time = time
awgur@0 27 self._my_path = None
awgur@0 28
awgur@0 29 def get_path(self):
awgur@0 30 if self._my_path is None:
awgur@0 31 self._my_path = self._controller.get_filename(self.filename)
awgur@0 32
awgur@0 33 return self._my_path
awgur@0 34
awgur@1 35 def __str__(self) -> str:
awgur@1 36 return self.filename
awgur@1 37
awgur@0 38 def get_datetime(self):
awgur@0 39 return datetime.fromtimestamp(self.time)
awgur@0 40
awgur@0 41 def is_exists(self) -> bool:
awgur@0 42 return exists(self.get_path())
awgur@0 43
awgur@0 44 def to_dict(self) -> dict:
awgur@0 45 return dict((i, getattr(self, i)) for i in ('filename', 'time'))
awgur@0 46
awgur@0 47 @classmethod
awgur@0 48 def from_dict(cls, controller: StoreControllerBase, d: Dict):
awgur@0 49 return cls(
awgur@0 50 controller=controller,
awgur@0 51 **d
awgur@0 52 )
awgur@0 53
awgur@0 54 @classmethod
awgur@0 55 def new(cls, controller: StoreControllerBase, short_filename: str):
awgur@0 56 time = datetime.now()
awgur@0 57 time_prefix = time.strftime('%Y-%m-%d_%H-%M-%S')
awgur@0 58 filename = f'{time_prefix} - {short_filename}'
awgur@0 59
awgur@0 60 return cls(
awgur@0 61 controller=controller,
awgur@0 62 time=int(time.timestamp()),
awgur@0 63 filename=filename
awgur@0 64 )
awgur@0 65
awgur@0 66
awgur@0 67 class StoreController(StoreControllerBase):
awgur@0 68 def __init__(self, task: DBTask):
awgur@0 69 self.task = task
awgur@0 70 self.index_name = self.get_filename(f'{task.name}.index')
awgur@0 71 self.idx: Dict[int, IndexItem] = {}
awgur@1 72 self.op_adv_status = []
awgur@0 73
awgur@0 74 if exists(self.index_name):
awgur@0 75 self.load_index()
awgur@0 76
awgur@0 77 def get_filename(self, filename: str) -> str:
awgur@0 78 return str(p_join(self.task.dst_dir, filename))
awgur@0 79
awgur@0 80 def load_index(self) -> None:
awgur@0 81 with open(self.index_name, 'r') as IN:
awgur@0 82 result = {}
awgur@0 83 for itm in json.load(IN):
awgur@0 84 itm = IndexItem.from_dict(self, itm)
awgur@0 85 if itm.is_exists():
awgur@0 86 result[itm.time] = itm
awgur@0 87
awgur@0 88 self.idx = result
awgur@0 89
awgur@0 90 def save_index(self) -> None:
awgur@0 91 with open(self.index_name, 'w') as OUT:
awgur@0 92 json.dump(list(map(lambda x: x.to_dict(), self.idx.values())), OUT)
awgur@0 93
awgur@1 94 def add_op_status(self, msg: str) -> None:
awgur@1 95 self.op_adv_status.append(msg)
awgur@1 96
awgur@0 97 def new_item(self) -> IndexItem:
awgur@0 98 return IndexItem.new(self, f'{self.task.name}.backup')
awgur@0 99
awgur@0 100 def add_item(self, item: IndexItem) -> None:
awgur@0 101 if not item.is_exists():
awgur@0 102 raise StoreError(f'Storing to index file not found: {item.get_path()}')
awgur@0 103
awgur@0 104 if item.time in self.idx and self.idx[item.time].filename != item.filename:
awgur@0 105 if self.idx[item.time].is_exists():
awgur@0 106 file_remove(self.idx[item.time].get_path())
awgur@0 107
awgur@0 108 self.idx[item.time] = item
awgur@0 109
awgur@0 110 def remove(self, item: IndexItem) -> None:
awgur@0 111 if item.time in self.idx:
awgur@0 112 del self.idx[item.time]
awgur@0 113
awgur@0 114 if item.is_exists():
awgur@0 115 file_remove(item.get_path())
awgur@0 116
awgur@0 117 def __iter__(self):
awgur@1 118 for i in sorted(self.idx, reverse=True):
awgur@0 119 yield self.idx[i]
awgur@0 120
awgur@0 121 def clean(self, tier1_days: int, tier2_copies_interval: int, tier2_store_days: int) -> List[str]:
awgur@0 122 to_remove = []
awgur@0 123 tier2_idx = {}
awgur@0 124 now = datetime.now()
awgur@0 125
awgur@0 126 for item in self:
awgur@0 127 if not item.is_exists():
awgur@0 128 to_remove.append(item)
awgur@0 129
awgur@0 130 else:
awgur@0 131 storing_days = (now - item.get_datetime()).days
awgur@0 132
awgur@0 133 if not storing_days <= tier1_days:
awgur@0 134 if storing_days > tier2_store_days:
awgur@0 135 to_remove.append(item)
awgur@0 136
awgur@0 137 else:
awgur@0 138 # Магия: Делим старые резервные копии на эпохи. Эпоха - целая часть от деления количества
awgur@0 139 # дней, которое лежит резервная копия, на интервал, за который нам нужно хранить
awgur@0 140 # хотя бы одну копию. В одной эпохе старший файл вытесняет младший. Из вычислений
awgur@0 141 # убираем период tier1
awgur@0 142
awgur@1 143 storing_days_clean = storing_days - tier1_days
awgur@0 144
awgur@1 145 _epoch = divmod(storing_days_clean, tier2_copies_interval)[0]
awgur@1 146 self.add_op_status(f'"{item}": epoch={_epoch} storing_days={storing_days}'
awgur@1 147 f' clean_storing_days={storing_days_clean}')
awgur@1 148
awgur@0 149 if _epoch in tier2_idx:
awgur@0 150 to_remove.append(tier2_idx[_epoch])
awgur@0 151
awgur@0 152 tier2_idx[_epoch] = item
awgur@0 153
awgur@0 154 result = []
awgur@0 155 for item in to_remove:
awgur@0 156 file_remove(item.get_path())
awgur@0 157 result.append(item.filename)
awgur@0 158
awgur@0 159 return result