tools.win_pg_dump_controller

     1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/win_pg_dump_controller/store_controller.py	Sun Jan 30 22:17:39 2022 +0300
     1.3 @@ -0,0 +1,150 @@
     1.4 +# coding: utf-8
     1.5 +
     1.6 +import json
     1.7 +from datetime import datetime
     1.8 +from os.path import exists, join as p_join, basename
     1.9 +from os import remove as file_remove
    1.10 +from typing import Dict, List
    1.11 +
    1.12 +from .error import Error
    1.13 +from .config import DBTask
    1.14 +
    1.15 +
    1.16 +class StoreError(Error):
    1.17 +    pass
    1.18 +
    1.19 +
    1.20 +class StoreControllerBase(object):
    1.21 +    def get_filename(self, filename: str) -> str:
    1.22 +        raise NotImplemented()
    1.23 +
    1.24 +
    1.25 +class IndexItem(object):
    1.26 +    def __init__(self, controller: StoreControllerBase, time: int, filename: str):
    1.27 +        self._controller = controller
    1.28 +        self.filename = filename
    1.29 +        self.time = time
    1.30 +        self._my_path = None
    1.31 +
    1.32 +    def get_path(self):
    1.33 +        if self._my_path is None:
    1.34 +            self._my_path = self._controller.get_filename(self.filename)
    1.35 +
    1.36 +        return self._my_path
    1.37 +
    1.38 +    def get_datetime(self):
    1.39 +        return datetime.fromtimestamp(self.time)
    1.40 +
    1.41 +    def is_exists(self) -> bool:
    1.42 +        return exists(self.get_path())
    1.43 +
    1.44 +    def to_dict(self) -> dict:
    1.45 +        return dict((i, getattr(self, i)) for i in ('filename', 'time'))
    1.46 +
    1.47 +    @classmethod
    1.48 +    def from_dict(cls, controller: StoreControllerBase, d: Dict):
    1.49 +        return cls(
    1.50 +            controller=controller,
    1.51 +            **d
    1.52 +        )
    1.53 +
    1.54 +    @classmethod
    1.55 +    def new(cls, controller: StoreControllerBase, short_filename: str):
    1.56 +        time = datetime.now()
    1.57 +        time_prefix = time.strftime('%Y-%m-%d_%H-%M-%S')
    1.58 +        filename = f'{time_prefix} - {short_filename}'
    1.59 +
    1.60 +        return cls(
    1.61 +            controller=controller,
    1.62 +            time=int(time.timestamp()),
    1.63 +            filename=filename
    1.64 +        )
    1.65 +
    1.66 +
    1.67 +class StoreController(StoreControllerBase):
    1.68 +    def __init__(self, task: DBTask):
    1.69 +        self.task = task
    1.70 +        self.index_name = self.get_filename(f'{task.name}.index')
    1.71 +        self.idx: Dict[int, IndexItem] = {}
    1.72 +
    1.73 +        if exists(self.index_name):
    1.74 +            self.load_index()
    1.75 +
    1.76 +    def get_filename(self, filename: str) -> str:
    1.77 +        return str(p_join(self.task.dst_dir, filename))
    1.78 +
    1.79 +    def load_index(self) -> None:
    1.80 +        with open(self.index_name, 'r') as IN:
    1.81 +            result = {}
    1.82 +            for itm in json.load(IN):
    1.83 +                itm = IndexItem.from_dict(self, itm)
    1.84 +                if itm.is_exists():
    1.85 +                    result[itm.time] = itm
    1.86 +
    1.87 +        self.idx = result
    1.88 +
    1.89 +    def save_index(self) -> None:
    1.90 +        with open(self.index_name, 'w') as OUT:
    1.91 +            json.dump(list(map(lambda x: x.to_dict(), self.idx.values())), OUT)
    1.92 +
    1.93 +    def new_item(self) -> IndexItem:
    1.94 +        return IndexItem.new(self, f'{self.task.name}.backup')
    1.95 +
    1.96 +    def add_item(self, item: IndexItem) -> None:
    1.97 +        item_path = item.get_path()
    1.98 +        if not item.is_exists():
    1.99 +            raise StoreError(f'Storing to index file not found: {item.get_path()}')
   1.100 +
   1.101 +        if item.time in self.idx and self.idx[item.time].filename != item.filename:
   1.102 +            if self.idx[item.time].is_exists():
   1.103 +                file_remove(self.idx[item.time].get_path())
   1.104 +
   1.105 +        self.idx[item.time] = item
   1.106 +
   1.107 +    def remove(self, item: IndexItem) -> None:
   1.108 +        if item.time in self.idx:
   1.109 +            del self.idx[item.time]
   1.110 +
   1.111 +        if item.is_exists():
   1.112 +            file_remove(item.get_path())
   1.113 +
   1.114 +    def __iter__(self):
   1.115 +        for i in sorted(self.idx):
   1.116 +            yield self.idx[i]
   1.117 +
   1.118 +    def clean(self, tier1_days: int, tier2_copies_interval: int, tier2_store_days: int) -> List[str]:
   1.119 +        to_remove = []
   1.120 +        tier2_idx = {}
   1.121 +        now = datetime.now()
   1.122 +
   1.123 +        for item in self:
   1.124 +            if not item.is_exists():
   1.125 +                to_remove.append(item)
   1.126 +
   1.127 +            else:
   1.128 +                storing_days = (now - item.get_datetime()).days
   1.129 +
   1.130 +                if not storing_days <= tier1_days:
   1.131 +                    if storing_days > tier2_store_days:
   1.132 +                        to_remove.append(item)
   1.133 +
   1.134 +                    else:
   1.135 +                        # Магия: Делим старые резервные копии на эпохи. Эпоха - целая часть от деления количества
   1.136 +                        #        дней, которое лежит резервная копия, на интервал, за который нам нужно хранить
   1.137 +                        #        хотя бы одну копию. В одной эпохе старший файл вытесняет младший. Из вычислений
   1.138 +                        #        убираем период tier1
   1.139 +
   1.140 +                        storing_days -= tier1_days
   1.141 +
   1.142 +                        _epoch = divmod(storing_days, tier2_copies_interval)[0]
   1.143 +                        if _epoch in tier2_idx:
   1.144 +                            to_remove.append(tier2_idx[_epoch])
   1.145 +
   1.146 +                        tier2_idx[_epoch] = item
   1.147 +
   1.148 +        result = []
   1.149 +        for item in to_remove:
   1.150 +            file_remove(item.get_path())
   1.151 +            result.append(item.filename)
   1.152 +
   1.153 +        return result