diff --git a/src/extractor.py b/src/extractor.py index 638d0d9..0064102 100644 --- a/src/extractor.py +++ b/src/extractor.py @@ -8,13 +8,14 @@ import backoff from environs import load_dotenv from models import Abitr -from settings import ApiConfig, EtlConfig +from settings import ApiConfig, EtlConfig, MiscSettings from state import State load_dotenv() etl_config = EtlConfig() +misc_settings = MiscSettings() class ApiExtractor: @@ -32,8 +33,9 @@ class ApiExtractor: data=data) as resp: return json.loads(await resp.text()) - @backoff.on_exception(backoff.expo, (aiohttp.ClientResponseError, aiohttp.ClientConnectorError), base=2, factor=1, - max_value=int(etl_config.max_wait_size), max_tries=None) + @backoff.on_exception(backoff.expo, (aiohttp.ClientResponseError, aiohttp.ClientConnectorError, + aiohttp.ServerDisconnectedError), base=2, factor=1, + max_value=int(misc_settings.max_wait_size), max_tries=None) async def get_extract_data(self, state: State, iblock_id: int, fields: list = None, **kwargs) -> list[Abitr]: data = { diff --git a/src/loader.py b/src/loader.py index 1e9f5c0..9a24dbc 100644 --- a/src/loader.py +++ b/src/loader.py @@ -8,11 +8,12 @@ import backoff as backoff from environs import load_dotenv from models import Abitr -from settings import ApiConfig, EtlConfig +from settings import ApiConfig, EtlConfig, MiscSettings from state import State, BaseStorage load_dotenv() etl_config = EtlConfig() +misc_settings = MiscSettings() class EtlLoader: @@ -21,8 +22,8 @@ class EtlLoader: self.etl = EtlConfig() self.api_config = ApiConfig() - @backoff.on_exception(backoff.expo, (asyncio.TimeoutError, aiohttp.ClientConnectorError), base=2, factor=1, - max_value=int(etl_config.max_wait_size), max_tries=None) + @backoff.on_exception(backoff.expo, (asyncio.TimeoutError, aiohttp.ClientConnectorError,), base=2, factor=1, + max_value=int(misc_settings.max_wait_size), max_tries=None) async def load_data(self, state: State, abitr: Abitr, iblock_id: int, storage: BaseStorage): async with aiohttp.ClientSession() as session: logging.info(f"Информация об абитуриенте: {abitr.FIO}") @@ -33,7 +34,7 @@ class EtlLoader: async with session.post( url=f"{self.etl.protocol}://{self.etl.host}:{self.etl.port}", data=data_dict, - timeout=aiohttp.ClientTimeout(total=int(etl_config.max_wait_size)) + timeout=aiohttp.ClientTimeout(total=int(misc_settings.max_wait_size)) ) as response: pass state.set_state(f'iblock_{iblock_id}', abitr.ID) diff --git a/src/settings.py b/src/settings.py index b67030c..39c73e6 100644 --- a/src/settings.py +++ b/src/settings.py @@ -18,4 +18,8 @@ class EtlConfig: host: str = os.environ.get('DESTINATION_HOST') port: int = int(os.environ.get('DESTINATION_PORT')) protocol: str = os.environ.get('DESTINATION_PROTOCOL') + +@dataclass +class MiscSettings: max_wait_size: int = os.environ.get('MAX_WAIT_SIZE', 60) + use_notify: bool = os.environ.get('USE_NOTIFY', False) == 'True'