From 9121013eb85a831a00b083a269cc927c0d29761f Mon Sep 17 00:00:00 2001 From: jsdio Date: Tue, 27 Jun 2023 12:29:32 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=B0=D0=BD=D0=BD=D1=8B=D0=B5=20=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2=D1=82=D0=BE=D1=80=D0=BD=D0=BE=20=D0=BE=D1=82=D0=BF?= =?UTF-8?q?=D1=80=D0=B0=D0=B2=D0=BB=D1=8F=D1=82=D1=8C=D1=81=D1=8F=20=D0=BD?= =?UTF-8?q?=D0=B5=20=D0=B1=D1=83=D0=B4=D1=83=D1=82,=20=D0=B5=D1=81=D0=BB?= =?UTF-8?q?=D0=B8=20=D0=B4=D0=BE=D1=81=D1=82=D0=B0=D0=B2=D0=BA=D0=B0=20?= =?UTF-8?q?=D0=B1=D1=8B=D0=BB=D0=B0=20=D1=81=20=D0=BE=D1=88=D0=B8=D0=B1?= =?UTF-8?q?=D0=BA=D0=BE=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/extractor.py | 1 - src/loader.py | 22 ++++++++++------------ 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/extractor.py b/src/extractor.py index d6b472d..ca11c0d 100644 --- a/src/extractor.py +++ b/src/extractor.py @@ -1,7 +1,6 @@ import logging import json import os -import time import aiohttp import backoff diff --git a/src/loader.py b/src/loader.py index 45b97f3..13a6a47 100644 --- a/src/loader.py +++ b/src/loader.py @@ -1,10 +1,8 @@ import asyncio import json import logging -import time import aiohttp -import backoff as backoff from environs import load_dotenv from models import Abitr @@ -22,9 +20,6 @@ class EtlLoader: self.etl = EtlConfig() self.api_config = ApiConfig() - @backoff.on_exception(backoff.expo, (asyncio.TimeoutError, aiohttp.ClientConnectorError, - aiohttp.ServerDisconnectedError), base=2, factor=1, - max_value=int(misc_settings.max_wait_size), max_tries=None) async def load_data(self, state: State, abitr: Abitr, iblock_id: int, storage: BaseStorage): async with aiohttp.ClientSession() as session: logging.info(f"Информация об абитуриенте: {abitr.FIO}") @@ -32,14 +27,17 @@ class EtlLoader: data_dict.pop('ID') logging.info('Отправка данных к принимающей стороне... ') logging.info(f'Данные: \n{json.dumps(data_dict, ensure_ascii=False)}') - async with session.post( - url=f"{self.etl.protocol}://{self.etl.host}:{self.etl.port}", - data=data_dict, - timeout=aiohttp.ClientTimeout(total=int(misc_settings.max_wait_size)) - ) as response: - pass + try: + async with session.post( + url=f"{self.etl.protocol}://{self.etl.host}:{self.etl.port}", + data=data_dict, + timeout=aiohttp.ClientTimeout(total=int(misc_settings.max_wait_size)) + ) as response: + logging.info('Данные успешно доставлены!') + except (asyncio.TimeoutError, aiohttp.ClientConnectorError, aiohttp.ServerDisconnectedError): + logging.error('Данные доставлены с ошибкой') state.set_state(f'iblock_{iblock_id}', abitr.ID) storage.save_state(state.local_state) - logging.info('Данные успешно доставлены!') + await asyncio.sleep(1)