diff --git a/.env.dist b/.env.dist index 53442b9..031fd34 100644 --- a/.env.dist +++ b/.env.dist @@ -4,5 +4,9 @@ IBLOCKS=1,2,3 DESTINATION_HOST=127.0.0.1 DESTINATION_PORT=8000 DESTINATION_PROTOCOL=http +SIGNED=ERIGJHEJKRGH + + +# ETL_PROCESS LAST_ID=0 -SIGNED=ERIGJHEJKRGH \ No newline at end of file +MAX_WAIT_SIZE=60 \ No newline at end of file diff --git a/src/extractor.py b/src/extractor.py index 460394b..c338608 100644 --- a/src/extractor.py +++ b/src/extractor.py @@ -7,10 +7,11 @@ import backoff from environs import load_dotenv from models import Abitr -from settings import ApiConfig +from settings import ApiConfig, EtlConfig from state import State load_dotenv() +etl_config = EtlConfig() class ApiExtractor: @@ -21,7 +22,7 @@ class ApiExtractor: @backoff.on_exception(backoff.expo, (aiohttp.ClientResponseError, aiohttp.ClientConnectorError, aiohttp.ServerDisconnectedError), base=2, factor=1, - max_value=5, max_tries=None) + max_value=etl_config.max_wait_size, max_tries=None) async def get_extract_data(self, state: State, iblock_id: int, fields: list = None, **kwargs) -> list[Abitr]: data = { diff --git a/src/loader.py b/src/loader.py index 0db2232..d3e59b0 100644 --- a/src/loader.py +++ b/src/loader.py @@ -12,6 +12,7 @@ from settings import ApiConfig, EtlConfig from state import State, BaseStorage load_dotenv() +etl_config = EtlConfig() class EtlLoader: @@ -21,7 +22,7 @@ class EtlLoader: self.api_config = ApiConfig() @backoff.on_exception(backoff.expo, (asyncio.TimeoutError, ), base=2, factor=1, - max_value=5, max_tries=None) + max_value=etl_config.max_wait_size, max_tries=None) async def load_data(self, state: State, abitr: Abitr, iblock_id: int, storage: BaseStorage): async with aiohttp.ClientSession() as session: logging.info(f"Информация об абитуриенте: {abitr.FIO}") diff --git a/src/settings.py b/src/settings.py index 787b0fd..b67030c 100644 --- a/src/settings.py +++ b/src/settings.py @@ -18,3 +18,4 @@ class EtlConfig: host: str = os.environ.get('DESTINATION_HOST') port: int = int(os.environ.get('DESTINATION_PORT')) protocol: str = os.environ.get('DESTINATION_PROTOCOL') + max_wait_size: int = os.environ.get('MAX_WAIT_SIZE', 60)