.env.dist

@@ -4,9 +4,5 @@ IBLOCKS=1,2,3
 DESTINATION_HOST=127.0.0.1
 DESTINATION_PORT=8000
 DESTINATION_PROTOCOL=http
-SIGNED=ERIGJHEJKRGH
-
-
-# ETL_PROCESS
 LAST_ID=0
-MAX_WAIT_SIZE=60
+SIGNED=ERIGJHEJKRGH

README.md

@@ -19,5 +19,4 @@
     - `DESTINATION_PORT`: порт назначения.
     - `DESTINATION_PROTOCOL`: протокол назначения (http или https).
     - `LAST_ID`: последний успешно обработанный ID.
-    - `MAX_WAIT_SIZE`: максимальное время ожидания backoff.
 4. После настройки переменных, запустите проект при помощи docker-compose командой `docker-compose up -d`.

src/extractor.py

@@ -7,11 +7,10 @@ import backoff
 from environs import load_dotenv
 
 from models import Abitr
-from settings import ApiConfig, EtlConfig
+from settings import ApiConfig
 from state import State
 
 load_dotenv()
-etl_config = EtlConfig()
 
 class ApiExtractor:
 
@@ -22,7 +21,7 @@ class ApiExtractor:
 
     @backoff.on_exception(backoff.expo, (aiohttp.ClientResponseError, aiohttp.ClientConnectorError,
                                          aiohttp.ServerDisconnectedError), base=2, factor=1,
-                          max_value=etl_config.max_wait_size, max_tries=None)
+                          max_value=5, max_tries=None)
     async def get_extract_data(self, state: State, iblock_id: int, fields: list = None, **kwargs) -> list[Abitr]:
 
         data = {

src/loader.py

@@ -12,7 +12,6 @@ from settings import ApiConfig, EtlConfig
 from state import State, BaseStorage
 
 load_dotenv()
-etl_config = EtlConfig()
 
 
 class EtlLoader:
@@ -22,7 +21,7 @@ class EtlLoader:
         self.api_config = ApiConfig()
 
     @backoff.on_exception(backoff.expo, (asyncio.TimeoutError, ), base=2, factor=1,
-                          max_value=etl_config.max_wait_size, max_tries=None)
+                          max_value=5, max_tries=None)
     async def load_data(self, state: State, abitr: Abitr, iblock_id: int, storage: BaseStorage):
         async with aiohttp.ClientSession() as session:
             logging.info(f"Информация об абитуриенте: {abitr.FIO}")

src/settings.py

@@ -18,4 +18,3 @@ class EtlConfig:
     host: str = os.environ.get('DESTINATION_HOST')
     port: int = int(os.environ.get('DESTINATION_PORT'))
     protocol: str = os.environ.get('DESTINATION_PROTOCOL')
-    max_wait_size: int = os.environ.get('MAX_WAIT_SIZE', 60)