From e5794029abcc17dc0467d45a1efba5be242e4d41 Mon Sep 17 00:00:00 2001
From: jsdio <jsdio@jsdio.ru>
Date: Tue, 18 Jul 2023 11:00:36 +0300
Subject: [PATCH] =?UTF-8?q?=D0=98=D0=B3=D0=BD=D0=BE=D1=80=20=D0=BE=D1=88?=
 =?UTF-8?q?=D0=B8=D0=B1=D0=BA=D0=B8=20=D0=BF=D1=80=D0=B8=20=D0=B2=D0=B0?=
 =?UTF-8?q?=D0=BB=D0=B8=D0=B4=D0=B0=D1=86=D0=B8=D0=B8?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/extractor.py | 18 +++++++++++++++---
 src/main.py      |  2 +-
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/src/extractor.py b/src/extractor.py
index 25654aa..a8595da 100644
--- a/src/extractor.py
+++ b/src/extractor.py
@@ -4,11 +4,13 @@ import os
 
 import aiohttp
 import backoff
+import pydantic.error_wrappers
 from environs import load_dotenv
 
 from models import Abitr
 from settings import ApiConfig, EtlConfig, MiscSettings
-from state import State
+from state import State, BaseStorage
+from tg_bot import TgBot
 
 load_dotenv()
 
@@ -39,7 +41,8 @@ class ApiExtractor:
     @backoff.on_exception(backoff.expo, (aiohttp.ClientResponseError, aiohttp.ClientConnectorError,
                                          aiohttp.ServerDisconnectedError), base=2, factor=1,
                           max_value=int(misc_settings.max_wait_size), max_tries=None)
-    async def get_extract_data(self, state: State, iblock_id: int, fields: list = None, **kwargs) -> list[Abitr]:
+    async def get_extract_data(self, state: State, iblock_id: int, storage: BaseStorage, fields: list = None,
+                               **kwargs) -> list[Abitr]:
 
         data = {
             'iblockId': iblock_id,
@@ -80,7 +83,16 @@ class ApiExtractor:
                     abitr[key] = res[key]['VALUE']
             if abitr['BENEFITS'] == '' or abitr['BENEFITS'] is None:
                   abitr['BENEFITS'] = 8
-            abitrs.append(Abitr(**{key: value for key, value in abitr.items()}))
+            try:
+                abitrs.append(Abitr(**{key: value for key, value in abitr.items()}))
+            except pydantic.error_wrappers.ValidationError as e:
+                logging.info(f'Ошибка валидации ID {abitr["ID"]} - {str(e)}')
+                tgbot = TgBot()
+                await tgbot.send_notify(f'Ошибка валидации ID {abitr["ID"]} - {str(e)}')
+                state.set_state(f'iblock_{iblock_id}', abitr["ID"])
+                storage.save_state(state.local_state)
+
+
         if len(abitrs) > 0:
             logging.info(f'Получено абитуриентов - {len(abitrs)}')
         return abitrs
diff --git a/src/main.py b/src/main.py
index 14ef5bc..6a04bc5 100644
--- a/src/main.py
+++ b/src/main.py
@@ -30,7 +30,7 @@ async def main():
 
     while True:
         for iblock_id in api.iblocks:
-            abitrs = await extractor.get_extract_data(iblock_id=int(iblock_id), state=state)
+            abitrs = await extractor.get_extract_data(iblock_id=int(iblock_id), state=state, storage=storage)
             for abitr in abitrs:
                 await loader.load_data(state=state, abitr=abitr, storage=storage, iblock_id=int(iblock_id))
         await asyncio.sleep(10)