From b7272a6e1e5247fe7d1f0a788456fcfdf0ecb598 Mon Sep 17 00:00:00 2001
From: jsdio <jsdio@jsdio.ru>
Date: Tue, 6 Jun 2023 11:28:20 +0300
Subject: [PATCH] =?UTF-8?q?=D0=92=D1=8B=D0=B2=D0=B5=D0=BB=20=D0=BF=D0=BE?=
 =?UTF-8?q?=D0=BB=D1=83=D1=87=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=B4=D0=B0=D0=BD?=
 =?UTF-8?q?=D0=BD=D1=8B=D1=85=20=D0=B2=20=D0=B5=D0=B4=D0=B8=D0=BD=D1=83?=
 =?UTF-8?q?=D1=8E=20=D1=84=D1=83=D0=BD=D0=BA=D1=86=D0=B8=D1=8E?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/extractor.py | 80 ++++++++++++++++++++++++++----------------------
 1 file changed, 44 insertions(+), 36 deletions(-)

diff --git a/src/extractor.py b/src/extractor.py
index c338608..b7f89d2 100644
--- a/src/extractor.py
+++ b/src/extractor.py
@@ -1,6 +1,7 @@
 import logging
 import json
 import os
+import time
 
 import aiohttp
 import backoff
@@ -11,18 +12,28 @@ from settings import ApiConfig, EtlConfig
 from state import State
 
 load_dotenv()
+
+
 etl_config = EtlConfig()
 
+
 class ApiExtractor:
 
     def __init__(self):
         self.api_config = ApiConfig()
-        self.headers = {"Authorization-Token": self.api_config.token}
+        self.headers = {
+            "Authorization-Token": self.api_config.token,
+        }
         self.fields = ["ID", "IBLOCK_ID", "NAME", "CODE", "SECTION_ID"]
 
-    @backoff.on_exception(backoff.expo, (aiohttp.ClientResponseError, aiohttp.ClientConnectorError,
-                                         aiohttp.ServerDisconnectedError), base=2, factor=1,
-                          max_value=etl_config.max_wait_size, max_tries=None)
+    async def __get_data(self, url, data):
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url=url, headers=self.headers,
+                                   data=data) as resp:
+                return json.loads(await resp.text())
+
+    @backoff.on_exception(backoff.expo, (aiohttp.ClientResponseError, aiohttp.ClientConnectorError), base=2, factor=1,
+                          max_value=int(etl_config.max_wait_size), max_tries=None)
     async def get_extract_data(self, state: State, iblock_id: int, fields: list = None, **kwargs) -> list[Abitr]:
 
         data = {
@@ -38,39 +49,36 @@ class ApiExtractor:
         data['bitrFilter'] = json.dumps({'>ID': str(min_id)}, ensure_ascii=False)
         abitrs = []
 
-        async with aiohttp.ClientSession() as session:
-            async with session.get(f'{self.api_config.host}/getnativeiblockelementslist', headers=self.headers,
-                                   data=data) as resp:
+        # Получаем всех новых абитуриентов
+        results = await self.__get_data(f'{self.api_config.host}/getnativeiblockelementslist', data)
+        if len(results) > 0:
+            logging.info(f'Получение абитуриентов ID > {min_id}. Iblock - {iblock_id}')
+        for result in results:
+            data = {
+                'elementId': result,
+            }
 
-                results = json.loads(await resp.text())
-                if len(results) > 0:
-                    logging.info(f'Получение абитуриентов ID > {min_id}. Iblock - {iblock_id}')
-                for result in results:
-                    data = {
-                        'elementId': result,
-                    }
-                    async with session.get(f'{self.api_config.host}/getiblockelement/', headers=self.headers,
-                                           data=data) as resp:
-                        res = json.loads(await resp.text())
-                        res = res[str(result)]
-                        class_attrs = vars(Abitr)['__annotations__']
-                        try:
-                            class_attrs.pop('ID')
-                        except KeyError:
-                            pass
-                        abitr = {}
-                        abitr['ID'] = res['ID']
-                        for key, value in class_attrs.items():
-                            if key in res:
-                                abitr[key] = res[key]['VALUE']
-                        if res['BENEFITS']['VALUE'] != '-' or res['BENEFITS2']['VALUE'] != '-':
-                            if res['BENEFITS']['VALUE'] != '-':
-                                abitr['BENEFITS'] = res['BENEFITS']['VALUE']
-                                abitr['PRIVILEGES_QUESTION'] = 2
-                            if res['BENEFITS2']['VALUE'] != '-':
-                                abitr['BENEFITS'] = res['BENEFITS2']['VALUE']
-                                abitr['PRIVILEGES_QUESTION'] = 1
-                        abitrs.append(Abitr(**{key: value for key, value in abitr.items()}))
+            # Получаем информацию об конкретном абитуриенте
+            res = await self.__get_data(f'{self.api_config.host}/getiblockelement', data)
+            res = res[str(result)]
+            class_attrs = vars(Abitr)['__annotations__']
+            try:
+                class_attrs.pop('ID')
+            except KeyError:
+                pass
+            abitr = {}
+            abitr['ID'] = res['ID']
+            for key, value in class_attrs.items():
+                if key in res:
+                    abitr[key] = res[key]['VALUE']
+            if res['BENEFITS']['VALUE'] != '-' or res['BENEFITS2']['VALUE'] != '-':
+                if res['BENEFITS']['VALUE'] != '-':
+                    abitr['BENEFITS'] = res['BENEFITS']['VALUE']
+                    abitr['PRIVILEGES_QUESTION'] = 2
+                if res['BENEFITS2']['VALUE'] != '-':
+                    abitr['BENEFITS'] = res['BENEFITS2']['VALUE']
+                    abitr['PRIVILEGES_QUESTION'] = 1
+            abitrs.append(Abitr(**{key: value for key, value in abitr.items()}))
         if len(abitrs) > 0:
             logging.info(f'Получено абитуриентов - {len(abitrs)}')
         return abitrs