From 173598f9ab69f90bcd1ab5650bb5e2fd0a0f6794 Mon Sep 17 00:00:00 2001
From: jsdio <jsdio@jsdio.ru>
Date: Mon, 5 Jun 2023 14:31:44 +0300
Subject: [PATCH] =?UTF-8?q?=D0=9D=D0=B0=D0=BF=D0=B8=D1=81=D0=B0=D0=BD=20?=
 =?UTF-8?q?=D0=BA=D0=BB=D0=B0=D1=81=D1=81=20=D0=BF=D0=BE=D0=BB=D1=83=D1=87?=
 =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=B4=D0=B0=D0=BD=D0=BD=D1=8B=D1=85?=
 =?UTF-8?q?=20=D0=B8=D0=B7=20=D1=85=D1=80=D0=B0=D0=BD=D0=B8=D0=BB=D0=B8?=
 =?UTF-8?q?=D1=89=D0=B0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 extractor.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 72 insertions(+)
 create mode 100644 extractor.py

diff --git a/extractor.py b/extractor.py
new file mode 100644
index 0000000..9ab4960
--- /dev/null
+++ b/extractor.py
@@ -0,0 +1,72 @@
+import logging
+import json
+import aiohttp
+import backoff
+from environs import load_dotenv
+
+from models import Abitr
+from settings import ApiConfig
+from state import State
+
+load_dotenv()
+
+class ApiExtractor:
+
+    def __init__(self):
+        self.api_config = ApiConfig()
+        self.headers = {"Authorization-Token": self.api_config.token}
+        self.fields = ["ID", "IBLOCK_ID", "NAME", "CODE", "SECTION_ID"]
+
+    @backoff.on_exception(backoff.expo, (aiohttp.ClientResponseError, aiohttp.ClientConnectorError), base=2, factor=1,
+                          max_value=5, max_tries=None)
+    async def get_extract_data(self, state: State, iblock_id: int, fields: list = None, **kwargs) -> list[Abitr]:
+
+        data = {
+            'iblockId': iblock_id,
+            'order': json.dumps({'ID': 'ASC'}, ensure_ascii=False)
+        }
+        if fields is not None:
+            data['fields'] = json.dumps(self.fields + fields, ensure_ascii=False)
+        else:
+            data['fields'] = json.dumps(self.fields + ["PROPERTY_*"], ensure_ascii=False)
+
+        min_id = state.get_state(f'iblock_{iblock_id}') or 0
+        data['bitrFilter'] = json.dumps({'>ID': str(min_id)}, ensure_ascii=False)
+        abitrs = []
+
+        async with aiohttp.ClientSession() as session:
+            async with session.get(f'{self.api_config.host}/getnativeiblockelementslist', headers=self.headers,
+                                   data=data) as resp:
+
+                results = json.loads(await resp.text())
+                if len(results) > 0:
+                    logging.info(f'Получение абитуриентов ID > {min_id}. Iblock - {iblock_id}')
+                for result in results:
+                    data = {
+                        'elementId': result,
+                    }
+                    async with session.get(f'{self.api_config.host}/getiblockelement/', headers=self.headers,
+                                           data=data) as resp:
+                        res = json.loads(await resp.text())
+                        res = res[str(result)]
+                        class_attrs = vars(Abitr)['__annotations__']
+                        try:
+                            class_attrs.pop('ID')
+                        except KeyError:
+                            pass
+                        abitr = {}
+                        abitr['ID'] = res['ID']
+                        for key, value in class_attrs.items():
+                            if key in res:
+                                abitr[key] = res[key]['VALUE']
+                        if res['BENEFITS']['VALUE'] != '-' or res['BENEFITS2']['VALUE'] != '-':
+                            if res['BENEFITS']['VALUE'] != '-':
+                                abitr['BENEFITS'] = res['BENEFITS']['VALUE']
+                                abitr['PRIVILEGES_QUESTION'] = 2
+                            if res['BENEFITS2']['VALUE'] != '-':
+                                abitr['BENEFITS'] = res['BENEFITS2']['VALUE']
+                                abitr['PRIVILEGES_QUESTION'] = 1
+                        abitrs.append(Abitr(**{key: value for key, value in abitr.items()}))
+        if len(abitrs) > 0:
+            logging.info(f'Получено абитуриентов - {len(abitrs)}')
+        return abitrs