Написан класс получения данных из хранилища

extractor.py

@@ -0,0 +1,72 @@
+import logging
+import json
+import aiohttp
+import backoff
+from environs import load_dotenv
+
+from models import Abitr
+from settings import ApiConfig
+from state import State
+
+load_dotenv()
+
+class ApiExtractor:
+
+    def __init__(self):
+        self.api_config = ApiConfig()
+        self.headers = {"Authorization-Token": self.api_config.token}
+        self.fields = ["ID", "IBLOCK_ID", "NAME", "CODE", "SECTION_ID"]
+
+    @backoff.on_exception(backoff.expo, (aiohttp.ClientResponseError, aiohttp.ClientConnectorError), base=2, factor=1,
+                          max_value=5, max_tries=None)
+    async def get_extract_data(self, state: State, iblock_id: int, fields: list = None, **kwargs) -> list[Abitr]:
+
+        data = {
+            'iblockId': iblock_id,
+            'order': json.dumps({'ID': 'ASC'}, ensure_ascii=False)
+        }
+        if fields is not None:
+            data['fields'] = json.dumps(self.fields + fields, ensure_ascii=False)
+        else:
+            data['fields'] = json.dumps(self.fields + ["PROPERTY_*"], ensure_ascii=False)
+
+        min_id = state.get_state(f'iblock_{iblock_id}') or 0
+        data['bitrFilter'] = json.dumps({'>ID': str(min_id)}, ensure_ascii=False)
+        abitrs = []
+
+        async with aiohttp.ClientSession() as session:
+            async with session.get(f'{self.api_config.host}/getnativeiblockelementslist', headers=self.headers,
+                                   data=data) as resp:
+
+                results = json.loads(await resp.text())
+                if len(results) > 0:
+                    logging.info(f'Получение абитуриентов ID > {min_id}. Iblock - {iblock_id}')
+                for result in results:
+                    data = {
+                        'elementId': result,
+                    }
+                    async with session.get(f'{self.api_config.host}/getiblockelement/', headers=self.headers,
+                                           data=data) as resp:
+                        res = json.loads(await resp.text())
+                        res = res[str(result)]
+                        class_attrs = vars(Abitr)['__annotations__']
+                        try:
+                            class_attrs.pop('ID')
+                        except KeyError:
+                            pass
+                        abitr = {}
+                        abitr['ID'] = res['ID']
+                        for key, value in class_attrs.items():
+                            if key in res:
+                                abitr[key] = res[key]['VALUE']
+                        if res['BENEFITS']['VALUE'] != '-' or res['BENEFITS2']['VALUE'] != '-':
+                            if res['BENEFITS']['VALUE'] != '-':
+                                abitr['BENEFITS'] = res['BENEFITS']['VALUE']
+                                abitr['PRIVILEGES_QUESTION'] = 2
+                            if res['BENEFITS2']['VALUE'] != '-':
+                                abitr['BENEFITS'] = res['BENEFITS2']['VALUE']
+                                abitr['PRIVILEGES_QUESTION'] = 1
+                        abitrs.append(Abitr(**{key: value for key, value in abitr.items()}))
+        if len(abitrs) > 0:
+            logging.info(f'Получено абитуриентов - {len(abitrs)}')
+        return abitrs