75 lines
3.2 KiB
Python
75 lines
3.2 KiB
Python
import logging
|
|
import json
|
|
import os
|
|
|
|
import aiohttp
|
|
import backoff
|
|
from environs import load_dotenv
|
|
|
|
from models import Abitr
|
|
from settings import ApiConfig
|
|
from state import State
|
|
|
|
load_dotenv()
|
|
|
|
class ApiExtractor:
|
|
|
|
def __init__(self):
|
|
self.api_config = ApiConfig()
|
|
self.headers = {"Authorization-Token": self.api_config.token}
|
|
self.fields = ["ID", "IBLOCK_ID", "NAME", "CODE", "SECTION_ID"]
|
|
|
|
@backoff.on_exception(backoff.expo, (aiohttp.ClientResponseError, aiohttp.ClientConnectorError), base=2, factor=1,
|
|
max_value=5, max_tries=None)
|
|
async def get_extract_data(self, state: State, iblock_id: int, fields: list = None, **kwargs) -> list[Abitr]:
|
|
|
|
data = {
|
|
'iblockId': iblock_id,
|
|
'order': json.dumps({'ID': 'ASC'}, ensure_ascii=False)
|
|
}
|
|
if fields is not None:
|
|
data['fields'] = json.dumps(self.fields + fields, ensure_ascii=False)
|
|
else:
|
|
data['fields'] = json.dumps(self.fields + ["PROPERTY_*"], ensure_ascii=False)
|
|
|
|
min_id = state.get_state(f'iblock_{iblock_id}') or os.getenv('LAST_ID') or 0
|
|
data['bitrFilter'] = json.dumps({'>ID': str(min_id)}, ensure_ascii=False)
|
|
abitrs = []
|
|
|
|
async with aiohttp.ClientSession() as session:
|
|
async with session.get(f'{self.api_config.host}/getnativeiblockelementslist', headers=self.headers,
|
|
data=data) as resp:
|
|
|
|
results = json.loads(await resp.text())
|
|
if len(results) > 0:
|
|
logging.info(f'Получение абитуриентов ID > {min_id}. Iblock - {iblock_id}')
|
|
for result in results:
|
|
data = {
|
|
'elementId': result,
|
|
}
|
|
async with session.get(f'{self.api_config.host}/getiblockelement/', headers=self.headers,
|
|
data=data) as resp:
|
|
res = json.loads(await resp.text())
|
|
res = res[str(result)]
|
|
class_attrs = vars(Abitr)['__annotations__']
|
|
try:
|
|
class_attrs.pop('ID')
|
|
except KeyError:
|
|
pass
|
|
abitr = {}
|
|
abitr['ID'] = res['ID']
|
|
for key, value in class_attrs.items():
|
|
if key in res:
|
|
abitr[key] = res[key]['VALUE']
|
|
if res['BENEFITS']['VALUE'] != '-' or res['BENEFITS2']['VALUE'] != '-':
|
|
if res['BENEFITS']['VALUE'] != '-':
|
|
abitr['BENEFITS'] = res['BENEFITS']['VALUE']
|
|
abitr['PRIVILEGES_QUESTION'] = 2
|
|
if res['BENEFITS2']['VALUE'] != '-':
|
|
abitr['BENEFITS'] = res['BENEFITS2']['VALUE']
|
|
abitr['PRIVILEGES_QUESTION'] = 1
|
|
abitrs.append(Abitr(**{key: value for key, value in abitr.items()}))
|
|
if len(abitrs) > 0:
|
|
logging.info(f'Получено абитуриентов - {len(abitrs)}')
|
|
return abitrs
|