jsdio/ips_etl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Вывел получение данных в единую функцию

Browse Source
This commit is contained in:
Григорич 2023-06-06 11:28:20 +03:00
parent 4fc9b0fc64
commit b7272a6e1e
1 changed files with 44 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
src/extractor.py
View File

@ -1,6 +1,7 @@
import logging
import json
import os
import time
import aiohttp
import backoff
@ -11,18 +12,28 @@ from settings import ApiConfig, EtlConfig
from state import State
load_dotenv()
etl_config = EtlConfig()
class ApiExtractor:
def __init__(self):
self.api_config = ApiConfig()
self.headers = {"Authorization-Token": self.api_config.token}
self.headers = {
"Authorization-Token": self.api_config.token,
}
self.fields = ["ID", "IBLOCK_ID", "NAME", "CODE", "SECTION_ID"]
@backoff.on_exception(backoff.expo, (aiohttp.ClientResponseError, aiohttp.ClientConnectorError,
aiohttp.ServerDisconnectedError), base=2, factor=1,
max_value=etl_config.max_wait_size, max_tries=None)
async def __get_data(self, url, data):
async with aiohttp.ClientSession() as session:
async with session.get(url=url, headers=self.headers,
data=data) as resp:
return json.loads(await resp.text())
@backoff.on_exception(backoff.expo, (aiohttp.ClientResponseError, aiohttp.ClientConnectorError), base=2, factor=1,
max_value=int(etl_config.max_wait_size), max_tries=None)
async def get_extract_data(self, state: State, iblock_id: int, fields: list = None, **kwargs) -> list[Abitr]:
data = {
@ -38,39 +49,36 @@ class ApiExtractor:
data['bitrFilter'] = json.dumps({'>ID': str(min_id)}, ensure_ascii=False)
abitrs = []
async with aiohttp.ClientSession() as session:
async with session.get(f'{self.api_config.host}/getnativeiblockelementslist', headers=self.headers,
data=data) as resp:
# Получаем всех новых абитуриентов
results = await self.__get_data(f'{self.api_config.host}/getnativeiblockelementslist', data)
if len(results) > 0:
logging.info(f'Получение абитуриентов ID > {min_id}. Iblock - {iblock_id}')
for result in results:
data = {
'elementId': result,
}
results = json.loads(await resp.text())
if len(results) > 0:
logging.info(f'Получение абитуриентов ID > {min_id}. Iblock - {iblock_id}')
for result in results:
data = {
'elementId': result,
}
async with session.get(f'{self.api_config.host}/getiblockelement/', headers=self.headers,
data=data) as resp:
res = json.loads(await resp.text())
res = res[str(result)]
class_attrs = vars(Abitr)['__annotations__']
try:
class_attrs.pop('ID')
except KeyError:
pass
abitr = {}
abitr['ID'] = res['ID']
for key, value in class_attrs.items():
if key in res:
abitr[key] = res[key]['VALUE']
if res['BENEFITS']['VALUE'] != '-' or res['BENEFITS2']['VALUE'] != '-':
if res['BENEFITS']['VALUE'] != '-':
abitr['BENEFITS'] = res['BENEFITS']['VALUE']
abitr['PRIVILEGES_QUESTION'] = 2
if res['BENEFITS2']['VALUE'] != '-':
abitr['BENEFITS'] = res['BENEFITS2']['VALUE']
abitr['PRIVILEGES_QUESTION'] = 1
abitrs.append(Abitr(**{key: value for key, value in abitr.items()}))
# Получаем информацию об конкретном абитуриенте
res = await self.__get_data(f'{self.api_config.host}/getiblockelement', data)
res = res[str(result)]
class_attrs = vars(Abitr)['__annotations__']
try:
class_attrs.pop('ID')
except KeyError:
pass
abitr = {}
abitr['ID'] = res['ID']
for key, value in class_attrs.items():
if key in res:
abitr[key] = res[key]['VALUE']
if res['BENEFITS']['VALUE'] != '-' or res['BENEFITS2']['VALUE'] != '-':
if res['BENEFITS']['VALUE'] != '-':
abitr['BENEFITS'] = res['BENEFITS']['VALUE']
abitr['PRIVILEGES_QUESTION'] = 2
if res['BENEFITS2']['VALUE'] != '-':
abitr['BENEFITS'] = res['BENEFITS2']['VALUE']
abitr['PRIVILEGES_QUESTION'] = 1
abitrs.append(Abitr(**{key: value for key, value in abitr.items()}))
if len(abitrs) > 0:
logging.info(f'Получено абитуриентов - {len(abitrs)}')
return abitrs