jsdio/ips_etl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Вывел получение данных в единую функцию

Browse Source
This commit is contained in:
Григорич 2023-06-06 11:28:20 +03:00
parent 4fc9b0fc64
commit b7272a6e1e
1 changed files with 44 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
src/extractor.py
View File

@ -1,6 +1,7 @@
import logging import logging
import json import json
import os import os
import time
import aiohttp import aiohttp
import backoff import backoff
@ -11,18 +12,28 @@ from settings import ApiConfig, EtlConfig
from state import State from state import State
load_dotenv() load_dotenv()
etl_config = EtlConfig() etl_config = EtlConfig()
class ApiExtractor: class ApiExtractor:
def __init__(self): def __init__(self):
self.api_config = ApiConfig() self.api_config = ApiConfig()
self.headers = {"Authorization-Token": self.api_config.token} self.headers = {
"Authorization-Token": self.api_config.token,
}
self.fields = ["ID", "IBLOCK_ID", "NAME", "CODE", "SECTION_ID"] self.fields = ["ID", "IBLOCK_ID", "NAME", "CODE", "SECTION_ID"]
@backoff.on_exception(backoff.expo, (aiohttp.ClientResponseError, aiohttp.ClientConnectorError, async def __get_data(self, url, data):
aiohttp.ServerDisconnectedError), base=2, factor=1, async with aiohttp.ClientSession() as session:
max_value=etl_config.max_wait_size, max_tries=None) async with session.get(url=url, headers=self.headers,
data=data) as resp:
return json.loads(await resp.text())
@backoff.on_exception(backoff.expo, (aiohttp.ClientResponseError, aiohttp.ClientConnectorError), base=2, factor=1,
max_value=int(etl_config.max_wait_size), max_tries=None)
async def get_extract_data(self, state: State, iblock_id: int, fields: list = None, **kwargs) -> list[Abitr]: async def get_extract_data(self, state: State, iblock_id: int, fields: list = None, **kwargs) -> list[Abitr]:
data = { data = {
@ -38,39 +49,36 @@ class ApiExtractor:
data['bitrFilter'] = json.dumps({'>ID': str(min_id)}, ensure_ascii=False) data['bitrFilter'] = json.dumps({'>ID': str(min_id)}, ensure_ascii=False)
abitrs = [] abitrs = []
async with aiohttp.ClientSession() as session: # Получаем всех новых абитуриентов
async with session.get(f'{self.api_config.host}/getnativeiblockelementslist', headers=self.headers, results = await self.__get_data(f'{self.api_config.host}/getnativeiblockelementslist', data)
data=data) as resp: if len(results) > 0:
logging.info(f'Получение абитуриентов ID > {min_id}. Iblock - {iblock_id}')
for result in results:
data = {
'elementId': result,
}
results = json.loads(await resp.text()) # Получаем информацию об конкретном абитуриенте
if len(results) > 0: res = await self.__get_data(f'{self.api_config.host}/getiblockelement', data)
logging.info(f'Получение абитуриентов ID > {min_id}. Iblock - {iblock_id}') res = res[str(result)]
for result in results: class_attrs = vars(Abitr)['__annotations__']
data = { try:
'elementId': result, class_attrs.pop('ID')
} except KeyError:
async with session.get(f'{self.api_config.host}/getiblockelement/', headers=self.headers, pass
data=data) as resp: abitr = {}
res = json.loads(await resp.text()) abitr['ID'] = res['ID']
res = res[str(result)] for key, value in class_attrs.items():
class_attrs = vars(Abitr)['__annotations__'] if key in res:
try: abitr[key] = res[key]['VALUE']
class_attrs.pop('ID') if res['BENEFITS']['VALUE'] != '-' or res['BENEFITS2']['VALUE'] != '-':
except KeyError: if res['BENEFITS']['VALUE'] != '-':
pass abitr['BENEFITS'] = res['BENEFITS']['VALUE']
abitr = {} abitr['PRIVILEGES_QUESTION'] = 2
abitr['ID'] = res['ID'] if res['BENEFITS2']['VALUE'] != '-':
for key, value in class_attrs.items(): abitr['BENEFITS'] = res['BENEFITS2']['VALUE']
if key in res: abitr['PRIVILEGES_QUESTION'] = 1
abitr[key] = res[key]['VALUE'] abitrs.append(Abitr(**{key: value for key, value in abitr.items()}))
if res['BENEFITS']['VALUE'] != '-' or res['BENEFITS2']['VALUE'] != '-':
if res['BENEFITS']['VALUE'] != '-':
abitr['BENEFITS'] = res['BENEFITS']['VALUE']
abitr['PRIVILEGES_QUESTION'] = 2
if res['BENEFITS2']['VALUE'] != '-':
abitr['BENEFITS'] = res['BENEFITS2']['VALUE']
abitr['PRIVILEGES_QUESTION'] = 1
abitrs.append(Abitr(**{key: value for key, value in abitr.items()}))
if len(abitrs) > 0: if len(abitrs) > 0:
logging.info(f'Получено абитуриентов - {len(abitrs)}') logging.info(f'Получено абитуриентов - {len(abitrs)}')
return abitrs return abitrs