jsdio/ips_etl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Фикс ошибки неправильной загрузки обработки данных из getnativeiblockelementslist

Browse Source
This commit is contained in:
Григорич 2023-07-04 15:10:15 +03:00
parent 9121013eb8
commit 0dfe1d8b7e
1 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/extractor.py
View File

@ -30,7 +30,11 @@ class ApiExtractor:
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
async with session.get(url=url, headers=self.headers, async with session.get(url=url, headers=self.headers,
data=data) as resp: data=data) as resp:
return json.loads(await resp.text()) try:
return json.loads(await resp.text())
except json.JSONDecodeError:
logging.error(f'Получены неверные данные - {resp.text()}')
return {}
@backoff.on_exception(backoff.expo, (aiohttp.ClientResponseError, aiohttp.ClientConnectorError, @backoff.on_exception(backoff.expo, (aiohttp.ClientResponseError, aiohttp.ClientConnectorError,
aiohttp.ServerDisconnectedError), base=2, factor=1, aiohttp.ServerDisconnectedError), base=2, factor=1,
@ -45,8 +49,10 @@ class ApiExtractor:
data['fields'] = json.dumps(self.fields + fields, ensure_ascii=False) data['fields'] = json.dumps(self.fields + fields, ensure_ascii=False)
else: else:
data['fields'] = json.dumps(self.fields + ["PROPERTY_*"], ensure_ascii=False) data['fields'] = json.dumps(self.fields + ["PROPERTY_*"], ensure_ascii=False)
min_id = state.get_state(f'iblock_{iblock_id}') or os.getenv('LAST_ID') or 0 min_id = state.get_state(f'iblock_{iblock_id}') or os.getenv('LAST_ID') or 0
if os.getenv('LAST_ID').isdigit():
if int(min_id) < int(os.getenv('LAST_ID')):
min_id = os.getenv('LAST_ID')
data['bitrFilter'] = json.dumps({'>ID': str(min_id)}, ensure_ascii=False) data['bitrFilter'] = json.dumps({'>ID': str(min_id)}, ensure_ascii=False)
abitrs = [] abitrs = []