jsdio/ips_etl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Добавил возможность указывать последний успешный ID из переменной окружения LAST_ID

Browse Source
This commit is contained in:
Григорич 2023-06-05 15:39:25 +03:00
parent 3cc6a3a39a
commit 7bcfcc458c
2 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.env.dist
View File

@ -3,4 +3,5 @@ API_TOKEN=jmkdjsfopjsef
IBLOCKS=1,2,3 IBLOCKS=1,2,3
DESTINATION_HOST=127.0.0.1 DESTINATION_HOST=127.0.0.1
DESTINATION_PORT=8000 DESTINATION_PORT=8000
DESTINATION_PROTOCOL=http DESTINATION_PROTOCOL=http
LAST_ID=0

4
extractor.py
View File

@ -1,5 +1,7 @@
import logging import logging
import json import json
import os
import aiohttp import aiohttp
import backoff import backoff
from environs import load_dotenv from environs import load_dotenv
@ -30,7 +32,7 @@ class ApiExtractor:
else: else:
data['fields'] = json.dumps(self.fields + ["PROPERTY_*"], ensure_ascii=False) data['fields'] = json.dumps(self.fields + ["PROPERTY_*"], ensure_ascii=False)
min_id = state.get_state(f'iblock_{iblock_id}') or 0 min_id = state.get_state(f'iblock_{iblock_id}') or os.getenv('LAST_ID') or 0
data['bitrFilter'] = json.dumps({'>ID': str(min_id)}, ensure_ascii=False) data['bitrFilter'] = json.dumps({'>ID': str(min_id)}, ensure_ascii=False)
abitrs = [] abitrs = []