mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-23 03:06:55 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix lint errors

Browse source
This commit is contained in:
Nick Sweeting 2021-04-06 02:50:51 -04:00
parent e92db03f5f
commit 21bb84f627
2 changed files with 6 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

7
archivebox/search/backends/sonic.py
View file

@ -13,10 +13,13 @@ MAX_SONIC_TEXT_CHUNK_LENGTH = 2000 # dont index more than 2000 characte
def index(snapshot_id: str, texts: List[str]):
with IngestClient(SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT, SEARCH_BACKEND_PASSWORD) as ingestcl:
for text in texts:
max_length = 1000000
chunks = (
text[i:i+MAX_SONIC_TEXT_CHUNK_LENGTH]
for i in range(0, min(len(text), MAX_SONIC_TEXT_TOTAL_LENGTH), MAX_SONIC_TEXT_CHUNK_LENGTH)
for i in range(
0,
min(len(text), MAX_SONIC_TEXT_TOTAL_LENGTH),
MAX_SONIC_TEXT_CHUNK_LENGTH,
)
)
for chunk in chunks:
ingestcl.push(SONIC_COLLECTION, SONIC_BUCKET, snapshot_id, str(chunk))