mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-20 01:45:10 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

refactor: All parsers return snapshot instead of link

Browse source
This commit is contained in:
Cristian 2021-01-04 09:31:14 -05:00
parent 15d88be229
commit b8efaa5b6a
11 changed files with 72 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

11
archivebox/parsers/generic_txt.py
View file

@ -7,7 +7,8 @@ from typing import IO, Iterable
from datetime import datetime
from pathlib import Path
from ..index.schema import Link
from django.db.models import Model
from ..util import (
htmldecode,
enforce_types,
@ -16,7 +17,7 @@ from ..util import (
@enforce_types
def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Model]:
"""Parse raw links from each line in a text file"""
# TODO: Check if we should add sources list to the database
from core.models import Snapshot
@ -29,12 +30,12 @@ def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
# if the line is a local file path that resolves, then we can archive it
try:
if Path(line).exists():
yield Link(
yield Snapshot(
url=line,
timestamp=str(datetime.now().timestamp()),
title=None,
tags=None,
sources=[text_file.name],
#tags=None,
#sources=[text_file.name],
)
except (OSError, PermissionError):
# nvm, not a valid path...