mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-13 14:44:29 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

add git downloading

Browse source
This commit is contained in:
Nick Sweeting 2019-01-11 05:18:49 -05:00
parent 5b6c768a47
commit 827e15b31a
4 changed files with 48 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

16
archivebox/parse.py
View file

@ -81,9 +81,9 @@ def parse_pocket_export(html_file):
'url': fixed_url,
'domain': domain(fixed_url),
'base_url': base_url(fixed_url),
'timestamp': str(time.timestamp()),
'timestamp': str(datetime.now().timestamp()),
'tags': match.group(3),
'title': match.group(4).replace(' — Readability', '').replace('http://www.readability.com/read?url=', '') or base_url(fixed_url),
'title': match.group(4).replace(' — Readability', '').replace('http://www.readability.com/read?url=', '') or fetch_page_title(url),
'sources': [html_file.name],
}
info['type'] = get_link_type(info)
@ -157,7 +157,7 @@ def parse_rss_export(rss_file):
'url': url,
'domain': domain(url),
'base_url': base_url(url),
'timestamp': str(time.timestamp()),
'timestamp': str(datetime.now().timestamp()),
'tags': '',
'title': title or fetch_page_title(url),
'sources': [rss_file.name],
@ -184,7 +184,7 @@ def parse_bookmarks_export(html_file):
'url': url,
'domain': domain(url),
'base_url': base_url(url),
'timestamp': str(time.timestamp()),
'timestamp': str(datetime.now().timestamp()),
'tags': "",
'title': match.group(3).strip() or fetch_page_title(url),
'sources': [html_file.name],
@ -217,7 +217,7 @@ def parse_pinboard_rss_feed(rss_file):
'url': url,
'domain': domain(url),
'base_url': base_url(url),
'timestamp': str(time.timestamp()),
'timestamp': str(datetime.now().timestamp()),
'tags': tags,
'title': title or fetch_page_title(url),
'sources': [rss_file.name],
@ -242,7 +242,7 @@ def parse_medium_rss_feed(rss_file):
'url': url,
'domain': domain(url),
'base_url': base_url(url),
'timestamp': str(time.timestamp()),
'timestamp': str(datetime.now().timestamp()),
'tags': '',
'title': title or fetch_page_title(url),
'sources': [rss_file.name],
@ -261,13 +261,11 @@ def parse_plain_text(text_file):
urls = re.findall(URL_REGEX, line)
for url in urls:
timestamp = str(datetime.now().timestamp())
info = {
'url': url,
'domain': domain(url),
'base_url': base_url(url),
'timestamp': timestamp,
'timestamp': str(datetime.now().timestamp()),
'tags': '',
'title': fetch_page_title(url),
'sources': [text_file.name],