From b5c6d34860d1303c696867de12750deac65eb757 Mon Sep 17 00:00:00 2001 From: Chris Dzombak Date: Tue, 13 Mar 2018 21:11:11 -0400 Subject: [PATCH 1/3] Add support for Pinboard RSS feeds --- parse.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/parse.py b/parse.py index d5eda177..afa31341 100644 --- a/parse.py +++ b/parse.py @@ -17,6 +17,7 @@ Parsed link schema: { import re import json +import xml.etree.ElementTree as etree from datetime import datetime @@ -36,6 +37,7 @@ def get_parsers(file): 'pinboard': parse_json_export, 'bookmarks': parse_bookmarks_export, 'rss': parse_rss_export, + 'pinboard_rss': parse_pinboard_rss_feed, } def parse_links(path): @@ -169,3 +171,31 @@ def parse_bookmarks_export(html_file): info['type'] = get_link_type(info) yield info + +def parse_pinboard_rss_feed(rss_file): + """Parse Pinboard RSS feed files into links""" + + rss_file.seek(0) + root = etree.parse(rss_file).getroot() + items = root.findall("{http://purl.org/rss/1.0/}item") + for item in items: + url = item.find("{http://purl.org/rss/1.0/}link").text + tags = item.find("{http://purl.org/dc/elements/1.1/}subject").text + title = item.find("{http://purl.org/rss/1.0/}title").text + ts_str = item.find("{http://purl.org/dc/elements/1.1/}date").text + # Pinboard includes a colon in its date stamp timezone offsets, which + # Python can't parse. Remove it: + if ":" == ts_str[-3:-2]: + ts_str = ts_str[:-3]+ts_str[-2:] + time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z") + info = { + 'url': url, + 'domain': domain(url), + 'base_url': base_url(url), + 'timestamp': str(time.timestamp()), + 'tags': tags, + 'title': title, + 'sources': [rss_file.name], + } + info['type'] = get_link_type(info) + yield info From f49f1b5aa07fa5036e8e5a7c1443b4fecfec1e45 Mon Sep 17 00:00:00 2001 From: Chris Dzombak Date: Tue, 13 Mar 2018 22:23:52 -0400 Subject: [PATCH 2/3] Parse Medium RSS feeds closes https://github.com/pirate/bookmark-archiver/issues/64 --- parse.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/parse.py b/parse.py index afa31341..442efc2d 100644 --- a/parse.py +++ b/parse.py @@ -38,6 +38,7 @@ def get_parsers(file): 'bookmarks': parse_bookmarks_export, 'rss': parse_rss_export, 'pinboard_rss': parse_pinboard_rss_feed, + 'medium_rss': parse_medium_rss_feed, } def parse_links(path): @@ -199,3 +200,28 @@ def parse_pinboard_rss_feed(rss_file): } info['type'] = get_link_type(info) yield info + +def parse_medium_rss_feed(rss_file): + """Parse Medium RSS feed files into links""" + + rss_file.seek(0) + root = etree.parse(rss_file).getroot() + items = root.find("channel").findall("item") + for item in items: + for child in item: + print(child.tag, child.text) + url = item.find("link").text + title = item.find("title").text + ts_str = item.find("pubDate").text + time = datetime.strptime(ts_str, "%a, %d %b %Y %H:%M:%S %Z") + info = { + 'url': url, + 'domain': domain(url), + 'base_url': base_url(url), + 'timestamp': str(time.timestamp()), + 'tags': "", + 'title': title, + 'sources': [rss_file.name], + } + info['type'] = get_link_type(info) + yield info From 0012742d04d5ebb08aa77464e6ffae3be5499235 Mon Sep 17 00:00:00 2001 From: Chris Dzombak Date: Wed, 14 Mar 2018 17:38:39 -0400 Subject: [PATCH 3/3] =?UTF-8?q?Treat=20ParseError=20as=20an=20indication?= =?UTF-8?q?=20that=20a=20parser=20doesn=E2=80=99t=20apply=20to=20the=20inp?= =?UTF-8?q?ut=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parse.py b/parse.py index 442efc2d..49fe201c 100644 --- a/parse.py +++ b/parse.py @@ -52,7 +52,7 @@ def parse_links(path): links += list(parser_func(file)) if links: break - except (ValueError, TypeError, IndexError): + except (ValueError, TypeError, IndexError, etree.ParseError): # parser not supported on this file pass