diff --git a/tools/buttfeed.py b/tools/buttfeed.py index 69a8b81a..f27b0a89 100755 --- a/tools/buttfeed.py +++ b/tools/buttfeed.py @@ -25,6 +25,7 @@ def fix_title(entry): return entry.title.split('\n')[0] def get_entries(): + seen = set() results = [] for name, url in k_feeds.items(): feed = feedparser.parse(url) @@ -41,9 +42,13 @@ def get_entries(): continue if entry.summary.startswith('.*?$\s*^([^\n]+)$', entry.summary, re.S | re.M): - results.append((time.mktime(entry.get('updated_parsed')), name, m[0], m[1])) + if not m[0] in seen: + seen.add(m[0]) + results.append((time.mktime(entry.get('updated_parsed')), name, m[0], m[1])) else: - results.append((time.mktime(entry.get('updated_parsed')), name, entry.link, entry.title.split('\n')[0])) + if not entry.link in seen: + seen.add(entry.link) + results.append((time.mktime(entry.get('updated_parsed')), name, entry.link, entry.title.split('\n')[0])) results.sort() results.reverse() return results