buttfeed: De-duplicate updates by link.

tools/buttfeed.py

@@ -25,6 +25,7 @@ def fix_title(entry):
     return entry.title.split('\n')[0]
 
 def get_entries():
+    seen = set()
     results = []
     for name, url in k_feeds.items():
         feed = feedparser.parse(url)
@@ -41,9 +42,13 @@ def get_entries():
                     continue
             if entry.summary.startswith('<a href='):
                 for m in re.findall(r'<a href="(.*?)">.*?</a>$\s*^([^\n]+)$', entry.summary, re.S | re.M):
-                    results.append((time.mktime(entry.get('updated_parsed')), name, m[0], m[1]))
+                    if not m[0] in seen:
+                        seen.add(m[0])
+                        results.append((time.mktime(entry.get('updated_parsed')), name, m[0], m[1]))
             else:
-                results.append((time.mktime(entry.get('updated_parsed')), name, entry.link, entry.title.split('\n')[0]))
+                if not entry.link in seen:
+                    seen.add(entry.link)
+                    results.append((time.mktime(entry.get('updated_parsed')), name, entry.link, entry.title.split('\n')[0]))
     results.sort()
     results.reverse()
     return results