This newsreader thing can work, but it still has a long ways to go.

git-svn-id: https://www.unprompted.com/svn/projects/tildefriends/trunk@3373 ed5197a5-7fde-0310-b194-c3ffbd925b24

packages/cory/liblist/liblist.js

@@ -15,15 +15,15 @@ class DatabaseList {
 		} if (listNode) {
 			listNode = JSON.parse(listNode);
 			listNode.count++;
-			let id = desiredKey;
+			let id = this._prefix + ":node:" + desiredKey;
-			if (id && await database.get(this._prefix + ":node:" + id.toString())) {
+			if (desiredKey && await database.get(id)) {
 				throw new Error("Key '" + desiredKey + "' already exists.");
 			}
-			if (!id) {
+			if (!desiredKey) {
-				id = listNode.nextId++;
+				id = this._prefix + ":node:" + listNode.nextId++;
 			}
 			if (end) {
-				let newKey = this._prefix + ":node:" + id.toString();
+				let newKey = id;
 				await database.set(newKey, JSON.stringify({next: key, previous: listNode.previous, value: item}));
 
 				if (listNode.previous !== key) {
@@ -39,10 +39,10 @@ class DatabaseList {
 					await database.set(key, JSON.stringify(listNode));
 				}
 			} else {
-				let newKey = listNode.key || id.toString();
+				let newKey = listNode.key ? (this._prefix + ":node:" + listNode.key) : id;
 				await database.set(newKey, JSON.stringify({next: listNode.next, previous: key, value: listNode.value}));
 				listNode.value = item;
-				listNode.key = id;
+				listNode.key = desiredKey;
 
 				if (listNode.next !== key) {
 					let next = JSON.parse(await database.get(listNode.next));
@@ -178,7 +178,7 @@ class DatabaseList {
 	}
 
 	async getByKey(key) {
-		let value = await database.get(this._prefix + ":node:" + key.toString());
+		let value = await database.get(this._prefix + (key ? ":node:" + key.toString() : ":head"));
 		if (value !== undefined) {
 			value = JSON.parse(value);
 		} else {

packages/cory/news/news.js

@@ -1,6 +1,12 @@
 "use strict";
 
-//! {"require": ["libhttp", "liblist", "libxml"], "permissions": ["network"]}
+//! {"require": ["libencoding", "libhttp", "liblist", "libxml"], "permissions": ["network"]}
+
+/*
+	list news<url>:id {title, description, guid || link}
+	list users:username {subscriptions: []}
+	list feed:username,url {id, title, modified, read, ...}
+*/
 
 let http = require("libhttp");
 let liblist = require("liblist");
@@ -24,6 +30,22 @@ function parseNews(response) {
 					}
 				}
 			}
+		} else if (node0.name == "feed") {
+			for (let node1 of node0.children) {
+				if (node1.name == "entry") {
+					let item = {};
+					for (let node2 of node1.children) {
+						if (node2.name == "title") {
+							item.title = node2.text;
+						} else if (node2.name == "link") {
+							item.link = node2.attributes.href;
+						} else if (node2.name == "content" && node2.attributes.type == "html") {
+							item.description = node2.text;
+						}
+					}
+					news.items.push(item);
+				}
+			}
 		}
 	}
 	return news;
@@ -44,35 +66,259 @@ async function fetchNews(url) {
 }
 
 async function storeNews(url, news) {
-	let listStore = liblist.ListStore(url);
 	for (let item of news.items) {
 		let id = item.guid || item.link;
-		if (await listStore.getByKey(id) !== undefined) {
+		await transformListItem(url, id, entry => item);
-			await listStore.setByKey(id, item);
+	}
-			terminal.print("SET ", id);
+
-		} else {
+	let users = await liblist.ListStore("users").get(0, Number.MAX_SAFE_INTEGER);
-			await listStore.push(item, id);
+	terminal.print("users: ", JSON.stringify(users));
-			terminal.print("PUSH ", id);
+	for (let user of users) {
+		if (user.subscriptions.indexOf(url) != -1) {
+			terminal.print("USER!", user.name);
+			for (let item of news.items) {
+				let id = item.guid || item.link;
+				terminal.print("storing news in ", "feed:" + JSON.stringify([user.name, url]), " ", id);
+				await transformListItem("feed:" + JSON.stringify([user.name, url]), id, entry => {
+					entry = entry || {url: url, id: id, title: item.title};
+					if (item.title != entry.title || !entry.modified) {
+						entry.title = item.title;
+						entry.modified = new Date();
+						entry.read = false;
+					}
+					return entry;
+				});
+			}
 		}
 	}
 }
 
-function loadNews(url) {
+async function setRead(url, id, read) {
-	return liblist.ListStore(url).get(0, 10);
+	await transformListItem("feed:" + JSON.stringify([core.user.name, url]), id, entry => {
+		entry.read = read;
+		return entry;
+	});
 }
 
-async function test(url) {
+const kUrl = "https://www.reddit.com/.rss?feed=d05b33887cf432fd6a28c39acfb1d645bcd5e69b&user=unprompted";
+
+async function transformListItem(list, key, callback, back) {
+	let listStore = liblist.ListStore(list);
+	let value = await listStore.getByKey(key);
+	let have = value !== undefined;
+	value = callback(value !== undefined ? value.value : undefined);
+	if (have) {
+		await listStore.setByKey(key, value);
+	} else {
+		if (back) {
+			await listStore.unshift(value, key);
+		} else {
+			await listStore.push(value, key);
+		}
+	}
+	return value;
+}
+
+async function getAllSubscriptions() {
+	let urls = new Set();
+	let users = await liblist.ListStore("users").get(0, Number.MAX_SAFE_INTEGER);
+	terminal.print("users", JSON.stringify(users));
+	for (let user of users) {
+		terminal.print(JSON.stringify(user));
+		if (user && user.subscriptions) {
+			for (let url of user.subscriptions) {
+				urls.add(url);
+			}
+		}
+	}
+	return Array.from(urls);
+}
+
+async function getMySubscriptions() {
+	let urls = new Set();
+	let value = await liblist.ListStore("users").getByKey(core.user.name);
+	return value ? value.value.subscriptions : [];
+}
+
+async function subscribe(url) {
+	let entry = await transformListItem("users", core.user.name, user => {
+		user = user || {name: core.user.name, subscriptions: []};
+		if (user.subscriptions.indexOf(url) == -1) {
+			user.subscriptions.push(url);
+		}
+		return user;
+	});
+	return entry.subscriptions;
+}
+
+class TestInterface {
+	async fetchNews() {
+		try {
+			terminal.print("fetching");
+			let urls = await getMySubscriptions();
+			terminal.print("subscriptions: ", JSON.stringify(urls));
+			for (let url of urls) {
+				try {
+					terminal.print("fetch", url);
+					let response = await fetchNews(url);
+					terminal.print("parse");
+					let news = parseNews(response);
+					terminal.print("store", JSON.stringify(news).substring(0, 1024));
+					await storeNews(url, news);
+					terminal.print("done");
+				} catch (error) {
+					terminal.print("error", error);
+				}
+			}
+		} catch (error) {
+			terminal.print("error", error);
+		}
+	}
+
+	async aggregate(urls) {
+		let news = [];
+		for (let url of urls) {
+			news = news.concat(await liblist.ListStore("feed:" + JSON.stringify([core.user.name, url])).get(0, 100));
+		}
+		return news.sort((x, y) => y.modified.localeCompare(x.modified)).slice(0, 100);
+	}
+
+	async refreshNews() {
+		this.selectedIndex = -1;
+		terminal.select("headlines");
+		terminal.clear();
+		terminal.print("Loading...");
+		let subscriptions = await getMySubscriptions();
+		this.news = await this.aggregate(subscriptions);
+	}
+
+	async redisplay() {
+		terminal.cork();
+		try {
+			terminal.select("headlines");
+			terminal.clear();
+			this.news.forEach((article, index) => {
+				let color = "";
+				if (this.selectedIndex == index) {
+					color = "red";
+				} else if (article.read) {
+					color = "gray";
+				}
+				terminal.print(article.modified.toString(), " ", {
+					style: color ? ("color: " + color) : "",
+					value: article.title,
+				});
+			});
+			terminal.select("view");
+			terminal.clear();
+			if (this.news[this.selectedIndex]) {
+				let fullArticle = (await liblist.ListStore(this.news[this.selectedIndex].url).getByKey(this.news[this.selectedIndex].id)).value;
+				terminal.print({
+					iframe: `<h1>${fullArticle.title}</h1>${fullArticle.description}`,
+					style: "background-color: #fff; border: 0; margin: 0; padding: 0; flex: 1 1 auto",
+					width: null,
+					height: null,
+				});
+			}
+		} finally {
+			terminal.uncork();
+		}
+	}
+
+	async moveSelection(delta) {
+		this.selectedIndex += delta;
+		try {
+			let item = this.news[this.selectedIndex];
+			if (item) {
+				await setRead(item.url, item.id, true);
+				item.read = true;
+			}
+		} catch (error) {
+			print("error", error);
+		}
+		this.redisplay();
+	}
+
+	async activate() {
+		let self = this;
+		terminal.split([
+			{name: "headlines", basis: "30%", grow: 0, shrink: 1},
+			{name: "view", style: "display: flex", basis: "70%", grow: 2, shrink: 0},
+		]);
+		self.refreshNews().then(self.redisplay.bind(self)).catch(terminal.print);
+		terminal.setSendKeyEvents(true);
+		core.register("key", async function(event) {
+			if (event.type == "keypress") {
+				switch (event.keyCode) {
+					case 'j'.charCodeAt(0):
+						self.moveSelection(1);
+						break;
+					case 'k'.charCodeAt(0):
+						self.moveSelection(-1);
+						break;
+					case 'r'.charCodeAt(0):
+						await self.fetchNews();
+						await self.refreshNews();
+						self.redisplay();
+						break;
+					case 'R'.charCodeAt(0):
+						await self.fetchNews();
+						break;
+					case 'd'.charCodeAt(0):
+						self.redisplay();
+						break;
+				}
+			}
+		});
+	}
+}
+
+core.register("onInput", async function(input) {
+	try {
+		if (input == "wipe") {
+			await wipeDatabase();
+			terminal.print("database wiped");
+		} else if (input == "dump") {
+			await dumpDatabase();
+		} else if (input.startsWith("subscribe ")) {
+			let subscriptions = await subscribe(input.substring("subscribe ".length));
+			terminal.print("subscriptions: ", JSON.stringify(subscriptions));
+		}
+	} catch (error) {
+		terminal.print("error", error);
+	}
+});
+
+/*
+async function test() {
 	await wipeDatabase();
-	await dumpDatabase();
+	let l = liblist.ListStore("test");
-	let response = await fetchNews(url);
+	await transformListItem("test", "a", x => "value:a");
-	let news = parseNews(response);
+	await transformListItem("test", "b", x => "value:b");
-	await storeNews(url, news);
+	await transformListItem("test", "c", x => "value:c");
-	await storeNews(url, news);
+	terminal.print("contents: ", JSON.stringify(await l.get(0, 2)));
-	terminal.print("That's the news for today:");
+	await transformListItem("test", "d", x => "value:d");
-	terminal.print(JSON.stringify(await loadNews(url), 0, 2));
+	terminal.print("contents: ", JSON.stringify(await l.get(0, 2)));
-	terminal.print("Keys:");
+	terminal.print("a?", JSON.stringify(await l.getByKey("a")));
-	terminal.print(JSON.stringify(await database.getAll(), 0, 2));
+	terminal.print("b?", JSON.stringify(await l.getByKey("b")));
-	await dumpDatabase();
+	terminal.print("c?", JSON.stringify(await l.getByKey("c")));
+	terminal.print("d?", JSON.stringify(await l.getByKey("d")));
+	dumpDatabase();
+	await wipeDatabase();
+	await transformListItem("test", "a", x => "value:a", true);
+	await transformListItem("test", "b", x => "value:b", true);
+	await transformListItem("test", "c", x => "value:c", true);
+	terminal.print("contents: ", JSON.stringify(await l.get(0, 2)));
+	await transformListItem("test", "d", x => "value:d", true);
+	terminal.print("contents: ", JSON.stringify(await l.get(0, 2)));
+	terminal.print("a?", JSON.stringify(await l.getByKey("a")));
+	terminal.print("b?", JSON.stringify(await l.getByKey("b")));
+	terminal.print("c?", JSON.stringify(await l.getByKey("c")));
+	terminal.print("d?", JSON.stringify(await l.getByKey("d")));
+	dumpDatabase();
 }
+test().catch(terminal.print);
+//*/
 
-test("http://www.unprompted.com/rss").catch(terminal.print);
+//test("http://www.unprompted.com/rss").catch(terminal.print);
+new TestInterface().activate().catch(terminal.print);