From 54b5f6154e366e379582611d0c445ac4b40a9d15 Mon Sep 17 00:00:00 2001 From: Cory McWilliams Date: Mon, 22 Aug 2016 14:46:12 +0000 Subject: [PATCH] Adding a number of work-in-progress packages. Some data structures built on top of the key-value store and an http client, among others. git-svn-id: https://www.unprompted.com/svn/projects/tildefriends/trunk@3310 ed5197a5-7fde-0310-b194-c3ffbd925b24 --- packages/cory/await/await.js | 31 +++ packages/cory/blog/blog.js | 19 +- packages/cory/hello/hello.js | 5 + packages/cory/libdocument/libdocument.js | 106 ++++++++ packages/cory/libhttp/libhttp.js | 65 +++++ packages/cory/liblist/liblist.js | 307 +++++++++++++++++++++++ packages/cory/libxml/libxml.js | 254 +++++++++++++++++++ packages/cory/news/news.js | 78 ++++++ 8 files changed, 857 insertions(+), 8 deletions(-) create mode 100644 packages/cory/await/await.js create mode 100644 packages/cory/hello/hello.js create mode 100644 packages/cory/libdocument/libdocument.js create mode 100644 packages/cory/libhttp/libhttp.js create mode 100644 packages/cory/liblist/liblist.js create mode 100644 packages/cory/libxml/libxml.js create mode 100644 packages/cory/news/news.js diff --git a/packages/cory/await/await.js b/packages/cory/await/await.js new file mode 100644 index 00000000..d4937fb1 --- /dev/null +++ b/packages/cory/await/await.js @@ -0,0 +1,31 @@ +"use strict"; + +//! {"category": "tests"} + +async function main() { + terminal.print("Hi. What's your name?"); + let name = await terminal.readLine(); + terminal.print("Hello, " + name + "."); + + let number = Math.floor(Math.random() * 100); + let guesses = 0; + while (true) { + terminal.print("Guess the number."); + try { + let guess = parseInt(await terminal.readLine()); + guesses++; + if (guess < number) { + terminal.print("Too low."); + } else if (guess > number) { + terminal.print("Too high."); + } else { + terminal.print("You got it in " + guesses.toString() + " guesses! It was " + number.toString() + ". Good job, " + name + "."); + break; + } + } catch (error) { + terminal.print(error); + } + } +} + +main().catch(terminal.print); \ No newline at end of file diff --git a/packages/cory/blog/blog.js b/packages/cory/blog/blog.js index 1ca80ae8..5061e00b 100644 --- a/packages/cory/blog/blog.js +++ b/packages/cory/blog/blog.js @@ -50,17 +50,20 @@ class Blog { async submitPost(post) { let now = new Date(); let oldPost = await this._documents.get(post.name); - if (!oldPost) { - post.created = now; - post.author = core.user.name; + if (!await this._list.getByKey(post.name)) { this._list.push(post.name); - } else { - for (let key in oldPost) { - if (!post[key]) { - post[key] = oldPost[key]; - } + } + for (let key in oldPost) { + if (!post[key]) { + post[key] = oldPost[key]; } } + if (!post.created) { + post.created = now; + } + if (!post.author) { + post.author = core.user.name; + } post.modified = now; await this._documents.set(post.name, post); } diff --git a/packages/cory/hello/hello.js b/packages/cory/hello/hello.js new file mode 100644 index 00000000..67eceb64 --- /dev/null +++ b/packages/cory/hello/hello.js @@ -0,0 +1,5 @@ +"use strict"; + +//! {"category": "tests"} + +terminal.print("Hello, world!"); \ No newline at end of file diff --git a/packages/cory/libdocument/libdocument.js b/packages/cory/libdocument/libdocument.js new file mode 100644 index 00000000..02d01b9e --- /dev/null +++ b/packages/cory/libdocument/libdocument.js @@ -0,0 +1,106 @@ +"use strict"; + +// A document store. + +//! {"category": "libraries"} + +class DocumentStore { + constructor(prefix) { + this._prefix = prefix; + } + + async _get(name) { + let node; + try { + node = JSON.parse(await database.get(this._prefix + ":node:" + JSON.stringify(name))); + } catch (error) { + node = {version: null}; + } + return node; + } + + async _addKey(name) { + let list = JSON.parse(await database.get(this._prefix + ":keys") || "[]"); + if (list.indexOf(name) == -1) { + list.push(name); + list.sort(); + } + await database.set(this._prefix + ":keys", JSON.stringify(list)); + } + + async _removeKey(name) { + let list = JSON.parse(await database.get(this._prefix + ":keys") || "[]"); + let index = list.indexOf(name); + if (index != -1) { + list.splice(index, 1); + } + await database.set(this._prefix + ":keys", JSON.stringify(list)); + } + + async set(name, value) { + let node = await this._get(name); + let version = (node.version || 0) + 1; + await database.set(this._prefix + ":version:" + JSON.stringify(name) + ":" + version.toString(), JSON.stringify(value)); + node.deleted = value == undefined; + node.version = version; + await database.set(this._prefix + ":node:" + JSON.stringify(name), JSON.stringify(node)); + if (node.deleted) { + await this._removeKey(name); + } else { + await this._addKey(name); + } + } + + async get(name, version) { + let queryVersion = version || (await this._get(name)).version || 0; + let value = await database.get(this._prefix + ":version:" + JSON.stringify(name) + ":" + queryVersion.toString()); + return value ? JSON.parse(value) : undefined; + } + + async getAll() { + return JSON.parse(await database.get(this._prefix + ":keys") || "[]"); + } + + async setVersion(name, version, value) { + await database.set(this._prefix + ":version:" + JSON.stringify(name) + ":" + version.toString(), JSON.stringify(value)); + } +} + +async function dump() { + terminal.print("Dumping everything."); + let keys = await database.getAll(); + for (let key in keys) { + terminal.print(keys[key], " = ", await database.get(keys[key])); + database.remove(keys[key]); + } +} + +async function test() { + terminal.print("Running a test."); + let ds = new DocumentStore("cory"); + await ds.set("cory", 1); + await ds.set("cory", 2); + await ds.set("cory", 3); + terminal.print((await ds.get("cory")).toString()); + await ds.set("alice", "hello, world!"); + terminal.print(await ds.get("alice")); + terminal.print(JSON.stringify(await ds.getAll())); + await ds.set("cory", null); + terminal.print(JSON.stringify(await ds.getAll())); + terminal.print((await ds.get("cory", 2)).toString()); + terminal.print("Done."); +} + +if (imports.terminal) { + //dump().then(test).then(dump).catch(terminal.print); +} + +exports.DocumentStore = function(name) { + let ds = new DocumentStore(name); + return { + get: ds.get.bind(ds), + set: ds.set.bind(ds), + getAll: ds.getAll.bind(ds), + setVersion: ds.setVersion.bind(ds), + }; +} \ No newline at end of file diff --git a/packages/cory/libhttp/libhttp.js b/packages/cory/libhttp/libhttp.js new file mode 100644 index 00000000..cac80b93 --- /dev/null +++ b/packages/cory/libhttp/libhttp.js @@ -0,0 +1,65 @@ +"use strict"; + +//! {"permissions": ["network"]} + +function parseUrl(url) { + // XXX: Hack. + var match = url.match(new RegExp("(\\w+)://([^/]+)?(.*)")); + return { + protocol: match[1], + host: match[2], + path: match[3], + port: match[1] == "http" ? 80 : 443, + }; +} + +function parseResponse(data) { + var firstLine; + var headers = {}; + + while (true) { + var endLine = data.indexOf("\r\n"); + var line = data.substring(0, endLine); + if (!firstLine) { + firstLine = line; + } else if (!line.length) { + break; + } else { + var colon = line.indexOf(":"); + headers[line.substring(0, colon).toLowerCase()] = line.substring(colon + 1).trim(); + } + data = data.substring(endLine + 2); + } + return {body: data, headers: headers}; +} + +function get(url) { + return new Promise(async function(resolve, reject) { + try { + let parsed = parseUrl(url); + let buffer = ""; + + let socket = await network.newConnection(); + + await socket.connect(parsed.host, parsed.port); + socket.read(function(data) { + if (data) { + buffer += data; + } else { + resolve(parseResponse(buffer)); + } + }); + + if (parsed.port == 443) { + await socket.startTls(); + } + + socket.write(`GET ${parsed.path} HTTP/1.0\r\nHost: ${parsed.host}\r\nConnection: close\r\n\r\n`); + //socket.close(); + } catch(error) { + reject(error); + } + }); +} + +exports.get = get; \ No newline at end of file diff --git a/packages/cory/liblist/liblist.js b/packages/cory/liblist/liblist.js new file mode 100644 index 00000000..a72432ea --- /dev/null +++ b/packages/cory/liblist/liblist.js @@ -0,0 +1,307 @@ +//! {"category": "libraries"} + +"use strict"; + +class DatabaseList { + constructor(name) { + this._prefix = name; + } + + async _insert(item, end, desiredKey) { + let key = this._prefix + ":head"; + let listNode = await database.get(key); + if (!listNode) { + await database.set(key, JSON.stringify({next: key, previous: key, value: item, count: 1, nextId: 1, key: desiredKey})); + } if (listNode) { + listNode = JSON.parse(listNode); + listNode.count++; + let id = desiredKey; + if (id && await database.get(this._prefix + ":node:" + id.toString())) { + throw new Error("Key '" + desiredKey + "' already exists."); + } + if (!id) { + id = listNode.nextId++; + } + if (end) { + let newKey = this._prefix + ":node:" + id.toString(); + await database.set(newKey, JSON.stringify({next: key, previous: listNode.previous, value: item})); + + if (listNode.previous !== key) { + let previous = JSON.parse(await database.get(listNode.previous)); + previous.next = newKey; + await database.set(listNode.previous, JSON.stringify(previous)); + + listNode.previous = newKey; + await database.set(key, JSON.stringify(listNode)); + } else { + listNode.previous = newKey; + listNode.next = newKey; + await database.set(key, JSON.stringify(listNode)); + } + } else { + let newKey = listNode.key || id.toString(); + await database.set(newKey, JSON.stringify({next: listNode.next, previous: key, value: listNode.value})); + listNode.value = item; + listNode.key = id; + + if (listNode.next !== key) { + let next = JSON.parse(await database.get(listNode.next)); + next.previous = newKey; + await database.set(listNode.next, JSON.stringify(next)); + + listNode.next = newKey; + await database.set(key, JSON.stringify(listNode)); + } else { + listNode.previous = newKey; + listNode.next = newKey; + await database.set(key, JSON.stringify(listNode)); + } + } + } + } + + async _remove(end) { + let key = this._prefix + ":head"; + let listNode = await database.get(key); + let result; + if (listNode) { + listNode = JSON.parse(listNode); + listNode.count--; + if (end) { + if (listNode.previous === key) { + await database.remove(key); + result = listNode.value; + } else { + let removeKey = listNode.previous; + let previous = JSON.parse(await database.get(listNode.previous)); + result = previous.value; + if (previous.previous !== key) { + let previousPrevious = JSON.parse(await database.get(previous.previous)); + previousPrevious.next = key; + listNode.previous = previous.previous; + await database.set(previous.previous, JSON.stringify(previousPrevious)); + await database.set(key, JSON.stringify(listNode)); + await database.remove(removeKey); + } else { + listNode.next = key; + listNode.previous = key; + await database.set(key, JSON.stringify(listNode)); + await database.remove(removeKey); + } + } + } else { + result = listNode.value; + if (listNode.next === key) { + await database.remove(key); + } else { + let removeKey = listNode.next; + let next = JSON.parse(await database.get(listNode.next)); + listNode.value = next.value; + if (next.next !== key) { + let nextNext = JSON.parse(await database.get(next.next)); + nextNext.previous = key; + listNode.next = next.next; + await database.set(next.next, JSON.stringify(nextNext)); + await database.set(key, JSON.stringify(listNode)); + await database.remove(removeKey); + } else { + listNode.next = key; + listNode.previous = key; + await database.set(key, JSON.stringify(listNode)); + await database.remove(removeKey); + } + } + } + } + return result; + } + + push(item, key) { + return this._insert(item, true, key); + } + + unshift(item, key) { + return this._insert(item, false, key); + } + + pop() { + return this._remove(true); + } + + shift() { + return this._remove(false); + } + + async get(offset, count) { + const head = this._prefix + ":head"; + let key = head; + let result = []; + + while (offset) { + let node = await database.get(key); + if (!node) { + break; + } + node = JSON.parse(node); + if (offset > 0) { + key = node.next; + offset--; + } else if (offset < 0) { + key = node.previous; + offset++; + } + } + + while (count) { + let node = await database.get(key); + if (!node) { + break; + } + node = JSON.parse(node); + result.push(node.value); + if (count > 0) { + key = node.next; + if (key == head) { + break; + } + count--; + } else if (count < 0) { + key = node.previous; + count++; + if (key == head) { + count = -1; + } + } + } + + return result; + } + + async getByKey(key) { + let value = await database.get(this._prefix + ":node:" + key.toString()); + if (value !== undefined) { + value = JSON.parse(value); + } else { + let node = await database.get(this._prefix + ":head"); + if (node !== undefined) { + node = JSON.parse(node); + if (node.key == key) { + value = node; + } + } + } + return value; + } + + async setByKey(key, value) { + let node = await database.get(this._prefix + ":head"); + let done = false; + if (node !== undefined) { + node = JSON.parse(node); + if (node.key == key) { + node.value = value; + await database.set(this._prefix + ":head", JSON.stringify(node)); + done = true; + } + } + + if (!done) { + node = JSON.parse(await database.get(this._prefix + ":node:" + key)); + node.value = value; + await database.set(this._prefix + ":node:" + key, JSON.stringify(node)); + done = true; + } + } +} + +function wipeDatabase() { + let promises = []; + return database.getAll().then(function(list) { + for (let i = 0; i < list.length; i++) { + promises.push(database.remove(list[i])); + } + }); + return Promise.all(promises); +} + +async function dumpDatabase() { + for (let key of await database.getAll()) { + let value = await database.get(key); + try { + value = JSON.parse(value); + } catch (error) { + // eh + } + terminal.print("DUMP: ", key, " ", JSON.stringify(value, 0, 2)); + } +} + +/*async function test() { + let x = new DatabaseList("list"); + await x.push("1"); + await x.push("2"); + await x.push("3"); + await dumpDatabase(); + terminal.print(await x.get(0, 10)); + terminal.print(await x.get(-1, -10)); + terminal.print(await x.pop()); + terminal.print(await x.pop()); + terminal.print(await x.pop()); + await dumpDatabase(); + await x.unshift("1"); + await x.unshift("2"); + await x.unshift("3"); + await dumpDatabase(); + await x.push("cory", "coryKey"); + await x.push("yo", "yoKey"); + terminal.print(await x.get(0, 10)); + terminal.print(await x.shift()); + terminal.print(await x.shift()); + terminal.print(await x.shift()); + await dumpDatabase(); +}*/ + +if (imports.terminal) { + //wipeDatabase(); + //dumpDatabase().then(wipeDatabase).then(test).catch(terminal.print); + /*let x = new DatabaseList("list"); + core.register("onInput", function(input) { + if (input == "clear") { + wipeDatabase().then(function() { + terminal.print("Database is now empty."); + }); + } else if (input.substring(0, "push ".length) == "push ") { + x.push(input.substring("push ".length)).then(dumpDatabase).catch(terminal.print); + } else if (input.substring(0, "unshift ".length) == "unshift ") { + x.unshift(input.substring("unshift ".length)).then(dumpDatabase).catch(terminal.print); + } else if (input == "pop") { + x.pop().then(function(out) { + terminal.print("POPPED: ", out); + }).then(dumpDatabase).catch(terminal.print); + } else if (input == "shift") { + x.shift().then(function(out) { + terminal.print("SHIFTED: ", out); + }).then(dumpDatabase).catch(terminal.print); + } else if (input.substring(0, "get ".length) == "get ") { + let parts = input.split(" "); + x.get(parseInt(parts[1])).then(function(result) { + terminal.print(JSON.stringify(result)) + }).catch(terminal.print); + } else { + dumpDatabase(); + } + });*/ +} + +exports.ListStore = function(name) { + let ls = new DatabaseList(name); + return { + push: ls.push.bind(ls), + pop: ls.pop.bind(ls), + shift: ls.shift.bind(ls), + unshift: ls.unshift.bind(ls), + get: ls.get.bind(ls), + getByKey: ls.getByKey.bind(ls), + setByKey: ls.setByKey.bind(ls), + }; +} \ No newline at end of file diff --git a/packages/cory/libxml/libxml.js b/packages/cory/libxml/libxml.js new file mode 100644 index 00000000..49bae864 --- /dev/null +++ b/packages/cory/libxml/libxml.js @@ -0,0 +1,254 @@ +"use strict"; + +//! { "category": "libraries" } + +function xmlEncode(text) { + return text.replace(/([\&"'<>])/g, function(x, item) { + return {'&': '&', '"': '"', '<': '<', '>': '>', "'": '''}[item]; + }); +} +function xmlDecode(xml) { + return xml.replace(/("|<|>|&|')/g, function(x, item) { + return {'&': '&', '"': '"', '<': '<', '>': '>', ''': "'"}[item]; + }); +} + +function XmlStreamParser() { + this.buffer = ""; + this._parsed = []; + this.reset(); + return this; +} + +XmlStreamParser.kText = "text"; +XmlStreamParser.kElement = "element"; +XmlStreamParser.kEndElement = "endElement"; +XmlStreamParser.kAttributeName = "attributeName"; +XmlStreamParser.kAttributeValue = "attributeValue"; + +XmlStreamParser.prototype.reset = function() { + this._state = XmlStreamParser.kText; + this._attributes = {}; + this._attributeName = ""; + this._attributeValue = ""; + this._attributeEquals = false; + this._attributeQuote = ""; + this._slash = false; + this._value = ""; + this._decl = false; +} + +XmlStreamParser.prototype.parse = function(data) { + this._parsed = []; + + for (var i = 0; i < data.length; i++) { + var c = data.charAt(i); + this.parseCharacter(c); + } + + return this._parsed; +} + +XmlStreamParser.prototype.flush = function() { + var node = {type: this._state}; + if (this._value) { + node.value = xmlDecode(this._value); + } + if (this._attributes.length || this._state == XmlStreamParser.kElement) { + node.attributes = this._attributes; + } + if (this._state != XmlStreamParser.kText || this._value) { + this.emit(node); + } + this.reset(); +} + +XmlStreamParser.prototype.parseCharacter = function(c) { + switch (this._state) { + case XmlStreamParser.kText: + if (c == '<') { + this.flush(); + this._state = XmlStreamParser.kElement; + } else { + this._value += c; + } + break; + case XmlStreamParser.kElement: + case XmlStreamParser.kEndElement: + switch (c) { + case '>': + this.finishElement(); + break; + case '/': + if (!this._value) { + this._state = XmlStreamParser.kEndElement; + } else if (!this._slash) { + this._slash = true; + } else { + this._value += c; + } + break; + case '?': + if (!this._value) { + this._decl = true; + } else { + this._value += '?'; + } + break; + case ' ': + case '\t': + case '\r': + case '\n': + this._state = XmlStreamParser.kAttributeName; + break; + default: + if (this._slash) { + this._slash = false; + this._value += '/'; + } + this._value += c; + break; + } + break; + case XmlStreamParser.kAttributeName: + switch (c) { + case ' ': + case '\t': + case '\r': + case '\n': + if (this._attributeName) { + this._state = XmlStreamParser.kAttributeValue; + } + break; + case '/': + if (!this._slash) { + this._slash = true; + } else { + this._value += '/'; + } + break; + case '=': + this._state = XmlStreamParser.kAttributeValue; + break; + case '>': + if (this._attributeName) { + this._attributes[this._attributeName] = null; + } + this._state = XmlStreamParser.kElement; + this.finishElement(); + break; + default: + this._attributeName += c; + break; + } + break; + case XmlStreamParser.kAttributeValue: + switch (c) { + case ' ': + case '\t': + case '\r': + case '\n': + if (this._attributeValue) { + this._state = XmlStreamParser.kAttributeName; + } + break; + case '"': + case "'": + if (!this._attributeValue && !this._attributeQuote) { + this._attributeQuote = c; + } else if (this._attributeQuote == c) { + this._attributes[this._attributeName] = this._attributeValue; + this._attributeName = ""; + this._attributeValue = ""; + this._attributeQuote = ""; + this._state = XmlStreamParser.kAttributeName; + } else { + this._attributeValue += c; + } + break; + case '>': + this.finishElement(); + break; + default: + this._attributeValue += c; + break; + } + break; + } +} + +XmlStreamParser.prototype.finishElement = function() { + if (this._decl) { + this.reset(); + } else { + var value = this._value; + var slash = this._slash; + this.flush(); + if (slash) { + this._state = XmlStreamParser.kEndElement; + this._value = value; + this.flush(); + } + } + this._state = XmlStreamParser.kText; +} + +XmlStreamParser.prototype.emit = function(node) { + this._parsed.push(node); +} + +function XmlStanzaParser(depth) { + this._depth = depth || 0; + this._parsed = []; + this._stack = []; + this._stream = new XmlStreamParser(); + return this; +} + +XmlStanzaParser.prototype.reset = function() { + this._parsed = []; + this._stack = []; + this._stream.reset(); +} + +XmlStanzaParser.prototype.emit = function(stanza) { + this._parsed.push(stanza); +} + +XmlStanzaParser.prototype.parse = function(data) { + this._parsed = []; + var nodes = this._stream.parse(data); + for (var i = 0; i < nodes.length; i++) { + this.parseNode(nodes[i]); + } + return this._parsed; +} + +XmlStanzaParser.prototype.parseNode = function(node) { + switch (node.type) { + case XmlStreamParser.kElement: + this._stack.push({name: node.value, attributes: node.attributes, children: [], text: ""}); + break; + case XmlStreamParser.kEndElement: + if (this._stack.length == 1 + this._depth) { + this.emit(this._stack.pop()); + } else { + var last = this._stack.pop(); + this._stack[this._stack.length - 1].children.push(last); + } + break; + case XmlStreamParser.kText: + if (this._stack && this._stack.length) { + this._stack[this._stack.length - 1].text += node.value; + } + break; + } +} + +exports.StanzaParser = function(depth) { + let parser = new XmlStanzaParser(depth); + return { + parse: parser.parse.bind(parser), + reset: parser.reset.bind(parser), + }; +} \ No newline at end of file diff --git a/packages/cory/news/news.js b/packages/cory/news/news.js new file mode 100644 index 00000000..b352d7a0 --- /dev/null +++ b/packages/cory/news/news.js @@ -0,0 +1,78 @@ +"use strict"; + +//! {"require": ["libhttp", "liblist", "libxml"], "permissions": ["network"]} + +let http = require("libhttp"); +let liblist = require("liblist"); +let xml = require("libxml"); + +function parseNews(response) { + let news = {items: []}; + let nodes = xml.StanzaParser().parse(response.body); + for (let node0 of nodes) { + if (node0.name == "rss") { + for (let node1 of node0.children) { + if (node1.name == "channel") { + for (let node2 of node1.children) { + if (node2.name == "item") { + let item = {}; + for (let node3 of node2.children) { + item[node3.name] = node3.text; + } + news.items.push(item); + } + } + } + } + } + } + return news; +} + +async function fetchNews(url) { + let response; + let retries = 5; + while (retries--) { + response = await http.get(url); + if (response.headers.location) { + url = response.headers.location; + } else { + break; + } + } + return response; +} + +async function storeNews(url, news) { + let listStore = liblist.ListStore(url); + for (let item of news.items) { + let id = item.guid || item.link; + if (await listStore.getByKey(id) !== undefined) { + await listStore.setByKey(id, item); + terminal.print("SET ", id); + } else { + await listStore.push(item, id); + terminal.print("PUSH ", id); + } + } +} + +function loadNews(url) { + return liblist.ListStore(url).get(0, 10); +} + +async function test(url) { + await wipeDatabase(); + await dumpDatabase(); + let response = await fetchNews(url); + let news = parseNews(response); + await storeNews(url, news); + await storeNews(url, news); + terminal.print("That's the news for today:"); + terminal.print(JSON.stringify(await loadNews(url), 0, 2)); + terminal.print("Keys:"); + terminal.print(JSON.stringify(await database.getAll(), 0, 2)); + await dumpDatabase(); +} + +test("http://www.unprompted.com/rss").catch(terminal.print); \ No newline at end of file