Adding a number of work-in-progress packages. Some data structures built on top of the key-value store and an http client, among others.

git-svn-id: https://www.unprompted.com/svn/projects/tildefriends/trunk@3310 ed5197a5-7fde-0310-b194-c3ffbd925b24
This commit is contained in:
Cory McWilliams 2016-08-22 14:46:12 +00:00
parent 40b0de6c15
commit 54b5f6154e
8 changed files with 857 additions and 8 deletions

View File

@ -0,0 +1,31 @@
"use strict";
//! {"category": "tests"}
async function main() {
terminal.print("Hi. What's your name?");
let name = await terminal.readLine();
terminal.print("Hello, " + name + ".");
let number = Math.floor(Math.random() * 100);
let guesses = 0;
while (true) {
terminal.print("Guess the number.");
try {
let guess = parseInt(await terminal.readLine());
guesses++;
if (guess < number) {
terminal.print("Too low.");
} else if (guess > number) {
terminal.print("Too high.");
} else {
terminal.print("You got it in " + guesses.toString() + " guesses! It was " + number.toString() + ". Good job, " + name + ".");
break;
}
} catch (error) {
terminal.print(error);
}
}
}
main().catch(terminal.print);

View File

@ -50,16 +50,19 @@ class Blog {
async submitPost(post) { async submitPost(post) {
let now = new Date(); let now = new Date();
let oldPost = await this._documents.get(post.name); let oldPost = await this._documents.get(post.name);
if (!oldPost) { if (!await this._list.getByKey(post.name)) {
post.created = now;
post.author = core.user.name;
this._list.push(post.name); this._list.push(post.name);
} else { }
for (let key in oldPost) { for (let key in oldPost) {
if (!post[key]) { if (!post[key]) {
post[key] = oldPost[key]; post[key] = oldPost[key];
} }
} }
if (!post.created) {
post.created = now;
}
if (!post.author) {
post.author = core.user.name;
} }
post.modified = now; post.modified = now;
await this._documents.set(post.name, post); await this._documents.set(post.name, post);

View File

@ -0,0 +1,5 @@
"use strict";
//! {"category": "tests"}
terminal.print("Hello, world!");

View File

@ -0,0 +1,106 @@
"use strict";
// A document store.
//! {"category": "libraries"}
class DocumentStore {
constructor(prefix) {
this._prefix = prefix;
}
async _get(name) {
let node;
try {
node = JSON.parse(await database.get(this._prefix + ":node:" + JSON.stringify(name)));
} catch (error) {
node = {version: null};
}
return node;
}
async _addKey(name) {
let list = JSON.parse(await database.get(this._prefix + ":keys") || "[]");
if (list.indexOf(name) == -1) {
list.push(name);
list.sort();
}
await database.set(this._prefix + ":keys", JSON.stringify(list));
}
async _removeKey(name) {
let list = JSON.parse(await database.get(this._prefix + ":keys") || "[]");
let index = list.indexOf(name);
if (index != -1) {
list.splice(index, 1);
}
await database.set(this._prefix + ":keys", JSON.stringify(list));
}
async set(name, value) {
let node = await this._get(name);
let version = (node.version || 0) + 1;
await database.set(this._prefix + ":version:" + JSON.stringify(name) + ":" + version.toString(), JSON.stringify(value));
node.deleted = value == undefined;
node.version = version;
await database.set(this._prefix + ":node:" + JSON.stringify(name), JSON.stringify(node));
if (node.deleted) {
await this._removeKey(name);
} else {
await this._addKey(name);
}
}
async get(name, version) {
let queryVersion = version || (await this._get(name)).version || 0;
let value = await database.get(this._prefix + ":version:" + JSON.stringify(name) + ":" + queryVersion.toString());
return value ? JSON.parse(value) : undefined;
}
async getAll() {
return JSON.parse(await database.get(this._prefix + ":keys") || "[]");
}
async setVersion(name, version, value) {
await database.set(this._prefix + ":version:" + JSON.stringify(name) + ":" + version.toString(), JSON.stringify(value));
}
}
async function dump() {
terminal.print("Dumping everything.");
let keys = await database.getAll();
for (let key in keys) {
terminal.print(keys[key], " = ", await database.get(keys[key]));
database.remove(keys[key]);
}
}
async function test() {
terminal.print("Running a test.");
let ds = new DocumentStore("cory");
await ds.set("cory", 1);
await ds.set("cory", 2);
await ds.set("cory", 3);
terminal.print((await ds.get("cory")).toString());
await ds.set("alice", "hello, world!");
terminal.print(await ds.get("alice"));
terminal.print(JSON.stringify(await ds.getAll()));
await ds.set("cory", null);
terminal.print(JSON.stringify(await ds.getAll()));
terminal.print((await ds.get("cory", 2)).toString());
terminal.print("Done.");
}
if (imports.terminal) {
//dump().then(test).then(dump).catch(terminal.print);
}
exports.DocumentStore = function(name) {
let ds = new DocumentStore(name);
return {
get: ds.get.bind(ds),
set: ds.set.bind(ds),
getAll: ds.getAll.bind(ds),
setVersion: ds.setVersion.bind(ds),
};
}

View File

@ -0,0 +1,65 @@
"use strict";
//! {"permissions": ["network"]}
function parseUrl(url) {
// XXX: Hack.
var match = url.match(new RegExp("(\\w+)://([^/]+)?(.*)"));
return {
protocol: match[1],
host: match[2],
path: match[3],
port: match[1] == "http" ? 80 : 443,
};
}
function parseResponse(data) {
var firstLine;
var headers = {};
while (true) {
var endLine = data.indexOf("\r\n");
var line = data.substring(0, endLine);
if (!firstLine) {
firstLine = line;
} else if (!line.length) {
break;
} else {
var colon = line.indexOf(":");
headers[line.substring(0, colon).toLowerCase()] = line.substring(colon + 1).trim();
}
data = data.substring(endLine + 2);
}
return {body: data, headers: headers};
}
function get(url) {
return new Promise(async function(resolve, reject) {
try {
let parsed = parseUrl(url);
let buffer = "";
let socket = await network.newConnection();
await socket.connect(parsed.host, parsed.port);
socket.read(function(data) {
if (data) {
buffer += data;
} else {
resolve(parseResponse(buffer));
}
});
if (parsed.port == 443) {
await socket.startTls();
}
socket.write(`GET ${parsed.path} HTTP/1.0\r\nHost: ${parsed.host}\r\nConnection: close\r\n\r\n`);
//socket.close();
} catch(error) {
reject(error);
}
});
}
exports.get = get;

View File

@ -0,0 +1,307 @@
//! {"category": "libraries"}
"use strict";
class DatabaseList {
constructor(name) {
this._prefix = name;
}
async _insert(item, end, desiredKey) {
let key = this._prefix + ":head";
let listNode = await database.get(key);
if (!listNode) {
await database.set(key, JSON.stringify({next: key, previous: key, value: item, count: 1, nextId: 1, key: desiredKey}));
} if (listNode) {
listNode = JSON.parse(listNode);
listNode.count++;
let id = desiredKey;
if (id && await database.get(this._prefix + ":node:" + id.toString())) {
throw new Error("Key '" + desiredKey + "' already exists.");
}
if (!id) {
id = listNode.nextId++;
}
if (end) {
let newKey = this._prefix + ":node:" + id.toString();
await database.set(newKey, JSON.stringify({next: key, previous: listNode.previous, value: item}));
if (listNode.previous !== key) {
let previous = JSON.parse(await database.get(listNode.previous));
previous.next = newKey;
await database.set(listNode.previous, JSON.stringify(previous));
listNode.previous = newKey;
await database.set(key, JSON.stringify(listNode));
} else {
listNode.previous = newKey;
listNode.next = newKey;
await database.set(key, JSON.stringify(listNode));
}
} else {
let newKey = listNode.key || id.toString();
await database.set(newKey, JSON.stringify({next: listNode.next, previous: key, value: listNode.value}));
listNode.value = item;
listNode.key = id;
if (listNode.next !== key) {
let next = JSON.parse(await database.get(listNode.next));
next.previous = newKey;
await database.set(listNode.next, JSON.stringify(next));
listNode.next = newKey;
await database.set(key, JSON.stringify(listNode));
} else {
listNode.previous = newKey;
listNode.next = newKey;
await database.set(key, JSON.stringify(listNode));
}
}
}
}
async _remove(end) {
let key = this._prefix + ":head";
let listNode = await database.get(key);
let result;
if (listNode) {
listNode = JSON.parse(listNode);
listNode.count--;
if (end) {
if (listNode.previous === key) {
await database.remove(key);
result = listNode.value;
} else {
let removeKey = listNode.previous;
let previous = JSON.parse(await database.get(listNode.previous));
result = previous.value;
if (previous.previous !== key) {
let previousPrevious = JSON.parse(await database.get(previous.previous));
previousPrevious.next = key;
listNode.previous = previous.previous;
await database.set(previous.previous, JSON.stringify(previousPrevious));
await database.set(key, JSON.stringify(listNode));
await database.remove(removeKey);
} else {
listNode.next = key;
listNode.previous = key;
await database.set(key, JSON.stringify(listNode));
await database.remove(removeKey);
}
}
} else {
result = listNode.value;
if (listNode.next === key) {
await database.remove(key);
} else {
let removeKey = listNode.next;
let next = JSON.parse(await database.get(listNode.next));
listNode.value = next.value;
if (next.next !== key) {
let nextNext = JSON.parse(await database.get(next.next));
nextNext.previous = key;
listNode.next = next.next;
await database.set(next.next, JSON.stringify(nextNext));
await database.set(key, JSON.stringify(listNode));
await database.remove(removeKey);
} else {
listNode.next = key;
listNode.previous = key;
await database.set(key, JSON.stringify(listNode));
await database.remove(removeKey);
}
}
}
}
return result;
}
push(item, key) {
return this._insert(item, true, key);
}
unshift(item, key) {
return this._insert(item, false, key);
}
pop() {
return this._remove(true);
}
shift() {
return this._remove(false);
}
async get(offset, count) {
const head = this._prefix + ":head";
let key = head;
let result = [];
while (offset) {
let node = await database.get(key);
if (!node) {
break;
}
node = JSON.parse(node);
if (offset > 0) {
key = node.next;
offset--;
} else if (offset < 0) {
key = node.previous;
offset++;
}
}
while (count) {
let node = await database.get(key);
if (!node) {
break;
}
node = JSON.parse(node);
result.push(node.value);
if (count > 0) {
key = node.next;
if (key == head) {
break;
}
count--;
} else if (count < 0) {
key = node.previous;
count++;
if (key == head) {
count = -1;
}
}
}
return result;
}
async getByKey(key) {
let value = await database.get(this._prefix + ":node:" + key.toString());
if (value !== undefined) {
value = JSON.parse(value);
} else {
let node = await database.get(this._prefix + ":head");
if (node !== undefined) {
node = JSON.parse(node);
if (node.key == key) {
value = node;
}
}
}
return value;
}
async setByKey(key, value) {
let node = await database.get(this._prefix + ":head");
let done = false;
if (node !== undefined) {
node = JSON.parse(node);
if (node.key == key) {
node.value = value;
await database.set(this._prefix + ":head", JSON.stringify(node));
done = true;
}
}
if (!done) {
node = JSON.parse(await database.get(this._prefix + ":node:" + key));
node.value = value;
await database.set(this._prefix + ":node:" + key, JSON.stringify(node));
done = true;
}
}
}
function wipeDatabase() {
let promises = [];
return database.getAll().then(function(list) {
for (let i = 0; i < list.length; i++) {
promises.push(database.remove(list[i]));
}
});
return Promise.all(promises);
}
async function dumpDatabase() {
for (let key of await database.getAll()) {
let value = await database.get(key);
try {
value = JSON.parse(value);
} catch (error) {
// eh
}
terminal.print("DUMP: ", key, " ", JSON.stringify(value, 0, 2));
}
}
/*async function test() {
let x = new DatabaseList("list");
await x.push("1");
await x.push("2");
await x.push("3");
await dumpDatabase();
terminal.print(await x.get(0, 10));
terminal.print(await x.get(-1, -10));
terminal.print(await x.pop());
terminal.print(await x.pop());
terminal.print(await x.pop());
await dumpDatabase();
await x.unshift("1");
await x.unshift("2");
await x.unshift("3");
await dumpDatabase();
await x.push("cory", "coryKey");
await x.push("yo", "yoKey");
terminal.print(await x.get(0, 10));
terminal.print(await x.shift());
terminal.print(await x.shift());
terminal.print(await x.shift());
await dumpDatabase();
}*/
if (imports.terminal) {
//wipeDatabase();
//dumpDatabase().then(wipeDatabase).then(test).catch(terminal.print);
/*let x = new DatabaseList("list");
core.register("onInput", function(input) {
if (input == "clear") {
wipeDatabase().then(function() {
terminal.print("Database is now empty.");
});
} else if (input.substring(0, "push ".length) == "push ") {
x.push(input.substring("push ".length)).then(dumpDatabase).catch(terminal.print);
} else if (input.substring(0, "unshift ".length) == "unshift ") {
x.unshift(input.substring("unshift ".length)).then(dumpDatabase).catch(terminal.print);
} else if (input == "pop") {
x.pop().then(function(out) {
terminal.print("POPPED: ", out);
}).then(dumpDatabase).catch(terminal.print);
} else if (input == "shift") {
x.shift().then(function(out) {
terminal.print("SHIFTED: ", out);
}).then(dumpDatabase).catch(terminal.print);
} else if (input.substring(0, "get ".length) == "get ") {
let parts = input.split(" ");
x.get(parseInt(parts[1])).then(function(result) {
terminal.print(JSON.stringify(result))
}).catch(terminal.print);
} else {
dumpDatabase();
}
});*/
}
exports.ListStore = function(name) {
let ls = new DatabaseList(name);
return {
push: ls.push.bind(ls),
pop: ls.pop.bind(ls),
shift: ls.shift.bind(ls),
unshift: ls.unshift.bind(ls),
get: ls.get.bind(ls),
getByKey: ls.getByKey.bind(ls),
setByKey: ls.setByKey.bind(ls),
};
}

View File

@ -0,0 +1,254 @@
"use strict";
//! { "category": "libraries" }
function xmlEncode(text) {
return text.replace(/([\&"'<>])/g, function(x, item) {
return {'&': '&amp;', '"': '&quot;', '<': '&lt;', '>': '&gt;', "'": '&apos;'}[item];
});
}
function xmlDecode(xml) {
return xml.replace(/(&quot;|&lt;|&gt;|&amp;|&apos;)/g, function(x, item) {
return {'&amp;': '&', '&quot;': '"', '&lt;': '<', '&gt;': '>', '&apos;': "'"}[item];
});
}
function XmlStreamParser() {
this.buffer = "";
this._parsed = [];
this.reset();
return this;
}
XmlStreamParser.kText = "text";
XmlStreamParser.kElement = "element";
XmlStreamParser.kEndElement = "endElement";
XmlStreamParser.kAttributeName = "attributeName";
XmlStreamParser.kAttributeValue = "attributeValue";
XmlStreamParser.prototype.reset = function() {
this._state = XmlStreamParser.kText;
this._attributes = {};
this._attributeName = "";
this._attributeValue = "";
this._attributeEquals = false;
this._attributeQuote = "";
this._slash = false;
this._value = "";
this._decl = false;
}
XmlStreamParser.prototype.parse = function(data) {
this._parsed = [];
for (var i = 0; i < data.length; i++) {
var c = data.charAt(i);
this.parseCharacter(c);
}
return this._parsed;
}
XmlStreamParser.prototype.flush = function() {
var node = {type: this._state};
if (this._value) {
node.value = xmlDecode(this._value);
}
if (this._attributes.length || this._state == XmlStreamParser.kElement) {
node.attributes = this._attributes;
}
if (this._state != XmlStreamParser.kText || this._value) {
this.emit(node);
}
this.reset();
}
XmlStreamParser.prototype.parseCharacter = function(c) {
switch (this._state) {
case XmlStreamParser.kText:
if (c == '<') {
this.flush();
this._state = XmlStreamParser.kElement;
} else {
this._value += c;
}
break;
case XmlStreamParser.kElement:
case XmlStreamParser.kEndElement:
switch (c) {
case '>':
this.finishElement();
break;
case '/':
if (!this._value) {
this._state = XmlStreamParser.kEndElement;
} else if (!this._slash) {
this._slash = true;
} else {
this._value += c;
}
break;
case '?':
if (!this._value) {
this._decl = true;
} else {
this._value += '?';
}
break;
case ' ':
case '\t':
case '\r':
case '\n':
this._state = XmlStreamParser.kAttributeName;
break;
default:
if (this._slash) {
this._slash = false;
this._value += '/';
}
this._value += c;
break;
}
break;
case XmlStreamParser.kAttributeName:
switch (c) {
case ' ':
case '\t':
case '\r':
case '\n':
if (this._attributeName) {
this._state = XmlStreamParser.kAttributeValue;
}
break;
case '/':
if (!this._slash) {
this._slash = true;
} else {
this._value += '/';
}
break;
case '=':
this._state = XmlStreamParser.kAttributeValue;
break;
case '>':
if (this._attributeName) {
this._attributes[this._attributeName] = null;
}
this._state = XmlStreamParser.kElement;
this.finishElement();
break;
default:
this._attributeName += c;
break;
}
break;
case XmlStreamParser.kAttributeValue:
switch (c) {
case ' ':
case '\t':
case '\r':
case '\n':
if (this._attributeValue) {
this._state = XmlStreamParser.kAttributeName;
}
break;
case '"':
case "'":
if (!this._attributeValue && !this._attributeQuote) {
this._attributeQuote = c;
} else if (this._attributeQuote == c) {
this._attributes[this._attributeName] = this._attributeValue;
this._attributeName = "";
this._attributeValue = "";
this._attributeQuote = "";
this._state = XmlStreamParser.kAttributeName;
} else {
this._attributeValue += c;
}
break;
case '>':
this.finishElement();
break;
default:
this._attributeValue += c;
break;
}
break;
}
}
XmlStreamParser.prototype.finishElement = function() {
if (this._decl) {
this.reset();
} else {
var value = this._value;
var slash = this._slash;
this.flush();
if (slash) {
this._state = XmlStreamParser.kEndElement;
this._value = value;
this.flush();
}
}
this._state = XmlStreamParser.kText;
}
XmlStreamParser.prototype.emit = function(node) {
this._parsed.push(node);
}
function XmlStanzaParser(depth) {
this._depth = depth || 0;
this._parsed = [];
this._stack = [];
this._stream = new XmlStreamParser();
return this;
}
XmlStanzaParser.prototype.reset = function() {
this._parsed = [];
this._stack = [];
this._stream.reset();
}
XmlStanzaParser.prototype.emit = function(stanza) {
this._parsed.push(stanza);
}
XmlStanzaParser.prototype.parse = function(data) {
this._parsed = [];
var nodes = this._stream.parse(data);
for (var i = 0; i < nodes.length; i++) {
this.parseNode(nodes[i]);
}
return this._parsed;
}
XmlStanzaParser.prototype.parseNode = function(node) {
switch (node.type) {
case XmlStreamParser.kElement:
this._stack.push({name: node.value, attributes: node.attributes, children: [], text: ""});
break;
case XmlStreamParser.kEndElement:
if (this._stack.length == 1 + this._depth) {
this.emit(this._stack.pop());
} else {
var last = this._stack.pop();
this._stack[this._stack.length - 1].children.push(last);
}
break;
case XmlStreamParser.kText:
if (this._stack && this._stack.length) {
this._stack[this._stack.length - 1].text += node.value;
}
break;
}
}
exports.StanzaParser = function(depth) {
let parser = new XmlStanzaParser(depth);
return {
parse: parser.parse.bind(parser),
reset: parser.reset.bind(parser),
};
}

View File

@ -0,0 +1,78 @@
"use strict";
//! {"require": ["libhttp", "liblist", "libxml"], "permissions": ["network"]}
let http = require("libhttp");
let liblist = require("liblist");
let xml = require("libxml");
function parseNews(response) {
let news = {items: []};
let nodes = xml.StanzaParser().parse(response.body);
for (let node0 of nodes) {
if (node0.name == "rss") {
for (let node1 of node0.children) {
if (node1.name == "channel") {
for (let node2 of node1.children) {
if (node2.name == "item") {
let item = {};
for (let node3 of node2.children) {
item[node3.name] = node3.text;
}
news.items.push(item);
}
}
}
}
}
}
return news;
}
async function fetchNews(url) {
let response;
let retries = 5;
while (retries--) {
response = await http.get(url);
if (response.headers.location) {
url = response.headers.location;
} else {
break;
}
}
return response;
}
async function storeNews(url, news) {
let listStore = liblist.ListStore(url);
for (let item of news.items) {
let id = item.guid || item.link;
if (await listStore.getByKey(id) !== undefined) {
await listStore.setByKey(id, item);
terminal.print("SET ", id);
} else {
await listStore.push(item, id);
terminal.print("PUSH ", id);
}
}
}
function loadNews(url) {
return liblist.ListStore(url).get(0, 10);
}
async function test(url) {
await wipeDatabase();
await dumpDatabase();
let response = await fetchNews(url);
let news = parseNews(response);
await storeNews(url, news);
await storeNews(url, news);
terminal.print("That's the news for today:");
terminal.print(JSON.stringify(await loadNews(url), 0, 2));
terminal.print("Keys:");
terminal.print(JSON.stringify(await database.getAll(), 0, 2));
await dumpDatabase();
}
test("http://www.unprompted.com/rss").catch(terminal.print);