build: Let's release 0.0.26.

This reverts commit 7f4e2617ee.

GNUmakefile

@@ -16,11 +16,11 @@ MAKEFLAGS += --no-builtin-rules
 ## LD := Linker.
 ## ANDROID_SDK := Path to the Android SDK.
 
-VERSION_CODE := 30
-VERSION_NUMBER := 0.0.25
+VERSION_CODE := 31
+VERSION_NUMBER := 0.0.26
 VERSION_NAME := This program kills fascists.
 
-SQLITE_URL := https://www.sqlite.org/2024/sqlite-amalgamation-3470100.zip
+SQLITE_URL := https://www.sqlite.org/2024/sqlite-amalgamation-3470200.zip
 BUNDLETOOL_URL := https://github.com/google/bundletool/releases/download/1.17.0/bundletool-all-1.17.0.jar
 APPIMAGETOOL_URL := https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-x86_64.AppImage
 APPIMAGETOOL_MD5 := e989fadfc4d685fd3d6aeeb9b525d74d  out/appimagetool

apps/ssb.json

@@ -1,5 +1,5 @@
 {
 	"type": "tildefriends-app",
 	"emoji": "🐌",
-	"previous": "&ksxKqT3Bkp0Z2zV2dQU4ttVZ1k16zdWoJVv6R7m5yAQ=.sha256"
+	"previous": "&q/1uGp0jMvsYGW7Gj8E33kf6UFo/uNYDXg3zo1sVKQg=.sha256"
 }

apps/ssb/commonmark-hashtag.js

@@ -7,7 +7,7 @@ function textNode(text) {
 function linkNode(text, link) {
 	const linkNode = new commonmark.Node('link', undefined);
 	if (link.startsWith('#')) {
-		linkNode.destination = `#q=${encodeURIComponent(link)}`;
+		linkNode.destination = `#${encodeURIComponent('#' + link)}`;
 	} else {
 		linkNode.destination = link;
 	}

apps/ssb/emojis.js

@@ -37,10 +37,12 @@ export async function picker(callback, anchor, author) {
 	div.style.color = '#000';
 	div.style.background = '#fff';
 	div.style.border = '1px solid #000';
-	div.style.display = 'block';
+	div.style.display = 'flex';
 	div.style.overflow = 'scroll';
 	div.style.fontWeight = 'bold';
 	div.style.fontSize = 'xx-large';
+	div.style.flex = '1 1';
+	div.style.flexDirection = 'column';
 	let input = document.createElement('input');
 	input.type = 'text';
 	input.style.display = 'block';
@@ -50,6 +52,7 @@ export async function picker(callback, anchor, author) {
 	input.style.position = 'relative';
 	div.appendChild(input);
 	let list = document.createElement('div');
+	list.style.overflow = 'scroll';
 	div.appendChild(list);
 	div.addEventListener('mousedown', function (event) {
 		event.stopPropagation();
@@ -142,21 +145,40 @@ export async function picker(callback, anchor, author) {
 	}
 	refresh();
 	input.oninput = refresh;
-	let modal = html`
-		<style>
-			${styles}
-		</style>
-		<div class="w3-modal" style="display: block">
-			<div class="w3-modal-content w3-card-4">${div}</div>
-		</div>
-	`;
 	let parent = document.createElement('div');
-	document.body.appendChild(parent);
 	function cleanup() {
 		parent.parentElement.removeChild(parent);
 		window.removeEventListener('keydown', key_down);
 		document.body.removeEventListener('mousedown', cleanup);
 	}
+	let modal = html`
+		<style>
+			${styles}
+		</style>
+		<div
+			class="w3-modal"
+			style="display: block; box-sizing: border-box; z-index: 10"
+		>
+			<div class="w3-modal-content w3-card-4">
+				<div
+					class="w3-content w3-theme-d1"
+					style="display: flex; flex-direction: column; max-height: 50vh"
+				>
+					<header class="w3-container" style="flex: 0 0">
+						<h1>Choose a Reaction</h1>
+						<span class="w3-button w3-display-topright" @click=${cleanup}
+							>&times;</span
+						>
+					</header>
+					${div}
+					<footer class="w3-container w3-padding" style="flex: 0 0">
+						<button class="w3-button" @click=${cleanup}>Close</button>
+					</footer>
+				</div>
+			</div>
+		</div>
+	`;
+	document.body.appendChild(parent);
 	render(modal, parent);
 	input.focus();
 	document.body.addEventListener('mousedown', cleanup);

apps/ssb/script.js

@@ -8,7 +8,6 @@ import * as tf_compose from './tf-compose.js';
 import * as tf_news from './tf-news.js';
 import * as tf_profile from './tf-profile.js';
 import * as tf_reactions_modal from './tf-reactions-modal.js';
-import * as tf_tab_mentions from './tf-tab-mentions.js';
 import * as tf_tab_news from './tf-tab-news.js';
 import * as tf_tab_news_feed from './tf-tab-news-feed.js';
 import * as tf_tab_search from './tf-tab-search.js';

apps/ssb/tf-app.js

@@ -16,7 +16,9 @@ class TfElement extends LitElement {
 			following: {type: Array},
 			users: {type: Object},
 			ids: {type: Array},
-			tags: {type: Array},
+			channels: {type: Array},
+			channels_unread: {type: Object},
+			channels_latest: {type: Object},
 		};
 	}
 
@@ -33,7 +35,11 @@ class TfElement extends LitElement {
 		this.following = [];
 		this.users = {};
 		this.loaded = false;
-		this.tags = [];
+		this.channels = [];
+		this.channels_unread = {};
+		this.channels_latest = {};
+		this.loading_channels_latest = 0;
+		this.loading_channels_latest_scheduled = 0;
 		tfrpc.rpc.getBroadcasts().then((b) => {
 			self.broadcasts = b || [];
 		});
@@ -64,6 +70,68 @@ class TfElement extends LitElement {
 		let ids = (await tfrpc.rpc.getIdentities()) || [];
 		this.whoami = whoami ?? (ids.length ? ids[0] : undefined);
 		this.ids = ids;
+		await this.load_channels();
+	}
+
+	async load_channels() {
+		let channels = await tfrpc.rpc.query(
+			`
+			SELECT
+				content ->> 'channel' AS channel,
+				content ->> 'subscribed' AS subscribed
+			FROM
+				messages
+			WHERE
+				author = ? AND
+				content ->> 'type' = 'channel'
+			ORDER BY sequence
+		`,
+			[this.whoami]
+		);
+		let channel_map = {};
+		for (let row of channels) {
+			if (row.subscribed) {
+				channel_map[row.channel] = true;
+			} else {
+				delete channel_map[row.channel];
+			}
+		}
+		this.channels = Object.keys(channel_map).sort();
+	}
+
+	connectedCallback() {
+		super.connectedCallback();
+		this._keydown = this.keydown.bind(this);
+		window.addEventListener('keydown', this._keydown);
+	}
+
+	disconnectedCallback() {
+		super.disconnectedCallback();
+		window.removeEventListener('keydown', this._keydown);
+	}
+
+	keydown(event) {
+		if (event.altKey && event.key == 'ArrowUp') {
+			this.next_channel(1);
+			event.preventDefault();
+		} else if (event.altKey && event.key == 'ArrowDown') {
+			this.next_channel(-1);
+			event.preventDefault();
+		}
+	}
+
+	next_channel(delta) {
+		let channel_names = ['', '@'].concat(this.channels);
+		let index = channel_names.indexOf(this.hash.substring(1));
+		if (index != -1) {
+			index += delta;
+			this.set_hash(
+				'#' +
+					encodeURIComponent(
+						channel_names[(index + channel_names.length) % channel_names.length]
+					)
+			);
+		}
 	}
 
 	set_hash(hash) {
@@ -72,8 +140,6 @@ class TfElement extends LitElement {
 			this.tab = 'search';
 		} else if (this.hash === '#connections') {
 			this.tab = 'connections';
-		} else if (this.hash === '#mentions') {
-			this.tab = 'mentions';
 		} else if (this.hash.startsWith('#sql=')) {
 			this.tab = 'query';
 		} else {
@@ -167,10 +233,19 @@ class TfElement extends LitElement {
 			`,
 			[JSON.stringify(this.following), id]
 		);
+		for (let message of messages) {
+			if (message.author == this.whoami) {
+				let content = JSON.parse(message.content);
+				if (content?.type == 'channel') {
+					this.load_channels();
+				}
+			}
+		}
 		if (messages && messages.length) {
 			this.unread = [...this.unread, ...messages];
 			this.unread = this.unread.slice(this.unread.length - 1024);
 		}
+		this.schedule_load_channels_latest();
 	}
 
 	async _handle_whoami_changed(event) {
@@ -195,33 +270,95 @@ class TfElement extends LitElement {
 		}
 	}
 
-	async load_recent_tags() {
-		let start = new Date();
-		this.tags = await tfrpc.rpc.query(
-			`
-			WITH
-				recent AS (SELECT id, json(content) AS content FROM messages
-					WHERE messages.timestamp > ? AND json_extract(content, '$.type') = 'post'
-					ORDER BY timestamp DESC LIMIT 1024),
-				recent_channels AS (SELECT recent.id, '#' || json_extract(content, '$.channel') AS tag
-					FROM recent
-					WHERE json_extract(content, '$.channel') IS NOT NULL),
-				recent_mentions AS (SELECT recent.id, json_extract(mention.value, '$.link') AS tag
-					FROM recent, json_each(recent.content, '$.mentions') AS mention
-					WHERE json_valid(mention.value) AND tag LIKE '#%'),
-				combined AS (SELECT id, tag FROM recent_channels UNION ALL SELECT id, tag FROM recent_mentions),
-				by_message AS (SELECT DISTINCT id, tag FROM combined)
-			SELECT tag, COUNT(*) AS count FROM by_message GROUP BY tag ORDER BY count DESC LIMIT 10
-		`,
-			[new Date() - 7 * 24 * 60 * 60 * 1000]
-		);
-		console.log('tags took', (new Date() - start) / 1000.0, 'seconds');
+	async get_latest_private(following) {
+		let latest = (
+			await tfrpc.rpc.query('SELECT MAX(rowid) AS latest FROM messages')
+		)[0].latest;
+		const k_chunk_count = 256;
+		while (latest - k_chunk_count >= 0) {
+			let messages = await tfrpc.rpc.query(
+				`
+					SELECT messages.rowid, messages.id, previous, author, sequence, timestamp, hash, json(content) AS content, signature
+						FROM messages
+						JOIN json_each(?1) AS following ON messages.author = following.value
+						WHERE
+							messages.rowid > ?2 AND
+							messages.rowid <= ?3 AND
+							json(messages.content) LIKE '"%'
+						ORDER BY sequence DESC
+					`,
+				[JSON.stringify(following), latest - k_chunk_count, latest]
+			);
+			messages = (await this.decrypt(messages)).filter((x) => x.decrypted);
+			if (messages.length) {
+				return Math.max(...messages.map((x) => x.rowid));
+			}
+			latest -= k_chunk_count;
+		}
+		return -1;
+	}
+
+	async load_channels_latest(following) {
+		this.loading_channels_latest++;
+		try {
+			let start_time = new Date();
+			let latest_private = this.get_latest_private(following);
+			let channels = await tfrpc.rpc.query(
+				`
+				SELECT channels.value AS channel, MAX(messages.rowid) AS rowid FROM messages
+				JOIN json_each(?1) AS channels ON messages.content ->> 'channel' = channels.value
+				JOIN json_each(?2) AS following ON messages.author = following.value
+				WHERE messages.content ->> 'type' = 'post' AND messages.content ->> 'root' IS NULL
+				GROUP by channel
+				UNION
+				SELECT '' AS channel, MAX(messages.rowid) AS rowid FROM messages
+				JOIN json_each(?2) AS following ON messages.author = following.value
+				UNION
+				SELECT '@' AS channel, MAX(messages.rowid) AS rowid FROM messages_fts(?3)
+				JOIN messages ON messages.rowid = messages_fts.rowid
+				JOIN json_each(?2) AS following ON messages.author = following.value
+			`,
+				[
+					JSON.stringify(this.channels),
+					JSON.stringify(following),
+					'"' + this.whoami.replace('"', '""') + '"',
+				]
+			);
+			this.channels_latest = Object.fromEntries(
+				channels.map((x) => [x.channel, x.rowid])
+			);
+			console.log('latest', this.channels_latest);
+			console.log('unread', this.channels_unread);
+			console.log('channels took', (new Date() - start_time) / 1000.0);
+			let self = this;
+			latest_private.then(function (latest) {
+				self.channels_latest = Object.assign({}, self.channels_latest, {
+					'🔐': latest,
+				});
+				console.log('private took', (new Date() - start_time) / 1000.0);
+			});
+		} finally {
+			this.loading_channels_latest--;
+		}
+	}
+
+	_schedule_load_channels_latest_timer() {
+		--this.loading_channels_latest_scheduled;
+		this.schedule_load_channels_latest();
+	}
+
+	schedule_load_channels_latest() {
+		if (!this.loading_channels_latest) {
+			this.load_channels_latest(this.following);
+		} else if (!this.loading_channels_latest_scheduled) {
+			this.loading_channels_latest_scheduled++;
+			setTimeout(this._schedule_load_channels_latest_timer, 5000);
+		}
 	}
 
 	async load() {
 		let whoami = this.whoami;
-		let tags = this.load_recent_tags();
-		let following = await tfrpc.rpc.following([whoami], 3);
+		let following = await tfrpc.rpc.following([whoami], 2);
 		let users = {};
 		let by_count = [];
 		for (let [id, v] of Object.entries(following)) {
@@ -233,7 +370,10 @@ class TfElement extends LitElement {
 			};
 			by_count.push({count: v.of, id: id});
 		}
-		console.log(by_count.sort((x, y) => y.count - x.count).slice(0, 20));
+		let channels_latest = this.load_channels_latest(Object.keys(following));
+		this.channels_unread = JSON.parse(
+			(await tfrpc.rpc.databaseGet('unread')) ?? '{}'
+		);
 		let start_time = new Date();
 		users = await this.fetch_about(Object.keys(following).sort(), users);
 		console.log(
@@ -243,14 +383,47 @@ class TfElement extends LitElement {
 			Object.keys(users).length,
 			'users'
 		);
+		start_time = new Date();
+		await channels_latest;
 		this.following = Object.keys(following);
 		this.users = users;
-		await tags;
 		console.log(`load finished ${whoami} => ${this.whoami}`);
 		this.whoami = whoami;
 		this.loaded = whoami;
 	}
 
+	channel_set_unread(event) {
+		this.channels_unread[event.detail.channel ?? ''] = event.detail.unread;
+		this.channels_unread = Object.assign({}, this.channels_unread);
+		tfrpc.rpc.databaseSet('unread', JSON.stringify(this.channels_unread));
+	}
+
+	async decrypt(messages) {
+		let whoami = this.whoami;
+		return Promise.all(
+			messages.map(async function (message) {
+				let content;
+				try {
+					content = JSON.parse(message?.content);
+				} catch {}
+				if (typeof content === 'string') {
+					let decrypted;
+					try {
+						decrypted = await tfrpc.rpc.try_decrypt(whoami, content);
+					} catch {}
+					if (decrypted) {
+						try {
+							message.decrypted = JSON.parse(decrypted);
+						} catch {
+							message.decrypted = decrypted;
+						}
+					}
+				}
+				return message;
+			})
+		);
+	}
+
 	render_tab() {
 		let following = this.following;
 		let users = this.users;
@@ -265,6 +438,10 @@ class TfElement extends LitElement {
 					.unread=${this.unread}
 					@refresh=${() => (this.unread = [])}
 					?loading=${this.loading}
+					.channels=${this.channels}
+					.channels_latest=${this.channels_latest}
+					.channels_unread=${this.channels_unread}
+					@channelsetunread=${this.channel_set_unread}
 				></tf-tab-news>
 			`;
 		} else if (this.tab === 'connections') {
@@ -275,14 +452,6 @@ class TfElement extends LitElement {
 					.broadcasts=${this.broadcasts}
 				></tf-tab-connections>
 			`;
-		} else if (this.tab === 'mentions') {
-			return html`
-				<tf-tab-mentions
-					.following=${this.following}
-					whoami=${this.whoami}
-					.users="${this.users}}"
-				></tf-tab-mentions>
-			`;
 		} else if (this.tab === 'search') {
 			return html`
 				<tf-tab-search
@@ -314,8 +483,6 @@ class TfElement extends LitElement {
 			await tfrpc.rpc.setHash('#');
 		} else if (tab === 'connections') {
 			await tfrpc.rpc.setHash('#connections');
-		} else if (tab === 'mentions') {
-			await tfrpc.rpc.setHash('#mentions');
 		} else if (tab === 'query') {
 			await tfrpc.rpc.setHash('#sql=');
 		}
@@ -338,13 +505,15 @@ class TfElement extends LitElement {
 		const k_tabs = {
 			'📰': 'news',
 			'📡': 'connections',
-			'@': 'mentions',
 			'🔍': 'search',
 			'👩‍💻': 'query',
 		};
 
 		let tabs = html`
-			<div class="w3-bar w3-theme-l1">
+			<div
+				class="w3-bar w3-theme-l1"
+				style="position: sticky; top: 0; z-index: 10"
+			>
 				<button
 					class="w3-bar-item w3-button w3-circle w3-ripple"
 					@click=${this.refresh}
@@ -384,13 +553,7 @@ class TfElement extends LitElement {
 				style="width: 100vw; min-height: 100vh; height: 100%"
 				class="w3-theme-dark"
 			>
-				${tabs}
-				<div style="padding: 8px">
-					${this.tags.map(
-						(x) => html`<tf-tag tag=${x.tag} count=${x.count}></tf-tag>`
-					)}
-					${contents}
-				</div>
+				${tabs} ${contents}
 			</div>
 		`;
 	}

apps/ssb/tf-compose.js

@@ -14,6 +14,7 @@ class TfComposeElement extends LitElement {
 			apps: {type: Object},
 			drafts: {type: Object},
 			author: {type: String},
+			channel: {type: String},
 		};
 	}
 
@@ -196,6 +197,7 @@ class TfComposeElement extends LitElement {
 		let message = {
 			type: 'post',
 			text: edit.innerText,
+			channel: this.channel,
 		};
 		if (this.root || this.branch) {
 			message.root = this.root;
@@ -535,6 +537,9 @@ class TfComposeElement extends LitElement {
 				class="w3-card-4 w3-theme-d4 w3-padding-small"
 				style="box-sizing: border-box"
 			>
+				${this.channel !== undefined
+					? html`<p>To #${this.channel}:</p>`
+					: undefined}
 				${this.render_encrypt()}
 				<div class="w3-container w3-padding-small">
 					<div class="w3-half">

apps/ssb/tf-message.js

@@ -14,6 +14,8 @@ class TfMessageElement extends LitElement {
 			format: {type: String},
 			blog_data: {type: String},
 			expanded: {type: Object},
+			channel: {type: String},
+			channel_unread: {type: Number},
 		};
 	}
 
@@ -28,6 +30,7 @@ class TfMessageElement extends LitElement {
 		this.drafts = {};
 		this.format = 'message';
 		this.expanded = {};
+		this.channel_unread = -1;
 	}
 
 	show_reply() {
@@ -228,7 +231,7 @@ class TfMessageElement extends LitElement {
 				>${mention.name}</a
 			>`;
 		} else if (mention.link?.startsWith('#')) {
-			return html` <a href=${'#q=' + encodeURIComponent(mention.link)}
+			return html` <a href=${'#' + encodeURIComponent('#' + mention.link)}
 				>${mention.link}</a
 			>`;
 		} else if (
@@ -312,12 +315,27 @@ ${JSON.stringify(mention, null, 2)}</pre
 								.users=${this.users}
 								.drafts=${this.drafts}
 								.expanded=${this.expanded}
+								channel=${this.channel}
+								channel_unread=${this.channel_unread}
 							></tf-message>`
 					)}`;
 			}
 		}
 	}
 
+	mark_unread() {
+		this.dispatchEvent(
+			new CustomEvent('channelsetunread', {
+				bubbles: true,
+				composed: true,
+				detail: {
+					channel: this.channel,
+					unread: this.message.rowid,
+				},
+			})
+		);
+	}
+
 	render_channels() {
 		let content = this.message?.content;
 		if (this?.messsage?.decrypted?.type == 'post') {
@@ -344,7 +362,9 @@ ${JSON.stringify(mention, null, 2)}</pre
 		}
 		let class_background = this.message?.decrypted
 			? 'w3-pale-red'
-			: 'w3-theme-d4';
+			: this.message?.rowid >= this.channel_unread
+				? 'w3-theme-d2'
+				: 'w3-theme-d4';
 		let self = this;
 		let raw_button;
 		switch (this.format) {
@@ -403,7 +423,7 @@ ${JSON.stringify(mention, null, 2)}</pre
 			let body;
 			return html`
 				<div
-					class="w3-card-4 w3-theme-d4 w3-border-theme"
+					class="w3-card-4 ${class_background} w3-border-theme"
 					style="margin-top: 8px; padding: 16px; display: inline-block; overflow-wrap: anywhere"
 				>
 					<tf-user id=${self.message.author} .users=${self.users}></tf-user>
@@ -423,6 +443,8 @@ ${JSON.stringify(mention, null, 2)}</pre
 								.users=${self.users}
 								.drafts=${self.drafts}
 								.expanded=${self.expanded}
+								channel=${self.channel}
+								channel_unread=${self.channel_unread}
 							></tf-message>
 						`
 					)}
@@ -431,7 +453,7 @@ ${JSON.stringify(mention, null, 2)}</pre
 		}
 		if (this.message?.type === 'contact_group') {
 			return html` <div
-				class="w3-card-4 w3-theme-d4 w3-border-theme"
+				class="w3-card-4 ${class_background} w3-border-theme"
 				style="margin-top: 8px; padding: 16px; overflow-wrap: anywhere"
 			>
 				${this.message.messages.map(
@@ -442,12 +464,14 @@ ${JSON.stringify(mention, null, 2)}</pre
 							.users=${this.users}
 							.drafts=${this.drafts}
 							.expanded=${this.expanded}
+							channel=${this.channel}
+							channel_unread=${this.channel_unread}
 						></tf-message>`
 				)}
 			</div>`;
 		} else if (this.message.placeholder) {
 			return html` <div
-				class="w3-card-4 w3-theme-d4 w3-border-theme"
+				class="w3-card-4 ${class_background} w3-border-theme"
 				style="margin-top: 8px; padding: 16px; overflow-wrap: anywhere"
 			>
 				<a target="_top" href=${'#' + encodeURIComponent(this.message.id)}
@@ -463,6 +487,8 @@ ${JSON.stringify(mention, null, 2)}</pre
 							.users=${this.users}
 							.drafts=${this.drafts}
 							.expanded=${this.expanded}
+							channel=${this.channel}
+							channel_unread=${this.channel_unread}
 						></tf-message>
 					`
 				)}
@@ -618,6 +644,16 @@ ${JSON.stringify(content, null, 2)}</pre
 							<button class="w3-button w3-theme-d1" @click=${this.react}>
 								React
 							</button>
+							${!content.root && this.message.rowid < this.channel_unread
+								? html`
+										<button
+											class="w3-button w3-theme-d1"
+											@click=${this.mark_unread}
+										>
+											Mark Unread
+										</button>
+									`
+								: undefined}
 						</p>
 						${this.render_children()}
 					</div>
@@ -737,7 +773,7 @@ ${JSON.stringify(content, null, 2)}</pre
 						}
 					</style>
 					<div
-						class="w3-card-4 w3-theme-d4 w3-border-theme"
+						class="w3-card-4 ${class_background} w3-border-theme"
 						style="margin-top: 8px; padding: 16px"
 					>
 						<div style="display: flex; flex-direction: row">
@@ -787,7 +823,7 @@ ${JSON.stringify(content, null, 2)}</pre
 				return small_frame(html`
 					<div>
 						${content.subscribed ? 'subscribed to' : 'unsubscribed from'}
-						<a href=${'#q=' + encodeURIComponent('#' + content.channel)}
+						<a href=${'#' + encodeURIComponent('#' + content.channel)}
 							>#${content.channel}</a
 						>
 					</div>

apps/ssb/tf-news.js

@@ -11,6 +11,8 @@ class TfNewsElement extends LitElement {
 			following: {type: Array},
 			drafts: {type: Object},
 			expanded: {type: Object},
+			channel: {type: String},
+			channel_unread: {type: Number},
 		};
 	}
 
@@ -25,6 +27,7 @@ class TfNewsElement extends LitElement {
 		this.following = [];
 		this.drafts = {};
 		this.expanded = {};
+		this.channel_unread = -1;
 	}
 
 	process_messages(messages) {
@@ -33,12 +36,13 @@ class TfNewsElement extends LitElement {
 
 		console.log('processing', messages.length, 'messages');
 
-		function ensure_message(id) {
+		function ensure_message(id, rowid) {
 			let found = messages_by_id[id];
 			if (found) {
 				return found;
 			} else {
 				let added = {
+					rowid: rowid,
 					id: id,
 					placeholder: true,
 					content: '"placeholder"',
@@ -53,7 +57,7 @@ class TfNewsElement extends LitElement {
 
 		function link_message(message) {
 			if (message.content.type === 'vote') {
-				let parent = ensure_message(message.content.vote.link);
+				let parent = ensure_message(message.content.vote.link, message.rowid);
 				if (!parent.votes) {
 					parent.votes = [];
 				}
@@ -62,14 +66,14 @@ class TfNewsElement extends LitElement {
 			} else if (message.content.type == 'post') {
 				if (message.content.root) {
 					if (typeof message.content.root === 'string') {
-						let m = ensure_message(message.content.root);
+						let m = ensure_message(message.content.root, message.rowid);
 						if (!m.child_messages) {
 							m.child_messages = [];
 						}
 						m.child_messages.push(message);
 						message.parent_message = message.content.root;
 					} else {
-						let m = ensure_message(message.content.root[0]);
+						let m = ensure_message(message.content.root[0], message.rowid);
 						if (!m.child_messages) {
 							m.child_messages = [];
 						}
@@ -162,6 +166,7 @@ class TfNewsElement extends LitElement {
 			} else {
 				if (group.length > 0) {
 					result.push({
+						rowid: Math.max(...group.map((x) => x.rowid)),
 						type: 'contact_group',
 						messages: group,
 					});
@@ -170,6 +175,13 @@ class TfNewsElement extends LitElement {
 				result.push(message);
 			}
 		}
+		if (group.length > 0) {
+			result.push({
+				rowid: Math.max(...group.map((x) => x.rowid)),
+				type: 'contact_group',
+				messages: group,
+			});
+		}
 		return result;
 	}
 
@@ -178,18 +190,38 @@ class TfNewsElement extends LitElement {
 		let final_messages = this.group_following(
 			this.finalize_messages(messages_by_id)
 		);
+		let unread_rowid = -1;
+		for (let message of final_messages) {
+			if (message.rowid >= this.channel_unread) {
+				unread_rowid = message.rowid;
+			}
+		}
 		return html`
-			<div style="display: flex; flex-direction: column">
+			<div>
 				${final_messages.map(
-					(x) =>
-						html`<tf-message
+					(x) => html`
+						<tf-message
 							.message=${x}
 							whoami=${this.whoami}
 							.users=${this.users}
 							.drafts=${this.drafts}
 							.expanded=${this.expanded}
 							collapsed="true"
-						></tf-message>`
+							channel=${this.channel}
+							channel_unread=${this.channel_unread}
+						></tf-message>
+						${x.rowid == unread_rowid && x != final_messages[0]
+							? html`<div style="display: flex; flex-direction: row">
+									<div
+										style="border-bottom: 1px solid #f00; flex: 1; align-self: center; height: 1px"
+									></div>
+									<div style="color: #f00; padding: 8px">unread</div>
+									<div
+										style="border-bottom: 1px solid #f00; flex: 1; align-self: center; height: 1px"
+									></div>
+								</div>`
+							: undefined}
+					`
 				)}
 			</div>
 		`;

apps/ssb/tf-reactions-modal.js

@@ -26,7 +26,7 @@ class TfReactionsModalElement extends LitElement {
 		return this.votes?.length
 			? html` <div
 					class="w3-modal w3-animate-opacity"
-					style="display: block; box-sizing: border-box"
+					style="display: block; box-sizing: border-box; z-index: 10"
 					@click=${this.clear}
 				>
 					<div

apps/ssb/tf-styles.js

@@ -286,29 +286,29 @@ hr{border:0;border-top:1px solid #eee;margin:20px 0}
 `;
 
 // prettier-ignore
-const w3_2016_riverside = css`
-.w3-theme-l5 {color:#000 !important; background-color:#f4f6f9 !important}
-.w3-theme-l4 {color:#000 !important; background-color:#d9e1ec !important}
-.w3-theme-l3 {color:#000 !important; background-color:#b4c3d8 !important}
-.w3-theme-l2 {color:#fff !important; background-color:#8ea6c5 !important}
-.w3-theme-l1 {color:#fff !important; background-color:#6888b1 !important}
-.w3-theme-d1 {color:#fff !important; background-color:#456185 !important}
-.w3-theme-d2 {color:#fff !important; background-color:#3d5676 !important}
-.w3-theme-d3 {color:#fff !important; background-color:#354b68 !important}
-.w3-theme-d4 {color:#fff !important; background-color:#2e4059 !important}
-.w3-theme-d5 {color:#fff !important; background-color:#26364a !important}
+const w3_2016_snorkel_blue = css`
+.w3-theme-l5 {color:#000 !important; background-color:#e9f5ff !important}
+.w3-theme-l4 {color:#000 !important; background-color:#b5dffd !important}
+.w3-theme-l3 {color:#000 !important; background-color:#6bc0fc !important}
+.w3-theme-l2 {color:#fff !important; background-color:#21a0fa !important}
+.w3-theme-l1 {color:#fff !important; background-color:#0479cc !important}
+.w3-theme-d1 {color:#fff !important; background-color:#024575 !important}
+.w3-theme-d2 {color:#fff !important; background-color:#023e68 !important}
+.w3-theme-d3 {color:#fff !important; background-color:#02365b !important}
+.w3-theme-d4 {color:#fff !important; background-color:#022e4e !important}
+.w3-theme-d5 {color:#fff !important; background-color:#012641 !important}
 
-.w3-theme-light {color:#000 !important; background-color:#f4f6f9 !important}
-.w3-theme-dark {color:#fff !important; background-color:#26364a !important}
-.w3-theme-action {color:#fff !important; background-color:#26364a !important}
+.w3-theme-light {color:#000 !important; background-color:#e9f5ff !important}
+.w3-theme-dark {color:#fff !important; background-color:#012641 !important}
+.w3-theme-action {color:#fff !important; background-color:#012641 !important}
 
-.w3-theme {color:#fff !important; background-color:#4c6a92 !important}
-.w3-text-theme {color:#4c6a92 !important}
-.w3-border-theme {border-color:#4c6a92 !important}
+.w3-theme {color:#fff !important; background-color:#034f84 !important}
+.w3-text-theme {color:#034f84 !important}
+.w3-border-theme {border-color:#034f84 !important}
 
-.w3-hover-theme:hover {color:#fff !important; background-color:#4c6a92 !important}
-.w3-hover-text-theme:hover {color:#4c6a92 !important}
-.w3-hover-border-theme:hover {border-color:#4c6a92 !important}
+.w3-hover-theme:hover {color:#fff !important; background-color:#034f84 !important}
+.w3-hover-text-theme:hover {color:#034f84 !important}
+.w3-hover-border-theme:hover {border-color:#034f84 !important}
 `;
 
-export let styles = [tf, w3, w3_2016_riverside];
+export let styles = [tf, w3, w3_2016_snorkel_blue];

apps/ssb/tf-tab-mentions.js

@@ -1,78 +0,0 @@
-import {LitElement, html, unsafeHTML} from './lit-all.min.js';
-import * as tfrpc from '/static/tfrpc.js';
-import {styles} from './tf-styles.js';
-
-class TfTabMentionsElement extends LitElement {
-	static get properties() {
-		return {
-			whoami: {type: String},
-			users: {type: Object},
-			following: {type: Array},
-			expanded: {type: Object},
-			messages: {type: Array},
-		};
-	}
-
-	static styles = styles;
-
-	constructor() {
-		super();
-		let self = this;
-		this.whoami = null;
-		this.users = {};
-		this.following = [];
-		this.expanded = {};
-		this.messages = [];
-	}
-
-	async load() {
-		console.log('Loading...', this.whoami);
-		let results = await tfrpc.rpc.query(
-			`
-				SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
-				FROM messages_fts(?)
-				JOIN messages ON messages.rowid = messages_fts.rowid
-				JOIN json_each(?) AS following ON messages.author = following.value
-				WHERE messages.author != ?
-				ORDER BY timestamp DESC limit 20
-			`,
-			[
-				'"' + this.whoami.replace('"', '""') + '"',
-				JSON.stringify(this.following),
-				this.whoami,
-			]
-		);
-		console.log('Done.');
-		this.messages = results;
-	}
-
-	on_expand(event) {
-		if (event.detail.expanded) {
-			let expand = {};
-			expand[event.detail.id] = true;
-			this.expanded = Object.assign({}, this.expanded, expand);
-		} else {
-			delete this.expanded[event.detail.id];
-			this.expanded = Object.assign({}, this.expanded);
-		}
-	}
-
-	render() {
-		let self = this;
-		if (!this.loading) {
-			this.loading = true;
-			this.load();
-		}
-		return html`
-			<tf-news
-				id="news"
-				whoami=${this.whoami}
-				.messages=${this.messages}
-				.users=${this.users}
-				.expanded=${this.expanded}
-				@tf-expand=${this.on_expand}
-			></tf-news>
-		`;
-	}
-}
-customElements.define('tf-tab-mentions', TfTabMentionsElement);

apps/ssb/tf-tab-news-feed.js

@@ -12,6 +12,11 @@ class TfTabNewsFeedElement extends LitElement {
 			messages: {type: Array},
 			drafts: {type: Object},
 			expanded: {type: Object},
+			channels_unread: {type: Object},
+			channels_latest: {type: Object},
+			loading: {type: Number},
+			time_range: {type: Array},
+			time_loading: {type: Array},
 		};
 	}
 
@@ -26,30 +31,67 @@ class TfTabNewsFeedElement extends LitElement {
 		this.following = [];
 		this.drafts = {};
 		this.expanded = {};
-		this.start_time = new Date().valueOf() - 24 * 60 * 60 * 1000;
+		this.channels_unread = {};
+		this.channels_latest = {};
+		this.start_time = new Date().valueOf();
+		this.time_range = [0, 0];
+		this.time_loading = undefined;
+		this.loading = 0;
 	}
 
-	async fetch_messages() {
-		if (this.hash.startsWith('#@')) {
-			let r = await tfrpc.rpc.query(
+	channel() {
+		return this.hash.startsWith('##')
+			? this.hash.substring(2)
+			: this.hash.substring(1);
+	}
+
+	async fetch_messages(start_time, end_time) {
+		this.time_loading = [start_time, end_time];
+		let result;
+		if (this.hash == '#@') {
+			result = await tfrpc.rpc.query(
 				`
-					WITH mine AS (SELECT id, previous, author, sequence, timestamp, hash, json(content) AS content, signature
+					SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
+					FROM messages_fts(?1)
+					JOIN messages ON messages.rowid = messages_fts.rowid
+					JOIN json_each(?2) AS following ON messages.author = following.value
+					WHERE
+						messages.author != ?1 AND
+						messages.timestamp >= ?3 AND
+						messages.timestamp < ?4
+					ORDER BY timestamp DESC limit 20
+				`,
+				[
+					'"' + this.whoami.replace('"', '""') + '"',
+					JSON.stringify(this.following),
+					start_time,
+					end_time,
+				]
+			);
+		} else if (this.hash.startsWith('#@')) {
+			result = await tfrpc.rpc.query(
+				`
+					WITH mine AS (SELECT rowid, id, previous, author, sequence, timestamp, hash, json(content) AS content, signature
 						FROM messages
 						WHERE messages.author = ?
-						ORDER BY sequence DESC
-						LIMIT 20)
-					SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
+						ORDER BY sequence DESC)
+					SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
 						FROM mine
 						JOIN messages_refs ON mine.id = messages_refs.ref
 						JOIN messages ON messages_refs.message = messages.id
+						WHERE
+							mine.timestamp >= ?2 AND
+							mine.timestamp < ?3
 					UNION
 					SELECT * FROM mine
+					WHERE
+						mine.timestamp >= ?2 AND
+						mine.timestamp < ?3
 				`,
-				[this.hash.substring(1)]
+				[this.hash.substring(1), start_time, end_time]
 			);
-			return r;
 		} else if (this.hash.startsWith('#%')) {
-			return await tfrpc.rpc.query(
+			result = await tfrpc.rpc.query(
 				`
 					SELECT id, previous, author, sequence, timestamp, hash, json(content) AS content, signature
 					FROM messages
@@ -62,6 +104,68 @@ class TfTabNewsFeedElement extends LitElement {
 				`,
 				[this.hash.substring(1)]
 			);
+		} else if (this.hash.startsWith('##')) {
+			let promises = [];
+			const k_following_limit = 256;
+			for (let i = 0; i < this.following.length; i += k_following_limit) {
+				promises.push(
+					tfrpc.rpc.query(
+						`
+						WITH news AS (SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
+							FROM messages
+							JOIN json_each(?) AS following ON messages.author = following.value
+							WHERE
+								messages.timestamp >= ? AND
+								messages.timestamp < ? AND
+								messages.content ->> 'channel' = ?
+							ORDER BY messages.timestamp DESC)
+						SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
+							FROM news
+							JOIN messages_refs ON news.id = messages_refs.ref
+							JOIN messages ON messages_refs.message = messages.id
+						UNION
+						SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
+							FROM news
+							JOIN messages_refs ON news.id = messages_refs.message
+							JOIN messages ON messages_refs.ref = messages.id
+						UNION
+						SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
+							FROM messages_fts(?5)
+							JOIN messages ON messages.rowid = messages_fts.rowid
+							JOIN json_each(?1) AS following ON messages.author = following.value
+							JOIN json_tree(messages.content, '$.mentions') AS mention ON mention.value = '#' || ?4
+							WHERE
+								messages.timestamp >= ?2 AND
+								messages.timestamp < ?3
+						UNION
+						SELECT news.* FROM news
+					`,
+						[
+							JSON.stringify(this.following.slice(i, i + k_following_limit)),
+							start_time,
+							end_time,
+							this.hash.substring(2),
+							'"#' + this.hash.substring(2).replace('"', '""') + '"',
+						]
+					)
+				);
+			}
+			result = [].concat(...(await Promise.all(promises)));
+		} else if (this.hash == '#🔐') {
+			result = await tfrpc.rpc.query(
+				`
+					SELECT messages.rowid, messages.id, previous, author, sequence, timestamp, hash, json(content) AS content, signature
+						FROM messages
+						JOIN json_each(?1) AS following ON messages.author = following.value
+						WHERE
+							messages.timestamp >= ?2 AND
+							messages.timestamp < ?3 AND
+							json(messages.content) LIKE '"%'
+						ORDER BY sequence DESC
+				`,
+				[JSON.stringify(this.following), start_time, end_time]
+			);
+			result = (await this.decrypt(result)).filter((x) => x.decrypted);
 		} else {
 			let promises = [];
 			const k_following_limit = 256;
@@ -69,17 +173,17 @@ class TfTabNewsFeedElement extends LitElement {
 				promises.push(
 					tfrpc.rpc.query(
 						`
-						WITH news AS (SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
+						WITH news AS (SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
 						FROM messages
 						JOIN json_each(?) AS following ON messages.author = following.value
-						WHERE messages.timestamp > ? AND messages.timestamp < ?
+						WHERE messages.timestamp >= ? AND messages.timestamp < ?
 						ORDER BY messages.timestamp DESC)
-						SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
+						SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
 							FROM news
 							JOIN messages_refs ON news.id = messages_refs.ref
 							JOIN messages ON messages_refs.message = messages.id
 						UNION
-						SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
+						SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
 							FROM news
 							JOIN messages_refs ON news.id = messages_refs.message
 							JOIN messages ON messages_refs.ref = messages.id
@@ -88,50 +192,58 @@ class TfTabNewsFeedElement extends LitElement {
 					`,
 						[
 							JSON.stringify(this.following.slice(i, i + k_following_limit)),
-							this.start_time,
-							/*
-							 ** Don't show messages more than a day into the future to prevent
-							 ** messages with far-future timestamps from staying at the top forever.
-							 */
-							new Date().valueOf() + 24 * 60 * 60 * 1000,
+							start_time,
+							end_time,
 						]
 					)
 				);
 			}
-			return [].concat(...(await Promise.all(promises)));
+			result = [].concat(...(await Promise.all(promises)));
 		}
+		this.time_loading = undefined;
+		return result;
+	}
+
+	update_time_range_from_messages(messages) {
+		this.time_range = [
+			messages.reduce(
+				(accumulator, current) => Math.min(accumulator, current.timestamp),
+				this.time_range[0]
+			),
+			messages.reduce(
+				(accumulator, current) => Math.max(accumulator, current.timestamp),
+				this.time_range[1]
+			),
+		];
 	}
 
 	async load_more() {
-		let last_start_time = this.start_time;
-		this.start_time = last_start_time - 24 * 60 * 60 * 1000;
-		let more = await tfrpc.rpc.query(
-			`
-				WITH news AS (SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
-				FROM messages
-				JOIN json_each(?) AS following ON messages.author = following.value
-				WHERE messages.timestamp > ?
-				AND messages.timestamp <= ?
-				ORDER BY messages.timestamp DESC)
-				SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
-					FROM news
-					JOIN messages_refs ON news.id = messages_refs.ref
-					JOIN messages ON messages_refs.message = messages.id
-				UNION
-				SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
-					FROM news
-					JOIN messages_refs ON news.id = messages_refs.message
-					JOIN messages ON messages_refs.ref = messages.id
-				UNION
-				SELECT news.* FROM news
-			`,
-			[JSON.stringify(this.following), this.start_time, last_start_time]
-		);
-		this.messages = await this.decrypt([...more, ...this.messages]);
+		this.loading++;
+		this.loading_canceled = false;
+		try {
+			let more = [];
+			while (!more.length && !this.loading_canceled) {
+				let last_start_time = this.start_time;
+				this.start_time = last_start_time - 7 * 24 * 60 * 60 * 1000;
+				more = await this.fetch_messages(this.start_time, last_start_time);
+				this.update_time_range_from_messages(
+					more.filter(
+						(x) =>
+							x.timestamp >= this.start_time && x.timestamp < last_start_time
+					)
+				);
+			}
+			this.messages = await this.decrypt([...more, ...this.messages]);
+		} finally {
+			this.loading--;
+		}
+	}
+
+	cancel_load() {
+		this.loading_canceled = true;
 	}
 
 	async decrypt(messages) {
-		console.log('decrypt');
 		let result = [];
 		for (let message of messages) {
 			let content;
@@ -156,8 +268,94 @@ class TfTabNewsFeedElement extends LitElement {
 		return result;
 	}
 
-	async add_messages(messages) {
-		this.messages = await this.decrypt([...messages, ...this.messages]);
+	async load_latest() {
+		this.loading++;
+		let now = new Date().valueOf();
+		let end_time = now + 24 * 60 * 60 * 1000;
+		let messages = [];
+		try {
+			messages = await this.fetch_messages(
+				this.time_range[1] - 24 * 60 * 60 * 1000,
+				end_time
+			);
+			messages = await this.decrypt(messages);
+			this.update_time_range_from_messages(
+				messages.filter(
+					(x) => x.timestamp >= this.time_range[1] && x.timestamp < end_time
+				)
+			);
+		} finally {
+			this.loading--;
+		}
+		this.messages = Object.values(
+			Object.fromEntries([...this.messages, ...messages].map((x) => [x.id, x]))
+		);
+		console.log('done loading latest messages.');
+	}
+
+	async load_messages() {
+		let self = this;
+		this.loading++;
+		let messages = [];
+		try {
+			this.messages = [];
+			this._messages_hash = this.hash;
+			this._messages_following = this.following;
+			let now = new Date().valueOf();
+			let start_time = now - 24 * 60 * 60 * 1000;
+			this.start_time = start_time;
+			this.time_range = [this.start_time, now + 24 * 60 * 60 * 1000];
+			messages = await this.fetch_messages(
+				this.time_range[0],
+				this.time_range[1]
+			);
+			this.update_time_range_from_messages(
+				messages.filter(
+					(x) =>
+						x.timestamp >= this.time_range[0] &&
+						x.timestamp < this.time_range[1]
+				)
+			);
+			messages = await this.decrypt(messages);
+			if (!messages.length) {
+				let more = [];
+				while (!more.length && start_time >= 0) {
+					let last_start_time = start_time;
+					start_time = last_start_time - 7 * 24 * 60 * 60 * 1000;
+					more = await this.fetch_messages(start_time, last_start_time);
+					this.update_time_range_from_messages(
+						more.filter(
+							(x) => x.timestamp >= start_time && x.timestamp < last_start_time
+						)
+					);
+				}
+				messages = await this.decrypt([...more, ...this.messages]);
+			}
+		} finally {
+			this.loading--;
+		}
+		this.messages = messages;
+		this.time_loading = undefined;
+		console.log(`loading messages done for ${self.whoami}`);
+	}
+
+	mark_all_read() {
+		let newest = this.messages.reduce(
+			(accumulator, current) => Math.max(accumulator, current.rowid),
+			this.channels_latest[this.channel()] ?? -1
+		);
+		if (newest >= 0) {
+			this.dispatchEvent(
+				new CustomEvent('channelsetunread', {
+					bubbles: true,
+					composed: true,
+					detail: {
+						channel: this.channel(),
+						unread: newest + 1,
+					},
+				})
+			);
+		}
 	}
 
 	render() {
@@ -169,31 +367,49 @@ class TfTabNewsFeedElement extends LitElement {
 			console.log(
 				`loading messages for ${this.whoami} (following ${this.following.length})`
 			);
-			let self = this;
-			this.messages = [];
-			this._messages_hash = this.hash;
-			this._messages_following = this.following;
-			this.fetch_messages()
-				.then(this.decrypt.bind(this))
-				.then(function (messages) {
-					self.messages = messages;
-					console.log(`loading mesages done for ${self.whoami}`);
-				})
-				.catch(function (error) {
-					alert(JSON.stringify(error, null, 2));
-				});
+			this.load_messages();
 		}
 		let more;
-		if (!this.hash.startsWith('#@') && !this.hash.startsWith('#%')) {
+		if (!this.hash.startsWith('#%')) {
 			more = html`
 				<p>
-					<button class="w3-button w3-theme-d1" @click=${this.load_more}>
+					<button class="w3-button w3-theme-d1" @click=${this.mark_all_read}>
+						Mark All Read
+					</button>
+					<button
+						?disabled=${this.loading}
+						class="w3-button w3-theme-d1"
+						@click=${this.load_more}
+					>
 						Load More
 					</button>
+					<button
+						class=${'w3-button w3-theme-d1' + (this.loading ? '' : ' w3-hide')}
+						@click=${this.cancel_load}
+					>
+						Cancel
+					</button>
+					<span
+						>Showing
+						${new Date(
+							this.time_loading
+								? Math.min(this.time_loading[0], this.time_range[0])
+								: this.time_range[0]
+						).toLocaleDateString()}
+						-
+						${new Date(
+							this.time_loading
+								? Math.max(this.time_loading[1], this.time_range[1])
+								: this.time_range[1]
+						).toLocaleDateString()}.</span
+					>
 				</p>
 			`;
 		}
 		return html`
+			<button class="w3-button w3-theme-d1" @click=${this.mark_all_read}>
+				Mark All Read
+			</button>
 			<tf-news
 				id="news"
 				whoami=${this.whoami}
@@ -202,6 +418,8 @@ class TfTabNewsFeedElement extends LitElement {
 				.following=${this.following}
 				.drafts=${this.drafts}
 				.expanded=${this.expanded}
+				channel=${this.channel()}
+				channel_unread=${this.channels_unread?.[this.channel()]}
 			></tf-news>
 			${more}
 		`;

apps/ssb/tf-tab-news.js

@@ -13,6 +13,9 @@ class TfTabNewsElement extends LitElement {
 			drafts: {type: Object},
 			expanded: {type: Object},
 			loading: {type: Boolean},
+			channels: {type: Array},
+			channels_unread: {type: Object},
+			channels_latest: {type: Object},
 		};
 	}
 
@@ -29,6 +32,9 @@ class TfTabNewsElement extends LitElement {
 		this.cache = {};
 		this.drafts = {};
 		this.expanded = {};
+		this.channels_unread = {};
+		this.channels_latest = {};
+		this.channels = [];
 		tfrpc.rpc.localStorageGet('drafts').then(function (d) {
 			self.drafts = JSON.parse(d || '{}');
 		});
@@ -48,10 +54,7 @@ class TfTabNewsElement extends LitElement {
 		let unread = this.unread;
 		let news = this.shadowRoot?.getElementById('news');
 		if (news) {
-			console.log('injecting messages', news.messages);
-			news.add_messages(
-				Object.values(Object.fromEntries(this.unread.map((x) => [x.id, x])))
-			);
+			news.load_latest();
 			this.dispatchEvent(new CustomEvent('refresh'));
 		}
 	}
@@ -106,15 +109,57 @@ class TfTabNewsElement extends LitElement {
 		}
 	}
 
+	unread_status(channel) {
+		if (
+			this.channels_latest[channel] &&
+			(this.channels_unread[channel] === undefined ||
+				this.channels_unread[channel] <= this.channels_latest[channel])
+		) {
+			return '🔵';
+		}
+	}
+
+	show_sidebar() {
+		this.renderRoot.getElementById('sidebar').style.display = 'block';
+		this.renderRoot.getElementById('sidebar_overlay').style.display = 'block';
+	}
+
+	hide_sidebar() {
+		this.renderRoot.getElementById('sidebar').style.display = 'none';
+		this.renderRoot.getElementById('sidebar_overlay').style.display = 'none';
+	}
+
+	async channel_toggle_subscribed() {
+		let channel = this.hash.substring(2);
+		let subscribed = this.channels.indexOf(channel) != -1;
+		subscribed = !subscribed;
+
+		await tfrpc.rpc.appendMessage(this.whoami, {
+			type: 'channel',
+			channel: channel,
+			subscribed: subscribed,
+		});
+		if (subscribed) {
+			this.channels = [].concat([channel], this.channels).sort();
+		} else {
+			this.channels = this.channels.filter((x) => x != channel);
+		}
+	}
+
+	channel() {
+		return this.hash.startsWith('##') ? this.hash.substring(2) : undefined;
+	}
+
 	render() {
-		let profile = this.hash.startsWith('#@')
-			? html`<tf-profile
-					class="tf-profile"
-					id=${this.hash.substring(1)}
-					whoami=${this.whoami}
-					.users=${this.users}
-				></tf-profile>`
-			: undefined;
+		let profile =
+			this.hash.startsWith('#@') && this.hash != '#@'
+				? html`<tf-profile
+						class="tf-profile"
+						id=${this.hash.substring(1)}
+						whoami=${this.whoami}
+						.users=${this.users}
+					></tf-profile>`
+				: undefined;
 		let edit_profile;
 		if (
 			!this.loading &&
@@ -129,39 +174,118 @@ class TfTabNewsElement extends LitElement {
 			</div>`;
 		}
 		return html`
-			<p class="w3-bar">
-				<button
-					class="w3-bar-item w3-button w3-theme-d1"
-					@click=${this.show_more}
+			<div
+				class="w3-sidebar w3-bar-block w3-theme-d1 w3-collapse w3-animate-left"
+				style="width: 2in; left: 0; z-index: 5"
+				id="sidebar"
+			>
+				<div
+					class="w3-right w3-button w3-hide-large"
+					@click=${this.hide_sidebar}
 				>
-					${this.new_messages_text()}
-				</button>
-			</p>
-			<div class="w3-bar">
-				Welcome, <tf-user id=${this.whoami} .users=${this.users}></tf-user>!
-				${edit_profile}
+					&times;
+				</div>
+				${this.hash.startsWith('##') &&
+				this.channels.indexOf(this.hash.substring(2)) == -1
+					? html`
+							<div class="w3-bar-item w3-theme-d2">Viewing</div>
+							<a
+								href="#"
+								class="w3-bar-item w3-button"
+								style="font-weight: bold"
+								>${this.hash.substring(2)}</a
+							>
+						`
+					: undefined}
+				<div class="w3-bar-item w3-theme-d2">Channels</div>
+				<a
+					href="#"
+					class="w3-bar-item w3-button"
+					style=${this.hash == '#' ? 'font-weight: bold' : undefined}
+					>general ${this.unread_status('')}</a
+				>
+				<a
+					href="#@"
+					class="w3-bar-item w3-button"
+					style=${this.hash == '#@' ? 'font-weight: bold' : undefined}
+					>@mentions ${this.unread_status('@')}</a
+				>
+				<a
+					href="#🔐"
+					class="w3-bar-item w3-button"
+					style=${this.hash == '#🔐' ? 'font-weight: bold' : undefined}
+					>🔐private ${this.unread_status('🔐')}</a
+				>
+				${this.channels.map(
+					(x) => html`
+						<a
+							href=${'#' + encodeURIComponent('#' + x)}
+							class="w3-bar-item w3-button"
+							style=${this.hash == '##' + x ? 'font-weight: bold' : undefined}
+							>#${x} ${this.unread_status(x)}</a
+						>
+					`
+				)}
 			</div>
-			<div>
-				<tf-compose
-					id="tf-compose"
+			<div
+				class="w3-overlay"
+				id="sidebar_overlay"
+				@click=${this.hide_sidebar}
+			></div>
+			<div style="margin-left: 2in; padding: 8px" id="main" class="w3-main">
+				<div
+					id="show_sidebar"
+					class="w3-left w3-button w3-hide-large"
+					@click=${this.show_sidebar}
+				>
+					&#9776;
+				</div>
+				<p>
+					<button class="w3-button w3-theme-d1" @click=${this.show_more}>
+						${this.new_messages_text()}
+					</button>
+					${this.hash.startsWith('##')
+						? html`
+								<button
+									class="w3-button w3-theme-d1"
+									@click=${this.channel_toggle_subscribed}
+								>
+									${this.channels.indexOf(this.hash.substring(2)) != -1
+										? 'Unsubscribe from #'
+										: 'Subscribe to #'}${this.hash.substring(2)}
+								</button>
+							`
+						: undefined}
+				</p>
+				<div class="w3-bar">
+					Welcome, <tf-user id=${this.whoami} .users=${this.users}></tf-user>!
+					${edit_profile}
+				</div>
+				<div>
+					<tf-compose
+						id="tf-compose"
+						whoami=${this.whoami}
+						.users=${this.users}
+						.drafts=${this.drafts}
+						@tf-draft=${this.draft}
+						.channel=${this.channel()}
+					></tf-compose>
+				</div>
+				${profile}
+				<tf-tab-news-feed
+					id="news"
 					whoami=${this.whoami}
 					.users=${this.users}
+					.following=${this.following}
+					hash=${this.hash}
 					.drafts=${this.drafts}
+					.expanded=${this.expanded}
 					@tf-draft=${this.draft}
-				></tf-compose>
+					@tf-expand=${this.on_expand}
+					.channels_unread=${this.channels_unread}
+					.channels_latest=${this.channels_latest}
+				></tf-tab-news-feed>
 			</div>
-			${profile}
-			<tf-tab-news-feed
-				id="news"
-				whoami=${this.whoami}
-				.users=${this.users}
-				.following=${this.following}
-				hash=${this.hash}
-				.drafts=${this.drafts}
-				.expanded=${this.expanded}
-				@tf-draft=${this.draft}
-				@tf-expand=${this.on_expand}
-			></tf-tab-news-feed>
 		`;
 	}
 }

apps/ssb/tf-tag.js

@@ -18,7 +18,7 @@ class TfTagElement extends LitElement {
 	render() {
 		let number = this.count ? html` (${this.count})` : undefined;
 		return html`<a
-			href="#q=${this.tag}"
+			href=${'#' + encodeURIComponent(this.tag)}
 			style="display: inline-block; margin: 3px; border: 1px solid black; background-color: #444; padding: 4px; border-radius: 3px"
 			>${this.tag}${number}</a
 		>`;

apps/wiki.json

@@ -1,5 +1,5 @@
 {
 	"type": "tildefriends-app",
 	"emoji": "📝",
-	"previous": "&4F4D8+QlJVaxXywChQrNTdSV4Y3TvJ0xxqdq/i9HUWA=.sha256"
+	"previous": "&4UHlsfQJvSh7L3D86uFtr7KUKCMRVBBTFxRIMqIc5as=.sha256"
 }

core/core.js

@@ -419,6 +419,7 @@ async function getProcessBlob(blobId, key, options) {
 					return settings?.[key];
 				};
 				imports.core.globalSettingsSet = async function (key, value) {
+					await imports.core.permissionTest('set_global_setting');
 					print('Setting', key, value);
 					let settings = await loadSettings();
 					settings[key] = value;

default.nix

@@ -21,14 +21,14 @@
 }:
 pkgs.stdenv.mkDerivation rec {
   pname = "tildefriends";
-  version = "0.0.24";
+  version = "0.0.25";
 
   src = pkgs.fetchFromGitea {
     domain = "dev.tildefriends.net";
     owner = "cory";
     repo = "tildefriends";
     rev = "v${version}";
-    hash = "sha256-XlmRr08UmScY//qxUEXHzagXHCFqARRYr3q8RK/jKFY=";
+    hash = "sha256-Rfk+CUhi+Ss0z70CCgmtVM/w4nCL1GX/MsD4sPYIa5s=";
     fetchSubmodules = true;
   };
 

deps/codemirror_src/package-lock.json

@@ -19,21 +19,15 @@
       }
     },
     "node_modules/@codemirror/autocomplete": {
-      "version": "6.18.3",
-      "resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-6.18.3.tgz",
-      "integrity": "sha512-1dNIOmiM0z4BIBwxmxEfA1yoxh1MF/6KPBbh20a5vphGV0ictKlgQsbJs6D6SkR6iJpGbpwRsa6PFMNlg9T9pQ==",
+      "version": "6.18.4",
+      "resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-6.18.4.tgz",
+      "integrity": "sha512-sFAphGQIqyQZfP2ZBsSHV7xQvo9Py0rV0dW7W3IMRdS+zDuNb2l3no78CvUaWKGfzFjI4FTrLdUSj86IGb2hRA==",
       "license": "MIT",
       "dependencies": {
         "@codemirror/language": "^6.0.0",
         "@codemirror/state": "^6.0.0",
         "@codemirror/view": "^6.17.0",
         "@lezer/common": "^1.0.0"
-      },
-      "peerDependencies": {
-        "@codemirror/language": "^6.0.0",
-        "@codemirror/state": "^6.0.0",
-        "@codemirror/view": "^6.0.0",
-        "@lezer/common": "^1.0.0"
       }
     },
     "node_modules/@codemirror/commands": {
@@ -49,9 +43,9 @@
       }
     },
     "node_modules/@codemirror/lang-css": {
-      "version": "6.3.0",
-      "resolved": "https://registry.npmjs.org/@codemirror/lang-css/-/lang-css-6.3.0.tgz",
-      "integrity": "sha512-CyR4rUNG9OYcXDZwMPvJdtb6PHbBDKUc/6Na2BIwZ6dKab1JQqKa4di+RNRY9Myn7JB81vayKwJeQ7jEdmNVDA==",
+      "version": "6.3.1",
+      "resolved": "https://registry.npmjs.org/@codemirror/lang-css/-/lang-css-6.3.1.tgz",
+      "integrity": "sha512-kr5fwBGiGtmz6l0LSJIbno9QrifNMUusivHbnA1H6Dmqy4HZFte3UAICix1VuKo0lMPKQr2rqB+0BkKi/S3Ejg==",
       "license": "MIT",
       "dependencies": {
         "@codemirror/autocomplete": "^6.0.0",
@@ -104,9 +98,9 @@
       }
     },
     "node_modules/@codemirror/language": {
-      "version": "6.10.3",
-      "resolved": "https://registry.npmjs.org/@codemirror/language/-/language-6.10.3.tgz",
-      "integrity": "sha512-kDqEU5sCP55Oabl6E7m5N+vZRoc0iWqgDVhEKifcHzPzjqCegcO4amfrYVL9PmPZpl4G0yjkpTpUO/Ui8CzO8A==",
+      "version": "6.10.7",
+      "resolved": "https://registry.npmjs.org/@codemirror/language/-/language-6.10.7.tgz",
+      "integrity": "sha512-aOswhVOLYhMNeqykt4P7+ukQSpGL0ynZYaEyFDVHE7fl2xgluU3yuE9MdgYNfw6EmaNidoFMIQ2iTh1ADrnT6A==",
       "license": "MIT",
       "dependencies": {
         "@codemirror/state": "^6.0.0",
@@ -118,20 +112,20 @@
       }
     },
     "node_modules/@codemirror/lint": {
-      "version": "6.8.2",
-      "resolved": "https://registry.npmjs.org/@codemirror/lint/-/lint-6.8.2.tgz",
-      "integrity": "sha512-PDFG5DjHxSEjOXk9TQYYVjZDqlZTFaDBfhQixHnQOEVDDNHUbEh/hstAjcQJaA6FQdZTD1hquXTK0rVBLADR1g==",
+      "version": "6.8.4",
+      "resolved": "https://registry.npmjs.org/@codemirror/lint/-/lint-6.8.4.tgz",
+      "integrity": "sha512-u4q7PnZlJUojeRe8FJa/njJcMctISGgPQ4PnWsd9268R4ZTtU+tfFYmwkBvgcrK2+QQ8tYFVALVb5fVJykKc5A==",
       "license": "MIT",
       "dependencies": {
         "@codemirror/state": "^6.0.0",
-        "@codemirror/view": "^6.0.0",
+        "@codemirror/view": "^6.35.0",
         "crelt": "^1.0.5"
       }
     },
     "node_modules/@codemirror/search": {
-      "version": "6.5.7",
-      "resolved": "https://registry.npmjs.org/@codemirror/search/-/search-6.5.7.tgz",
-      "integrity": "sha512-6+iLsXvITWKHYlkgHPCs/qiX4dNzn8N78YfhOFvPtPYCkuXqZq10rAfsUMhOq7O/1VjJqdXRflyExlfVcu/9VQ==",
+      "version": "6.5.8",
+      "resolved": "https://registry.npmjs.org/@codemirror/search/-/search-6.5.8.tgz",
+      "integrity": "sha512-PoWtZvo7c1XFeZWmmyaOp2G0XVbOnm+fJzvghqGAktBW3cufwJUWvSCcNG0ppXiBEM05mZu6RhMtXPv2hpllig==",
       "license": "MIT",
       "dependencies": {
         "@codemirror/state": "^6.0.0",
@@ -140,10 +134,13 @@
       }
     },
     "node_modules/@codemirror/state": {
-      "version": "6.4.1",
-      "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.4.1.tgz",
-      "integrity": "sha512-QkEyUiLhsJoZkbumGZlswmAhA7CBU02Wrz7zvH4SrcifbsqwlXShVXg65f3v/ts57W3dqyamEriMhij1Z3Zz4A==",
-      "license": "MIT"
+      "version": "6.5.0",
+      "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.5.0.tgz",
+      "integrity": "sha512-MwBHVK60IiIHDcoMet78lxt6iw5gJOGSbNbOIVBHWVXIH4/Nq1+GQgLLGgI1KlnN86WDXsPudVaqYHKBIx7Eyw==",
+      "license": "MIT",
+      "dependencies": {
+        "@marijn/find-cluster-break": "^1.0.0"
+      }
     },
     "node_modules/@codemirror/theme-one-dark": {
       "version": "6.1.2",
@@ -158,20 +155,20 @@
       }
     },
     "node_modules/@codemirror/view": {
-      "version": "6.34.3",
-      "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.34.3.tgz",
-      "integrity": "sha512-Ph5d+u8DxIeSgssXEakaakImkzBV4+slwIbcxl9oc9evexJhImeu/G8TK7+zp+IFK9KuJ0BdSn6kTBJeH2CHvA==",
+      "version": "6.36.1",
+      "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.36.1.tgz",
+      "integrity": "sha512-miD1nyT4m4uopZaDdO2uXU/LLHliKNYL9kB1C1wJHrunHLm/rpkb5QVSokqgw9hFqEZakrdlb/VGWX8aYZTslQ==",
       "license": "MIT",
       "dependencies": {
-        "@codemirror/state": "^6.4.0",
+        "@codemirror/state": "^6.5.0",
         "style-mod": "^4.1.0",
         "w3c-keyname": "^2.2.4"
       }
     },
     "node_modules/@jridgewell/gen-mapping": {
-      "version": "0.3.5",
-      "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz",
-      "integrity": "sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==",
+      "version": "0.3.8",
+      "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.8.tgz",
+      "integrity": "sha512-imAbBGkb+ebQyxKgzv5Hu2nmROxoDOXHh80evxdoXNOrvAnVx7zimzc1Oo5h9RlfV4vPXaE2iM5pOFbvOCClWA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -270,9 +267,9 @@
       }
     },
     "node_modules/@lezer/javascript": {
-      "version": "1.4.19",
-      "resolved": "https://registry.npmjs.org/@lezer/javascript/-/javascript-1.4.19.tgz",
-      "integrity": "sha512-j44kbR1QL26l6dMunZ1uhKBFteVGLVCBGNUD2sUaMnic+rbTviVuoK0CD1l9FTW31EueWvFFswCKMH7Z+M3JRA==",
+      "version": "1.4.21",
+      "resolved": "https://registry.npmjs.org/@lezer/javascript/-/javascript-1.4.21.tgz",
+      "integrity": "sha512-lL+1fcuxWYPURMM/oFZLEDm0XuLN128QPV+VuGtKpeaOGdcl9F2LYC3nh1S9LkPqx9M0mndZFdXCipNAZpzIkQ==",
       "license": "MIT",
       "dependencies": {
         "@lezer/common": "^1.2.0",
@@ -300,10 +297,16 @@
         "@lezer/common": "^1.0.0"
       }
     },
+    "node_modules/@marijn/find-cluster-break": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@marijn/find-cluster-break/-/find-cluster-break-1.0.2.tgz",
+      "integrity": "sha512-l0h88YhZFyKdXIFNfSWpyjStDjGHwZ/U7iobcK1cQQD8sejsONdQtTVU+1wVN1PBw40PiiHB1vA5S7VTfQiP9g==",
+      "license": "MIT"
+    },
     "node_modules/@rollup/plugin-node-resolve": {
-      "version": "15.3.0",
-      "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-15.3.0.tgz",
-      "integrity": "sha512-9eO5McEICxMzJpDW9OnMYSv4Sta3hmt7VtBFz5zR9273suNOydOyq/FrGeGy+KsTRFm8w0SLVhzig2ILFT63Ag==",
+      "version": "15.3.1",
+      "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-15.3.1.tgz",
+      "integrity": "sha512-tgg6b91pAybXHJQMAAwW9VuWBO6Thi+q7BCNARLwSqlmsHz0XYURtGvh/AuwSADXSI4h/2uHbs7s4FzlZDGSGA==",
       "license": "MIT",
       "dependencies": {
         "@rollup/pluginutils": "^5.0.1",
@@ -348,9 +351,9 @@
       }
     },
     "node_modules/@rollup/pluginutils": {
-      "version": "5.1.3",
-      "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.1.3.tgz",
-      "integrity": "sha512-Pnsb6f32CD2W3uCaLZIzDmeFyQ2b8UWMFI7xtwUezpcGBDVDW6y9XgAWIlARiGAo6eNF5FK5aQTr0LFyNyqq5A==",
+      "version": "5.1.4",
+      "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.1.4.tgz",
+      "integrity": "sha512-USm05zrsFxYLPdWWq+K3STlWiT/3ELn3RcV5hJMghpeAIhxfsUIg6mt12CBJBInWMV4VneoV7SfGv8xIwo2qNQ==",
       "license": "MIT",
       "dependencies": {
         "@types/estree": "^1.0.0",
@@ -370,9 +373,9 @@
       }
     },
     "node_modules/@rollup/rollup-android-arm-eabi": {
-      "version": "4.27.3",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.27.3.tgz",
-      "integrity": "sha512-EzxVSkIvCFxUd4Mgm4xR9YXrcp976qVaHnqom/Tgm+vU79k4vV4eYTjmRvGfeoW8m9LVcsAy/lGjcgVegKEhLQ==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.29.1.tgz",
+      "integrity": "sha512-ssKhA8RNltTZLpG6/QNkCSge+7mBQGUqJRisZ2MDQcEGaK93QESEgWK2iOpIDZ7k9zPVkG5AS3ksvD5ZWxmItw==",
       "cpu": [
         "arm"
       ],
@@ -383,9 +386,9 @@
       ]
     },
     "node_modules/@rollup/rollup-android-arm64": {
-      "version": "4.27.3",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.27.3.tgz",
-      "integrity": "sha512-LJc5pDf1wjlt9o/Giaw9Ofl+k/vLUaYsE2zeQGH85giX2F+wn/Cg8b3c5CDP3qmVmeO5NzwVUzQQxwZvC2eQKw==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.29.1.tgz",
+      "integrity": "sha512-CaRfrV0cd+NIIcVVN/jx+hVLN+VRqnuzLRmfmlzpOzB87ajixsN/+9L5xNmkaUUvEbI5BmIKS+XTwXsHEb65Ew==",
       "cpu": [
         "arm64"
       ],
@@ -396,9 +399,9 @@
       ]
     },
     "node_modules/@rollup/rollup-darwin-arm64": {
-      "version": "4.27.3",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.27.3.tgz",
-      "integrity": "sha512-OuRysZ1Mt7wpWJ+aYKblVbJWtVn3Cy52h8nLuNSzTqSesYw1EuN6wKp5NW/4eSre3mp12gqFRXOKTcN3AI3LqA==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.29.1.tgz",
+      "integrity": "sha512-2ORr7T31Y0Mnk6qNuwtyNmy14MunTAMx06VAPI6/Ju52W10zk1i7i5U3vlDRWjhOI5quBcrvhkCHyF76bI7kEw==",
       "cpu": [
         "arm64"
       ],
@@ -409,9 +412,9 @@
       ]
     },
     "node_modules/@rollup/rollup-darwin-x64": {
-      "version": "4.27.3",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.27.3.tgz",
-      "integrity": "sha512-xW//zjJMlJs2sOrCmXdB4d0uiilZsOdlGQIC/jjmMWT47lkLLoB1nsNhPUcnoqyi5YR6I4h+FjBpILxbEy8JRg==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.29.1.tgz",
+      "integrity": "sha512-j/Ej1oanzPjmN0tirRd5K2/nncAhS9W6ICzgxV+9Y5ZsP0hiGhHJXZ2JQ53iSSjj8m6cRY6oB1GMzNn2EUt6Ng==",
       "cpu": [
         "x64"
       ],
@@ -422,9 +425,9 @@
       ]
     },
     "node_modules/@rollup/rollup-freebsd-arm64": {
-      "version": "4.27.3",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.27.3.tgz",
-      "integrity": "sha512-58E0tIcwZ+12nK1WiLzHOD8I0d0kdrY/+o7yFVPRHuVGY3twBwzwDdTIBGRxLmyjciMYl1B/U515GJy+yn46qw==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.29.1.tgz",
+      "integrity": "sha512-91C//G6Dm/cv724tpt7nTyP+JdN12iqeXGFM1SqnljCmi5yTXriH7B1r8AD9dAZByHpKAumqP1Qy2vVNIdLZqw==",
       "cpu": [
         "arm64"
       ],
@@ -435,9 +438,9 @@
       ]
     },
     "node_modules/@rollup/rollup-freebsd-x64": {
-      "version": "4.27.3",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.27.3.tgz",
-      "integrity": "sha512-78fohrpcVwTLxg1ZzBMlwEimoAJmY6B+5TsyAZ3Vok7YabRBUvjYTsRXPTjGEvv/mfgVBepbW28OlMEz4w8wGA==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.29.1.tgz",
+      "integrity": "sha512-hEioiEQ9Dec2nIRoeHUP6hr1PSkXzQaCUyqBDQ9I9ik4gCXQZjJMIVzoNLBRGet+hIUb3CISMh9KXuCcWVW/8w==",
       "cpu": [
         "x64"
       ],
@@ -448,9 +451,9 @@
       ]
     },
     "node_modules/@rollup/rollup-linux-arm-gnueabihf": {
-      "version": "4.27.3",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.27.3.tgz",
-      "integrity": "sha512-h2Ay79YFXyQi+QZKo3ISZDyKaVD7uUvukEHTOft7kh00WF9mxAaxZsNs3o/eukbeKuH35jBvQqrT61fzKfAB/Q==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.29.1.tgz",
+      "integrity": "sha512-Py5vFd5HWYN9zxBv3WMrLAXY3yYJ6Q/aVERoeUFwiDGiMOWsMs7FokXihSOaT/PMWUty/Pj60XDQndK3eAfE6A==",
       "cpu": [
         "arm"
       ],
@@ -461,9 +464,9 @@
       ]
     },
     "node_modules/@rollup/rollup-linux-arm-musleabihf": {
-      "version": "4.27.3",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.27.3.tgz",
-      "integrity": "sha512-Sv2GWmrJfRY57urktVLQ0VKZjNZGogVtASAgosDZ1aUB+ykPxSi3X1nWORL5Jk0sTIIwQiPH7iE3BMi9zGWfkg==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.29.1.tgz",
+      "integrity": "sha512-RiWpGgbayf7LUcuSNIbahr0ys2YnEERD4gYdISA06wa0i8RALrnzflh9Wxii7zQJEB2/Eh74dX4y/sHKLWp5uQ==",
       "cpu": [
         "arm"
       ],
@@ -474,9 +477,9 @@
       ]
     },
     "node_modules/@rollup/rollup-linux-arm64-gnu": {
-      "version": "4.27.3",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.27.3.tgz",
-      "integrity": "sha512-FPoJBLsPW2bDNWjSrwNuTPUt30VnfM8GPGRoLCYKZpPx0xiIEdFip3dH6CqgoT0RnoGXptaNziM0WlKgBc+OWQ==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.29.1.tgz",
+      "integrity": "sha512-Z80O+taYxTQITWMjm/YqNoe9d10OX6kDh8X5/rFCMuPqsKsSyDilvfg+vd3iXIqtfmp+cnfL1UrYirkaF8SBZA==",
       "cpu": [
         "arm64"
       ],
@@ -487,9 +490,9 @@
       ]
     },
     "node_modules/@rollup/rollup-linux-arm64-musl": {
-      "version": "4.27.3",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.27.3.tgz",
-      "integrity": "sha512-TKxiOvBorYq4sUpA0JT+Fkh+l+G9DScnG5Dqx7wiiqVMiRSkzTclP35pE6eQQYjP4Gc8yEkJGea6rz4qyWhp3g==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.29.1.tgz",
+      "integrity": "sha512-fOHRtF9gahwJk3QVp01a/GqS4hBEZCV1oKglVVq13kcK3NeVlS4BwIFzOHDbmKzt3i0OuHG4zfRP0YoG5OF/rA==",
       "cpu": [
         "arm64"
       ],
@@ -499,10 +502,23 @@
         "linux"
       ]
     },
+    "node_modules/@rollup/rollup-linux-loongarch64-gnu": {
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loongarch64-gnu/-/rollup-linux-loongarch64-gnu-4.29.1.tgz",
+      "integrity": "sha512-5a7q3tnlbcg0OodyxcAdrrCxFi0DgXJSoOuidFUzHZ2GixZXQs6Tc3CHmlvqKAmOs5eRde+JJxeIf9DonkmYkw==",
+      "cpu": [
+        "loong64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
     "node_modules/@rollup/rollup-linux-powerpc64le-gnu": {
-      "version": "4.27.3",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.27.3.tgz",
-      "integrity": "sha512-v2M/mPvVUKVOKITa0oCFksnQQ/TqGrT+yD0184/cWHIu0LoIuYHwox0Pm3ccXEz8cEQDLk6FPKd1CCm+PlsISw==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.29.1.tgz",
+      "integrity": "sha512-9b4Mg5Yfz6mRnlSPIdROcfw1BU22FQxmfjlp/CShWwO3LilKQuMISMTtAu/bxmmrE6A902W2cZJuzx8+gJ8e9w==",
       "cpu": [
         "ppc64"
       ],
@@ -513,9 +529,9 @@
       ]
     },
     "node_modules/@rollup/rollup-linux-riscv64-gnu": {
-      "version": "4.27.3",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.27.3.tgz",
-      "integrity": "sha512-LdrI4Yocb1a/tFVkzmOE5WyYRgEBOyEhWYJe4gsDWDiwnjYKjNs7PS6SGlTDB7maOHF4kxevsuNBl2iOcj3b4A==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.29.1.tgz",
+      "integrity": "sha512-G5pn0NChlbRM8OJWpJFMX4/i8OEU538uiSv0P6roZcbpe/WfhEO+AT8SHVKfp8qhDQzaz7Q+1/ixMy7hBRidnQ==",
       "cpu": [
         "riscv64"
       ],
@@ -526,9 +542,9 @@
       ]
     },
     "node_modules/@rollup/rollup-linux-s390x-gnu": {
-      "version": "4.27.3",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.27.3.tgz",
-      "integrity": "sha512-d4wVu6SXij/jyiwPvI6C4KxdGzuZOvJ6y9VfrcleHTwo68fl8vZC5ZYHsCVPUi4tndCfMlFniWgwonQ5CUpQcA==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.29.1.tgz",
+      "integrity": "sha512-WM9lIkNdkhVwiArmLxFXpWndFGuOka4oJOZh8EP3Vb8q5lzdSCBuhjavJsw68Q9AKDGeOOIHYzYm4ZFvmWez5g==",
       "cpu": [
         "s390x"
       ],
@@ -539,9 +555,9 @@
       ]
     },
     "node_modules/@rollup/rollup-linux-x64-gnu": {
-      "version": "4.27.3",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.27.3.tgz",
-      "integrity": "sha512-/6bn6pp1fsCGEY5n3yajmzZQAh+mW4QPItbiWxs69zskBzJuheb3tNynEjL+mKOsUSFK11X4LYF2BwwXnzWleA==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.29.1.tgz",
+      "integrity": "sha512-87xYCwb0cPGZFoGiErT1eDcssByaLX4fc0z2nRM6eMtV9njAfEE6OW3UniAoDhX4Iq5xQVpE6qO9aJbCFumKYQ==",
       "cpu": [
         "x64"
       ],
@@ -552,9 +568,9 @@
       ]
     },
     "node_modules/@rollup/rollup-linux-x64-musl": {
-      "version": "4.27.3",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.27.3.tgz",
-      "integrity": "sha512-nBXOfJds8OzUT1qUreT/en3eyOXd2EH5b0wr2bVB5999qHdGKkzGzIyKYaKj02lXk6wpN71ltLIaQpu58YFBoQ==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.29.1.tgz",
+      "integrity": "sha512-xufkSNppNOdVRCEC4WKvlR1FBDyqCSCpQeMMgv9ZyXqqtKBfkw1yfGMTUTs9Qsl6WQbJnsGboWCp7pJGkeMhKA==",
       "cpu": [
         "x64"
       ],
@@ -565,9 +581,9 @@
       ]
     },
     "node_modules/@rollup/rollup-win32-arm64-msvc": {
-      "version": "4.27.3",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.27.3.tgz",
-      "integrity": "sha512-ogfbEVQgIZOz5WPWXF2HVb6En+kWzScuxJo/WdQTqEgeyGkaa2ui5sQav9Zkr7bnNCLK48uxmmK0TySm22eiuw==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.29.1.tgz",
+      "integrity": "sha512-F2OiJ42m77lSkizZQLuC+jiZ2cgueWQL5YC9tjo3AgaEw+KJmVxHGSyQfDUoYR9cci0lAywv2Clmckzulcq6ig==",
       "cpu": [
         "arm64"
       ],
@@ -578,9 +594,9 @@
       ]
     },
     "node_modules/@rollup/rollup-win32-ia32-msvc": {
-      "version": "4.27.3",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.27.3.tgz",
-      "integrity": "sha512-ecE36ZBMLINqiTtSNQ1vzWc5pXLQHlf/oqGp/bSbi7iedcjcNb6QbCBNG73Euyy2C+l/fn8qKWEwxr+0SSfs3w==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.29.1.tgz",
+      "integrity": "sha512-rYRe5S0FcjlOBZQHgbTKNrqxCBUmgDJem/VQTCcTnA2KCabYSWQDrytOzX7avb79cAAweNmMUb/Zw18RNd4mng==",
       "cpu": [
         "ia32"
       ],
@@ -591,9 +607,9 @@
       ]
     },
     "node_modules/@rollup/rollup-win32-x64-msvc": {
-      "version": "4.27.3",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.27.3.tgz",
-      "integrity": "sha512-vliZLrDmYKyaUoMzEbMTg2JkerfBjn03KmAw9CykO0Zzkzoyd7o3iZNam/TpyWNjNT+Cz2iO3P9Smv2wgrR+Eg==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.29.1.tgz",
+      "integrity": "sha512-+10CMg9vt1MoHj6x1pxyjPSMjHTIlqs8/tBztXvPAx24SKs9jwVnKqHJumlH/IzhaPUaj3T6T6wfZr8okdXaIg==",
       "cpu": [
         "x64"
       ],
@@ -714,9 +730,9 @@
       }
     },
     "node_modules/is-core-module": {
-      "version": "2.15.1",
-      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.15.1.tgz",
-      "integrity": "sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ==",
+      "version": "2.16.1",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz",
+      "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==",
       "license": "MIT",
       "dependencies": {
         "hasown": "^2.0.2"
@@ -763,26 +779,29 @@
       }
     },
     "node_modules/resolve": {
-      "version": "1.22.8",
-      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz",
-      "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==",
+      "version": "1.22.10",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.10.tgz",
+      "integrity": "sha512-NPRy+/ncIMeDlTAsuqwKIiferiawhefFJtkNSW0qZJEqMEb+qBt/77B/jGeeek+F0uOeN05CDa6HXbbIgtVX4w==",
       "license": "MIT",
       "dependencies": {
-        "is-core-module": "^2.13.0",
+        "is-core-module": "^2.16.0",
         "path-parse": "^1.0.7",
         "supports-preserve-symlinks-flag": "^1.0.0"
       },
       "bin": {
         "resolve": "bin/resolve"
       },
+      "engines": {
+        "node": ">= 0.4"
+      },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
     },
     "node_modules/rollup": {
-      "version": "4.27.3",
-      "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.27.3.tgz",
-      "integrity": "sha512-SLsCOnlmGt9VoZ9Ek8yBK8tAdmPHeppkw+Xa7yDlCEhDTvwYei03JlWo1fdc7YTfLZ4tD8riJCUyAgTbszk1fQ==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.29.1.tgz",
+      "integrity": "sha512-RaJ45M/kmJUzSWDs1Nnd5DdV4eerC98idtUOVr6FfKcgxqvjwHmxc5upLF9qZU9EpsVzzhleFahrT3shLuJzIw==",
       "license": "MIT",
       "dependencies": {
         "@types/estree": "1.0.6"
@@ -795,24 +814,25 @@
         "npm": ">=8.0.0"
       },
       "optionalDependencies": {
-        "@rollup/rollup-android-arm-eabi": "4.27.3",
-        "@rollup/rollup-android-arm64": "4.27.3",
-        "@rollup/rollup-darwin-arm64": "4.27.3",
-        "@rollup/rollup-darwin-x64": "4.27.3",
-        "@rollup/rollup-freebsd-arm64": "4.27.3",
-        "@rollup/rollup-freebsd-x64": "4.27.3",
-        "@rollup/rollup-linux-arm-gnueabihf": "4.27.3",
-        "@rollup/rollup-linux-arm-musleabihf": "4.27.3",
-        "@rollup/rollup-linux-arm64-gnu": "4.27.3",
-        "@rollup/rollup-linux-arm64-musl": "4.27.3",
-        "@rollup/rollup-linux-powerpc64le-gnu": "4.27.3",
-        "@rollup/rollup-linux-riscv64-gnu": "4.27.3",
-        "@rollup/rollup-linux-s390x-gnu": "4.27.3",
-        "@rollup/rollup-linux-x64-gnu": "4.27.3",
-        "@rollup/rollup-linux-x64-musl": "4.27.3",
-        "@rollup/rollup-win32-arm64-msvc": "4.27.3",
-        "@rollup/rollup-win32-ia32-msvc": "4.27.3",
-        "@rollup/rollup-win32-x64-msvc": "4.27.3",
+        "@rollup/rollup-android-arm-eabi": "4.29.1",
+        "@rollup/rollup-android-arm64": "4.29.1",
+        "@rollup/rollup-darwin-arm64": "4.29.1",
+        "@rollup/rollup-darwin-x64": "4.29.1",
+        "@rollup/rollup-freebsd-arm64": "4.29.1",
+        "@rollup/rollup-freebsd-x64": "4.29.1",
+        "@rollup/rollup-linux-arm-gnueabihf": "4.29.1",
+        "@rollup/rollup-linux-arm-musleabihf": "4.29.1",
+        "@rollup/rollup-linux-arm64-gnu": "4.29.1",
+        "@rollup/rollup-linux-arm64-musl": "4.29.1",
+        "@rollup/rollup-linux-loongarch64-gnu": "4.29.1",
+        "@rollup/rollup-linux-powerpc64le-gnu": "4.29.1",
+        "@rollup/rollup-linux-riscv64-gnu": "4.29.1",
+        "@rollup/rollup-linux-s390x-gnu": "4.29.1",
+        "@rollup/rollup-linux-x64-gnu": "4.29.1",
+        "@rollup/rollup-linux-x64-musl": "4.29.1",
+        "@rollup/rollup-win32-arm64-msvc": "4.29.1",
+        "@rollup/rollup-win32-ia32-msvc": "4.29.1",
+        "@rollup/rollup-win32-x64-msvc": "4.29.1",
         "fsevents": "~2.3.2"
       }
     },
@@ -894,9 +914,9 @@
       }
     },
     "node_modules/terser": {
-      "version": "5.36.0",
-      "resolved": "https://registry.npmjs.org/terser/-/terser-5.36.0.tgz",
-      "integrity": "sha512-IYV9eNMuFAV4THUspIRXkLakHnV6XO7FEdtKjf/mDyrnqUg9LnlOn6/RwRvM9SZjR4GUq8Nk8zj67FzVARr74w==",
+      "version": "5.37.0",
+      "resolved": "https://registry.npmjs.org/terser/-/terser-5.37.0.tgz",
+      "integrity": "sha512-B8wRRkmre4ERucLM/uXx4MOV5cbnOlVAqUst+1+iLKPI0dOgFO28f84ptoQt9HEI537PMzfYa/d+GEPKTRXmYA==",
       "dev": true,
       "license": "BSD-2-Clause",
       "dependencies": {

deps/sqlite/shell.c

@@ -5072,10 +5072,10 @@ int sqlite3_percentile_init(
 ){
   int rc = SQLITE_OK;
   unsigned int i;
-#if defined(SQLITE3_H) || defined(SQLITE_STATIC_PERCENTILE)
-  (void)pApi;      /* Unused parameter */
-#else
+#ifdef SQLITE3EXT_H
   SQLITE_EXTENSION_INIT2(pApi);
+#else
+  (void)pApi;      /* Unused parameter */
 #endif
   (void)pzErrMsg;  /* Unused parameter */
   for(i=0; i<sizeof(aPercentFunc)/sizeof(aPercentFunc[0]); i++){

deps/sqlite/sqlite3.c

@@ -1,6 +1,6 @@
 /******************************************************************************
 ** This file is an amalgamation of many separate C source files from SQLite
-** version 3.47.1.  By combining all the individual C code files into this
+** version 3.47.2.  By combining all the individual C code files into this
 ** single large file, the entire code can be compiled as a single translation
 ** unit.  This allows many compilers to do optimizations that would not be
 ** possible if the files were compiled separately.  Performance improvements
@@ -18,7 +18,7 @@
 ** separate file. This file contains only code for the core SQLite library.
 **
 ** The content in this amalgamation comes from Fossil check-in
-** b95d11e958643b969c47a8e5857f3793b9e6.
+** 2aabe05e2e8cae4847a802ee2daddc1d7413.
 */
 #define SQLITE_CORE 1
 #define SQLITE_AMALGAMATION 1
@@ -462,9 +462,9 @@ extern "C" {
 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
 ** [sqlite_version()] and [sqlite_source_id()].
 */
-#define SQLITE_VERSION        "3.47.1"
-#define SQLITE_VERSION_NUMBER 3047001
-#define SQLITE_SOURCE_ID      "2024-11-25 12:07:48 b95d11e958643b969c47a8e5857f3793b9e69700b8f1469371386369a26e577e"
+#define SQLITE_VERSION        "3.47.2"
+#define SQLITE_VERSION_NUMBER 3047002
+#define SQLITE_SOURCE_ID      "2024-12-07 20:39:59 2aabe05e2e8cae4847a802ee2daddc1d7413d8fc560254d93ee3e72c14685b6c"
 
 /*
 ** CAPI3REF: Run-Time Library Version Numbers
@@ -35697,8 +35697,8 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en
   int eValid = 1;  /* True exponent is either not used or is well-formed */
   int nDigit = 0;  /* Number of digits processed */
   int eType = 1;   /* 1: pure integer,  2+: fractional  -1 or less: bad UTF16 */
+  u64 s2;          /* round-tripped significand */
   double rr[2];
-  u64 s2;
 
   assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE );
   *pResult = 0.0;   /* Default return value, in case of an error */
@@ -35801,7 +35801,7 @@ do_atof_calc:
   e = (e*esign) + d;
 
   /* Try to adjust the exponent to make it smaller */
-  while( e>0 && s<(LARGEST_UINT64/10) ){
+  while( e>0 && s<((LARGEST_UINT64-0x7ff)/10) ){
     s *= 10;
     e--;
   }
@@ -35811,11 +35811,16 @@ do_atof_calc:
   }
 
   rr[0] = (double)s;
-  s2 = (u64)rr[0];
-#if defined(_MSC_VER) && _MSC_VER<1700
-  if( s2==0x8000000000000000LL ){ s2 = 2*(u64)(0.5*rr[0]); }
-#endif
-  rr[1] = s>=s2 ? (double)(s - s2) : -(double)(s2 - s);
+  assert( sizeof(s2)==sizeof(rr[0]) );
+  memcpy(&s2, &rr[0], sizeof(s2));
+  if( s2<=0x43efffffffffffffLL ){
+    s2 = (u64)rr[0];
+    rr[1] = s>=s2 ? (double)(s - s2) : -(double)(s2 - s);
+  }else{
+    rr[1] = 0.0;
+  }
+  assert( rr[1]<=1.0e-10*rr[0] );  /* Equal only when rr[0]==0.0 */
+
   if( e>0 ){
     while( e>=100  ){
       e -= 100;
@@ -147605,32 +147610,32 @@ static Expr *substExpr(
         if( pSubst->isOuterJoin ){
           ExprSetProperty(pNew, EP_CanBeNull);
         }
+        if( pNew->op==TK_TRUEFALSE ){
+          pNew->u.iValue = sqlite3ExprTruthValue(pNew);
+          pNew->op = TK_INTEGER;
+          ExprSetProperty(pNew, EP_IntValue);
+        }
+
+        /* Ensure that the expression now has an implicit collation sequence,
+        ** just as it did when it was a column of a view or sub-query. */
+        {
+          CollSeq *pNat = sqlite3ExprCollSeq(pSubst->pParse, pNew);
+          CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse,
+                pSubst->pCList->a[iColumn].pExpr
+          );
+          if( pNat!=pColl || (pNew->op!=TK_COLUMN && pNew->op!=TK_COLLATE) ){
+            pNew = sqlite3ExprAddCollateString(pSubst->pParse, pNew,
+                (pColl ? pColl->zName : "BINARY")
+            );
+          }
+        }
+        ExprClearProperty(pNew, EP_Collate);
         if( ExprHasProperty(pExpr,EP_OuterON|EP_InnerON) ){
           sqlite3SetJoinExpr(pNew, pExpr->w.iJoin,
                              pExpr->flags & (EP_OuterON|EP_InnerON));
         }
         sqlite3ExprDelete(db, pExpr);
         pExpr = pNew;
-        if( pExpr->op==TK_TRUEFALSE ){
-          pExpr->u.iValue = sqlite3ExprTruthValue(pExpr);
-          pExpr->op = TK_INTEGER;
-          ExprSetProperty(pExpr, EP_IntValue);
-        }
-
-        /* Ensure that the expression now has an implicit collation sequence,
-        ** just as it did when it was a column of a view or sub-query. */
-        {
-          CollSeq *pNat = sqlite3ExprCollSeq(pSubst->pParse, pExpr);
-          CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse,
-                pSubst->pCList->a[iColumn].pExpr
-          );
-          if( pNat!=pColl || (pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE) ){
-            pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr,
-                (pColl ? pColl->zName : "BINARY")
-            );
-          }
-        }
-        ExprClearProperty(pExpr, EP_Collate);
       }
     }
   }else{
@@ -254938,7 +254943,7 @@ static void fts5SourceIdFunc(
 ){
   assert( nArg==0 );
   UNUSED_PARAM2(nArg, apUnused);
-  sqlite3_result_text(pCtx, "fts5: 2024-11-25 12:07:48 b95d11e958643b969c47a8e5857f3793b9e69700b8f1469371386369a26e577e", -1, SQLITE_TRANSIENT);
+  sqlite3_result_text(pCtx, "fts5: 2024-12-07 20:39:59 2aabe05e2e8cae4847a802ee2daddc1d7413d8fc560254d93ee3e72c14685b6c", -1, SQLITE_TRANSIENT);
 }
 
 /*

deps/sqlite/sqlite3.h

@@ -146,9 +146,9 @@ extern "C" {
 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
 ** [sqlite_version()] and [sqlite_source_id()].
 */
-#define SQLITE_VERSION        "3.47.1"
-#define SQLITE_VERSION_NUMBER 3047001
-#define SQLITE_SOURCE_ID      "2024-11-25 12:07:48 b95d11e958643b969c47a8e5857f3793b9e69700b8f1469371386369a26e577e"
+#define SQLITE_VERSION        "3.47.2"
+#define SQLITE_VERSION_NUMBER 3047002
+#define SQLITE_SOURCE_ID      "2024-12-07 20:39:59 2aabe05e2e8cae4847a802ee2daddc1d7413d8fc560254d93ee3e72c14685b6c"
 
 /*
 ** CAPI3REF: Run-Time Library Version Numbers

metadata/en-US/changelogs/31.txt

@@ -0,0 +1,15 @@
+* Add channels to the ssb user interface.
+* Confirm setting global settings.
+* Fix the wiki app.
+* Multiple crash fixes, especially on Android.
+* Following depth fixes.
+* Try harder to request profile images.
+* Updated the emoji list used for reactions.
+* Admin users can encrypt/decrypt using the server identity.
+* Request more messages as the follow graph changes.
+* Add a store_blob cli option.
+* Updated dependencies:
+  * CodeMirror
+  * Prettier
+  * sqlite 3.47.2
+  * c-ares 1.34.4

package-lock.json

@@ -11,9 +11,9 @@
 			}
 		},
 		"node_modules/prettier": {
-			"version": "3.3.3",
-			"resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.3.tgz",
-			"integrity": "sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==",
+			"version": "3.4.2",
+			"resolved": "https://registry.npmjs.org/prettier/-/prettier-3.4.2.tgz",
+			"integrity": "sha512-e9MewbtFo+Fevyuxn/4rrcDAaq0IYxPGLvObpQjiZBMAzB9IGmzlnG9RZy3FFas+eBMu2vA0CszMeduow5dIuQ==",
 			"license": "MIT",
 			"bin": {
 				"prettier": "bin/prettier.cjs"

src/android/AndroidManifest.xml

@@ -1,8 +1,8 @@
 <?xml version="1.0" encoding="utf-8"?>
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
 	package="com.unprompted.tildefriends"
-	android:versionCode="30"
-	android:versionName="0.0.25">
+	android:versionCode="31"
+	android:versionName="0.0.26">
 	<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE"/>
 	<uses-permission android:name="android.permission.INTERNET"/>
 	<application

src/android/com/unprompted/tildefriends/TildeFriendsActivity.java

@@ -385,8 +385,11 @@ public class TildeFriendsActivity extends Activity {
 			public void onServiceConnected(ComponentName name, IBinder binder) {
 				Log.w("tildefriends", "onServiceConnected");
 				Parcel data = Parcel.obtain();
-				ParcelFileDescriptor pfd = ParcelFileDescriptor.adoptFd(pipe_fd);
-				data.writeParcelable(pfd, 0);
+				try (ParcelFileDescriptor pfd = ParcelFileDescriptor.fromFd(pipe_fd)) {
+					data.writeParcelable(pfd, 0);
+				} catch (java.io.IOException e) {
+					Log.w("tildefriends", "IOException: " + e);
+				}
 				try {
 					binder.transact(TildeFriendsSandboxService.START_CALL,  data, null, IBinder.FLAG_ONEWAY);
 				} catch (RemoteException e) {

src/httpd.js.c

@@ -1360,7 +1360,7 @@ static void _httpd_endpoint_save_work(tf_ssb_t* ssb, void* user_data)
 		user_app_t* user_app = _parse_user_app_from_path(request->path, "/save");
 		if (user_app)
 		{
-			if (strcmp(user_string, user_app->user) == 0 || (strcmp(user_app->user, "core") == 0 && tf_ssb_db_user_has_permission(ssb, user_string, "administration")))
+			if (strcmp(user_string, user_app->user) == 0 || (strcmp(user_app->user, "core") == 0 && tf_ssb_db_user_has_permission(ssb, NULL, user_string, "administration")))
 			{
 				size_t path_length = strlen("path:") + strlen(user_app->app) + 1;
 				char* app_path = tf_malloc(path_length);
@@ -1516,7 +1516,7 @@ static void _httpd_endpoint_delete_work(tf_ssb_t* ssb, void* user_data)
 		user_app_t* user_app = _parse_user_app_from_path(request->path, "/delete");
 		if (user_app)
 		{
-			if (strcmp(user_string, user_app->user) == 0 || (strcmp(user_app->user, "core") == 0 && tf_ssb_db_user_has_permission(ssb, user_string, "administration")))
+			if (strcmp(user_string, user_app->user) == 0 || (strcmp(user_app->user, "core") == 0 && tf_ssb_db_user_has_permission(ssb, NULL, user_string, "administration")))
 			{
 				size_t path_length = strlen("path:") + strlen(user_app->app) + 1;
 				char* app_path = tf_malloc(path_length);

src/main.c

@@ -49,6 +49,7 @@ static int _tf_command_import(const char* file, int argc, char* argv[]);
 static int _tf_command_publish(const char* file, int argc, char* argv[]);
 static int _tf_command_run(const char* file, int argc, char* argv[]);
 static int _tf_command_sandbox(const char* file, int argc, char* argv[]);
+static int _tf_command_store_blob(const char* file, int argc, char* argv[]);
 static int _tf_command_test(const char* file, int argc, char* argv[]);
 static int _tf_command_verify(const char* file, int argc, char* argv[]);
 static int _tf_command_usage(const char* file);
@@ -66,6 +67,7 @@ const command_t k_commands[] = {
 	{ "import", _tf_command_import, "Import apps to SSB." },
 	{ "export", _tf_command_export, "Export apps from SSB." },
 	{ "publish", _tf_command_publish, "Append a message to a feed." },
+	{ "store_blob", _tf_command_store_blob, "Write a file to the blob store." },
 	{ "verify", _tf_command_verify, "Verify a feed." },
 	{ "test", _tf_command_test, "Test SSB." },
 };
@@ -387,6 +389,96 @@ static int _tf_command_publish(const char* file, int argc, char* argv[])
 	return result;
 }
 
+static int _tf_command_store_blob(const char* file, int argc, char* argv[])
+{
+	const char* db_path = k_db_path_default;
+	const char* file_path = NULL;
+	bool show_usage = false;
+
+	while (!show_usage)
+	{
+		static const struct option k_options[] = {
+			{ "db-path", required_argument, NULL, 'd' },
+			{ "file", required_argument, NULL, 'f' },
+			{ "help", no_argument, NULL, 'h' },
+			{ 0 },
+		};
+		int c = getopt_long(argc, argv, "d:f:h", k_options, NULL);
+		if (c == -1)
+		{
+			break;
+		}
+
+		switch (c)
+		{
+		case '?':
+		case 'h':
+		default:
+			show_usage = true;
+			break;
+		case 'd':
+			db_path = optarg;
+			break;
+		case 'f':
+			file_path = optarg;
+			break;
+		}
+	}
+
+	if (show_usage || !file_path)
+	{
+		tf_printf("\n%s store_blob [options]\n\n", file);
+		tf_printf("options:\n");
+		tf_printf("  -d, --db-path db_path    SQLite database path (default: %s).\n", k_db_path_default);
+		tf_printf("  -f, --file file_path     Path to file to add to the blob store.\n");
+		tf_printf("  -h, --help               Show this usage information.\n");
+		return EXIT_FAILURE;
+	}
+
+	char* data = NULL;
+	size_t size = 0;
+	FILE* blob_file = fopen(file_path, "rb");
+	if (!blob_file)
+	{
+		tf_printf("Failed to open %s: %s.\n", file_path, strerror(errno));
+		return EXIT_FAILURE;
+	}
+
+	char buffer[16 * 1024];
+	while (true)
+	{
+		size_t bytes = fread(buffer, 1, sizeof(buffer), blob_file);
+		if (bytes > 0)
+		{
+			data = tf_resize_vec(data, size + bytes);
+			memcpy(data + size, buffer, bytes);
+			size += bytes;
+		}
+		else
+		{
+			break;
+		}
+	}
+	if (ferror(blob_file))
+	{
+		tf_printf("Failed to read %s: %s.\n", file_path, strerror(errno));
+		fclose(blob_file);
+		tf_free(data);
+		return EXIT_FAILURE;
+	}
+	fclose(blob_file);
+
+	char id[256];
+	tf_ssb_t* ssb = tf_ssb_create(NULL, NULL, db_path, NULL);
+	if (tf_ssb_db_blob_store(ssb, (const uint8_t*)data, size, id, sizeof(id), NULL))
+	{
+		tf_printf("%s\n", id);
+	}
+	tf_ssb_destroy(ssb);
+	tf_free(data);
+	return EXIT_SUCCESS;
+}
+
 static int _tf_command_verify(const char* file, int argc, char* argv[])
 {
 	const char* identity = NULL;

src/packetstream.c

@@ -12,6 +12,8 @@ typedef struct _tf_packetstream_t
 {
 	tf_packetstream_onreceive_t* onreceive;
 	void* onreceive_user_data;
+	tf_packetstream_on_close_t* on_close;
+	void* on_close_user_data;
 	uv_pipe_t stream;
 	char* buffer;
 	size_t buffer_size;
@@ -30,6 +32,8 @@ void tf_packetstream_destroy(tf_packetstream_t* stream)
 {
 	stream->onreceive = NULL;
 	stream->onreceive_user_data = NULL;
+	stream->on_close = NULL;
+	stream->on_close_user_data = NULL;
 	stream->destroyed = true;
 	if (stream->buffer)
 	{
@@ -110,6 +114,14 @@ static void _packetstream_on_read(uv_stream_t* handle, ssize_t count, const uv_b
 	}
 	else
 	{
+		tf_packetstream_on_close_t* on_close = stream->on_close;
+		void* user_data = stream->on_close_user_data;
+		stream->on_close = NULL;
+		stream->on_close_user_data = NULL;
+		if (on_close)
+		{
+			on_close(user_data);
+		}
 		tf_packetstream_close(stream);
 	}
 	tf_free(buffer->base);
@@ -150,7 +162,7 @@ void tf_packetstream_send(tf_packetstream_t* stream, int packet_type, const char
 		int result = uv_write(request, (uv_stream_t*)&stream->stream, &write_buffer, 1, _packetstream_on_write);
 		if (result)
 		{
-			tf_printf("uv_write: %s\n", uv_strerror(result));
+			tf_printf("tf_packetstream_send: uv_write: %s\n", uv_strerror(result));
 			tf_free(request);
 		}
 	}
@@ -162,6 +174,12 @@ void tf_packetstream_set_on_receive(tf_packetstream_t* stream, tf_packetstream_o
 	stream->onreceive_user_data = user_data;
 }
 
+void tf_packetstream_set_on_close(tf_packetstream_t* stream, tf_packetstream_on_close_t* callback, void* user_data)
+{
+	stream->on_close = callback;
+	stream->on_close_user_data = user_data;
+}
+
 static void _tf_packetstream_handle_closed(uv_handle_t* handle)
 {
 	tf_packetstream_t* packetstream = handle->data;

src/packetstream.h

@@ -23,6 +23,12 @@ typedef struct _tf_packetstream_t tf_packetstream_t;
 */
 typedef void(tf_packetstream_onreceive_t)(int packet_type, const char* begin, size_t length, void* user_data);
 
+/**
+** A function called when a packetstream reads EOF.
+** @param user_data User data.
+*/
+typedef void(tf_packetstream_on_close_t)(void* user_data);
+
 /**
 ** Create a packet stream.
 ** @return The packet stream.
@@ -58,6 +64,14 @@ void tf_packetstream_send(tf_packetstream_t* stream, int packet_type, const char
 */
 void tf_packetstream_set_on_receive(tf_packetstream_t* stream, tf_packetstream_onreceive_t* callback, void* user_data);
 
+/**
+** Register a callback for when a stream reads EOF.
+** @param stream The packet stream.
+** @param callback The callback.
+** @param user_data User data to pass to the callback.
+*/
+void tf_packetstream_set_on_close(tf_packetstream_t* stream, tf_packetstream_on_close_t* callback, void* user_data);
+
 /**
 ** Close a packet stream.
 ** @param stream The packet stream.

src/ssb.c

@@ -294,6 +294,7 @@ typedef struct _tf_ssb_connection_t
 	uv_tcp_t tcp;
 	uv_connect_t connect;
 	uv_async_t async;
+	uv_async_t scheduled_async;
 	uv_timer_t handshake_timer;
 	bool closing;
 
@@ -385,6 +386,7 @@ static bool _tf_ssb_parse_broadcast(const char* in_broadcast, tf_ssb_broadcast_t
 static void _tf_ssb_start_update_settings(tf_ssb_t* ssb);
 static void _tf_ssb_update_settings(tf_ssb_t* ssb);
 static void _tf_ssb_write(tf_ssb_connection_t* connection, void* data, size_t size);
+static void _tf_ssb_connection_dispatch_scheduled(tf_ssb_connection_t* connection);
 
 static const char* _tf_ssb_connection_state_to_string(tf_ssb_state_t state)
 {
@@ -663,9 +665,15 @@ static void _tf_ssb_connection_box_stream_send(tf_ssb_connection_t* connection,
 	}
 }
 
+static void _tf_ssb_connection_scheduled_async(uv_async_t* async)
+{
+	tf_ssb_connection_t* connection = async->data;
+	_tf_ssb_connection_dispatch_scheduled(connection);
+}
+
 static void _tf_ssb_connection_dispatch_scheduled(tf_ssb_connection_t* connection)
 {
-	while ((connection->active_write_count == 0 || connection->closing) && connection->scheduled_count && connection->scheduled)
+	while (((connection->active_write_count == 0 && connection->read_back_pressure == 0) || connection->closing) && connection->scheduled_count && connection->scheduled)
 	{
 		tf_ssb_connection_scheduled_t scheduled = connection->scheduled[0];
 		memmove(connection->scheduled, connection->scheduled + 1, sizeof(tf_ssb_connection_scheduled_t) * (connection->scheduled_count - 1));
@@ -683,7 +691,7 @@ void tf_ssb_connection_schedule_idle(tf_ssb_connection_t* connection, tf_ssb_sch
 		.callback = callback,
 		.user_data = user_data,
 	};
-	_tf_ssb_connection_dispatch_scheduled(connection);
+	uv_async_send(&connection->scheduled_async);
 }
 
 static int _request_compare(const void* a, const void* b)
@@ -1988,6 +1996,10 @@ static void _tf_ssb_connection_destroy(tf_ssb_connection_t* connection, const ch
 	{
 		uv_close((uv_handle_t*)&connection->async, _tf_ssb_connection_on_close);
 	}
+	if (connection->scheduled_async.data && !uv_is_closing((uv_handle_t*)&connection->scheduled_async))
+	{
+		uv_close((uv_handle_t*)&connection->scheduled_async, _tf_ssb_connection_on_close);
+	}
 	if (connection->tcp.data && !uv_is_closing((uv_handle_t*)&connection->tcp))
 	{
 		uv_close((uv_handle_t*)&connection->tcp, _tf_ssb_connection_on_close);
@@ -1997,8 +2009,8 @@ static void _tf_ssb_connection_destroy(tf_ssb_connection_t* connection, const ch
 		uv_close((uv_handle_t*)&connection->handshake_timer, _tf_ssb_connection_on_close);
 	}
 
-	if (JS_IsUndefined(connection->object) && !connection->async.data && !connection->tcp.data && !connection->connect.data && !connection->handshake_timer.data &&
-		connection->ref_count == 0)
+	if (JS_IsUndefined(connection->object) && !connection->async.data && !connection->scheduled_async.data && !connection->tcp.data && !connection->connect.data &&
+		!connection->handshake_timer.data && connection->ref_count == 0)
 	{
 		tf_free(connection->message_requests);
 		connection->message_requests = NULL;
@@ -2792,6 +2804,8 @@ static tf_ssb_connection_t* _tf_ssb_connection_create(
 	connection->port = ntohs(addr->sin_port);
 	connection->async.data = connection;
 	uv_async_init(ssb->loop, &connection->async, _tf_ssb_connection_process_message_async);
+	connection->scheduled_async.data = connection;
+	uv_async_init(ssb->loop, &connection->scheduled_async, _tf_ssb_connection_scheduled_async);
 	connection->connect_callback = callback;
 	connection->connect_callback_user_data = user_data;
 
@@ -2848,6 +2862,10 @@ static void _tf_ssb_connection_tunnel_callback(
 tf_ssb_connection_t* tf_ssb_connection_tunnel_create(tf_ssb_t* ssb, const char* portal_id, int32_t request_number, const char* target_id, int connect_flags)
 {
 	tf_ssb_connection_t* connection = tf_ssb_connection_get(ssb, portal_id);
+	if (!connection)
+	{
+		return NULL;
+	}
 
 	JSContext* context = ssb->context;
 	tf_ssb_connection_t* tunnel = tf_malloc(sizeof(tf_ssb_connection_t));
@@ -2861,6 +2879,8 @@ tf_ssb_connection_t* tf_ssb_connection_tunnel_create(tf_ssb_t* ssb, const char*
 	tunnel->send_request_number = 1;
 	tunnel->async.data = tunnel;
 	uv_async_init(ssb->loop, &tunnel->async, _tf_ssb_connection_process_message_async);
+	tunnel->scheduled_async.data = tunnel;
+	uv_async_init(ssb->loop, &tunnel->scheduled_async, _tf_ssb_connection_scheduled_async);
 
 	tunnel->handshake_timer.data = tunnel;
 	uv_timer_init(ssb->loop, &tunnel->handshake_timer);
@@ -2998,6 +3018,8 @@ static void _tf_ssb_on_connection(uv_stream_t* stream, int status)
 	connection->send_request_number = 1;
 	connection->async.data = connection;
 	uv_async_init(ssb->loop, &connection->async, _tf_ssb_connection_process_message_async);
+	connection->scheduled_async.data = connection;
+	uv_async_init(ssb->loop, &connection->scheduled_async, _tf_ssb_connection_scheduled_async);
 
 	connection->object = JS_NewObjectClass(ssb->context, _connection_class_id);
 	JS_SetOpaque(connection->object, connection);
@@ -4376,6 +4398,7 @@ void tf_ssb_connection_adjust_read_backpressure(tf_ssb_connection_t* connection,
 	const int k_threshold = 256;
 	int old_pressure = connection->read_back_pressure;
 	connection->read_back_pressure += delta;
+	uv_async_send(&connection->scheduled_async);
 	if (!connection->closing)
 	{
 		if (old_pressure < k_threshold && connection->read_back_pressure >= k_threshold)
@@ -4397,7 +4420,7 @@ void tf_ssb_connection_adjust_read_backpressure(tf_ssb_connection_t* connection,
 void tf_ssb_connection_adjust_write_count(tf_ssb_connection_t* connection, int delta)
 {
 	connection->active_write_count += delta;
-	_tf_ssb_connection_dispatch_scheduled(connection);
+	uv_async_send(&connection->scheduled_async);
 }
 
 void tf_ssb_sync_start(tf_ssb_t* ssb)
@@ -4427,8 +4450,7 @@ bool tf_ssb_tunnel_create(tf_ssb_t* ssb, const char* portal_id, const char* targ
 		tf_ssb_connection_rpc_send_json(connection, k_ssb_rpc_flag_stream | k_ssb_rpc_flag_new_request, request_number, "tunnel.connect", message, NULL, NULL, NULL);
 		JS_FreeValue(context, message);
 
-		tf_ssb_connection_tunnel_create(ssb, portal_id, request_number, target_id, connect_flags);
-		return true;
+		return tf_ssb_connection_tunnel_create(ssb, portal_id, request_number, target_id, connect_flags) != NULL;
 	}
 	return false;
 }

src/ssb.db.c

@@ -135,6 +135,7 @@ void tf_ssb_db_init(tf_ssb_t* ssb)
 	_tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_timestamp_index ON messages (timestamp)");
 	_tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_type_timestamp_index ON messages (content ->> 'type', timestamp)");
 	_tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_size_by_author_index ON messages (author, length(content))");
+	_tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_type_author_channel_index ON messages (content ->> 'type', author, content ->> 'channel');");
 	_tf_ssb_db_exec(db,
 		"CREATE TABLE IF NOT EXISTS blobs ("
 		"  id TEXT PRIMARY KEY,"
@@ -233,13 +234,21 @@ void tf_ssb_db_init(tf_ssb_t* ssb)
 	_tf_ssb_db_exec(db, "DROP VIEW IF EXISTS blob_wants_view");
 	_tf_ssb_db_exec(db,
 		"CREATE VIEW IF NOT EXISTS blob_wants_view (id, timestamp) AS "
-		"  SELECT messages_refs.ref AS id, messages.timestamp AS timestamp "
-		"  FROM messages_refs "
-		"  JOIN messages ON messages.id = messages_refs.message "
-		"  LEFT OUTER JOIN blobs ON messages_refs.ref = blobs.id "
+		"  WITH wanted AS ( "
+		"    SELECT messages_refs.ref AS id, messages.timestamp AS timestamp "
+		"    FROM messages_refs "
+		"    JOIN messages ON messages.id = messages_refs.message "
+		"    UNION "
+		"    SELECT messages_refs.ref AS id, unixepoch() * 1000 AS timestamp "
+		"    FROM messages_refs "
+		"    JOIN messages ON messages.id = messages_refs.message "
+		"    WHERE messages.content ->> 'type' = 'about' "
+		"  ) "
+		"  SELECT wanted.id, wanted.timestamp FROM wanted "
+		"  LEFT OUTER JOIN blobs ON wanted.id = blobs.id "
 		"  WHERE blobs.id IS NULL "
-		"    AND LENGTH(messages_refs.ref) = 52 "
-		"    AND messages_refs.ref LIKE '&%.sha256'");
+		"    AND LENGTH(wanted.id) = 52 "
+		"    AND wanted.id LIKE '&%.sha256'");
 
 	bool need_add_flags = true;
 	bool need_convert_timestamp_to_real = false;
@@ -1998,12 +2007,12 @@ bool tf_ssb_db_verify(tf_ssb_t* ssb, const char* id)
 	return verified;
 }
 
-bool tf_ssb_db_user_has_permission(tf_ssb_t* ssb, const char* id, const char* permission)
+bool tf_ssb_db_user_has_permission(tf_ssb_t* ssb, sqlite3* db, const char* id, const char* permission)
 {
 	bool has_permission = false;
-	sqlite3* db = tf_ssb_acquire_db_reader(ssb);
+	sqlite3* reader = db ? db : tf_ssb_acquire_db_reader(ssb);
 	sqlite3_stmt* statement = NULL;
-	if (sqlite3_prepare(db,
+	if (sqlite3_prepare(reader,
 			"SELECT COUNT(*) FROM properties, json_each(properties.value -> 'permissions' -> ?) AS permission WHERE properties.id = 'core' AND properties.key = 'settings' AND "
 			"permission.value = ?",
 			-1, &statement, NULL) == SQLITE_OK)
@@ -2015,6 +2024,9 @@ bool tf_ssb_db_user_has_permission(tf_ssb_t* ssb, const char* id, const char* pe
 		}
 		sqlite3_finalize(statement);
 	}
-	tf_ssb_release_db_reader(ssb, db);
+	if (reader != db)
+	{
+		tf_ssb_release_db_reader(ssb, reader);
+	}
 	return has_permission;
 }

src/ssb.db.h

@@ -448,11 +448,12 @@ bool tf_ssb_db_verify(tf_ssb_t* ssb, const char* id);
 /**
 ** Check if a user has a specific permission.
 ** @param ssb The SSB instance.
+** @param db Optional database instance.  If NULL, one will be acquired from ssb.
 ** @param id The user ID.
 ** @param permission The name of the permission.
 ** @return true If the user has the requested permission.
 */
-bool tf_ssb_db_user_has_permission(tf_ssb_t* ssb, const char* id, const char* permission);
+bool tf_ssb_db_user_has_permission(tf_ssb_t* ssb, sqlite3* db, const char* id, const char* permission);
 
 /**
 ** An SQLite authorizer callback.  See https://www.sqlite.org/c3ref/set_authorizer.html for use.

src/ssb.js.c

@@ -368,9 +368,9 @@ typedef struct _swap_with_server_identity_t
 static void _tf_ssb_swap_with_server_identity_work(tf_ssb_t* ssb, void* user_data)
 {
 	swap_with_server_identity_t* work = user_data;
-	if (tf_ssb_db_user_has_permission(ssb, work->user, "administration"))
+	sqlite3* db = tf_ssb_acquire_db_writer(ssb);
+	if (tf_ssb_db_user_has_permission(ssb, db, work->user, "administration"))
 	{
-		sqlite3* db = tf_ssb_acquire_db_writer(ssb);
 		char* error = NULL;
 		if (sqlite3_exec(db, "BEGIN TRANSACTION", NULL, NULL, &error) == SQLITE_OK)
 		{
@@ -404,12 +404,12 @@ static void _tf_ssb_swap_with_server_identity_work(tf_ssb_t* ssb, void* user_dat
 		{
 			work->error = error ? tf_strdup(error) : tf_strdup(sqlite3_errmsg(db));
 		}
-		tf_ssb_release_db_writer(ssb, db);
 	}
 	else
 	{
 		work->error = tf_strdup("not administrator");
 	}
+	tf_ssb_release_db_writer(ssb, db);
 }
 
 static void _tf_ssb_swap_with_server_identity_after_work(tf_ssb_t* ssb, int status, void* user_data)
@@ -480,7 +480,7 @@ static void _tf_ssb_getIdentities_visit(const char* identity, void* user_data)
 static void _tf_ssb_get_identities_work(tf_ssb_t* ssb, void* user_data)
 {
 	identities_visit_t* work = user_data;
-	if (tf_ssb_db_user_has_permission(ssb, work->user, "administration"))
+	if (tf_ssb_db_user_has_permission(ssb, NULL, work->user, "administration"))
 	{
 		char id[k_id_base64_len] = "";
 		if (tf_ssb_whoami(ssb, id, sizeof(id)))
@@ -552,7 +552,7 @@ static void _tf_ssb_get_private_key_work(tf_ssb_t* ssb, void* user_data)
 {
 	get_private_key_t* work = user_data;
 	work->got_private_key = tf_ssb_db_identity_get_private_key(ssb, work->user, work->id, work->private_key, sizeof(work->private_key));
-	if (!work->got_private_key && tf_ssb_db_user_has_permission(ssb, work->user, "administration"))
+	if (!work->got_private_key && tf_ssb_db_user_has_permission(ssb, NULL, work->user, "administration"))
 	{
 		work->got_private_key = tf_ssb_db_identity_get_private_key(ssb, ":admin", work->id, work->private_key, sizeof(work->private_key));
 	}
@@ -658,7 +658,7 @@ static void _tf_ssb_getActiveIdentity_work(tf_ssb_t* ssb, void* user_data)
 		tf_ssb_db_identity_visit(ssb, request->name, _tf_ssb_getActiveIdentity_visit, request);
 	}
 
-	if (!*request->identity && tf_ssb_db_user_has_permission(ssb, request->name, "administration"))
+	if (!*request->identity && tf_ssb_db_user_has_permission(ssb, NULL, request->name, "administration"))
 	{
 		tf_ssb_whoami(ssb, request->identity, sizeof(request->identity));
 	}
@@ -742,7 +742,7 @@ static void _tf_ssb_getIdentityInfo_work(tf_ssb_t* ssb, void* user_data)
 {
 	identity_info_work_t* request = user_data;
 	char id[k_id_base64_len] = "";
-	if (tf_ssb_db_user_has_permission(ssb, request->name, "administration"))
+	if (tf_ssb_db_user_has_permission(ssb, NULL, request->name, "administration"))
 	{
 		if (tf_ssb_whoami(ssb, id, sizeof(id)))
 		{
@@ -904,7 +904,7 @@ static void _tf_ssb_append_message_with_identity_get_key_work(tf_ssb_t* ssb, voi
 {
 	append_message_t* work = user_data;
 	work->got_private_key = tf_ssb_db_identity_get_private_key(ssb, work->user, work->id, work->private_key, sizeof(work->private_key));
-	if (!work->got_private_key && tf_ssb_db_user_has_permission(ssb, work->user, "administration"))
+	if (!work->got_private_key && tf_ssb_db_user_has_permission(ssb, NULL, work->user, "administration"))
 	{
 		work->got_private_key = tf_ssb_db_identity_get_private_key(ssb, ":admin", work->id, work->private_key, sizeof(work->private_key));
 	}
@@ -1974,7 +1974,7 @@ enum
 	k_max_private_message_recipients = 8
 };
 
-static bool _tf_ssb_get_private_key_curve25519(sqlite3* db, const char* user, const char* identity, uint8_t out_private_key[static crypto_sign_SECRETKEYBYTES])
+static bool _tf_ssb_get_private_key_curve25519_internal(sqlite3* db, const char* user, const char* identity, uint8_t out_private_key[static crypto_sign_SECRETKEYBYTES])
 {
 	if (!user || !identity)
 	{
@@ -2003,6 +2003,21 @@ static bool _tf_ssb_get_private_key_curve25519(sqlite3* db, const char* user, co
 	return success;
 }
 
+static bool _tf_ssb_get_private_key_curve25519(tf_ssb_t* ssb, sqlite3* db, const char* user, const char* identity, uint8_t out_private_key[static crypto_sign_SECRETKEYBYTES])
+{
+	if (_tf_ssb_get_private_key_curve25519_internal(db, user, identity, out_private_key))
+	{
+		return true;
+	}
+
+	if (tf_ssb_db_user_has_permission(ssb, db, user, "administration"))
+	{
+		return _tf_ssb_get_private_key_curve25519_internal(db, ":admin", identity, out_private_key);
+	}
+
+	return false;
+}
+
 typedef struct _private_message_encrypt_t
 {
 	const char* signer_user;
@@ -2025,7 +2040,7 @@ static void _tf_ssb_private_message_encrypt_work(tf_ssb_t* ssb, void* user_data)
 
 	uint8_t private_key[crypto_sign_SECRETKEYBYTES] = { 0 };
 	sqlite3* db = tf_ssb_acquire_db_reader(ssb);
-	bool found = _tf_ssb_get_private_key_curve25519(db, work->signer_user, work->signer_identity, private_key);
+	bool found = _tf_ssb_get_private_key_curve25519(ssb, db, work->signer_user, work->signer_identity, private_key);
 	tf_ssb_release_db_reader(ssb, db);
 
 	if (found)
@@ -2214,7 +2229,7 @@ static void _tf_ssb_private_message_decrypt_work(tf_ssb_t* ssb, void* user_data)
 
 	uint8_t private_key[crypto_sign_SECRETKEYBYTES] = { 0 };
 	sqlite3* db = tf_ssb_acquire_db_reader(ssb);
-	bool found = _tf_ssb_get_private_key_curve25519(db, work->user, work->identity, private_key);
+	bool found = _tf_ssb_get_private_key_curve25519(ssb, db, work->user, work->identity, private_key);
 	tf_ssb_release_db_reader(ssb, db);
 
 	if (found)
@@ -2341,7 +2356,7 @@ typedef struct _following_t
 static void _tf_ssb_following_work(tf_ssb_t* ssb, void* user_data)
 {
 	following_t* following = user_data;
-	following->out_following = tf_ssb_db_following_deep(ssb, following->ids, following->ids_count, following->depth - 1);
+	following->out_following = tf_ssb_db_following_deep(ssb, following->ids, following->ids_count, following->depth);
 }
 
 static void _tf_ssb_following_after_work(tf_ssb_t* ssb, int status, void* user_data)

src/ssb.rpc.c

@@ -816,8 +816,7 @@ static void _tf_ssb_connection_send_history_stream_work(tf_ssb_connection_t* con
 	const int k_max = 32;
 	if (sqlite3_prepare(db,
 			"SELECT previous, author, id, sequence, timestamp, hash, json(content), signature, flags FROM messages WHERE author = ?1 AND sequence > ?2 AND "
-			"sequence "
-			"< ?3 ORDER BY sequence",
+			"sequence < ?3 ORDER BY sequence",
 			-1, &statement, NULL) == SQLITE_OK)
 	{
 		if (sqlite3_bind_text(statement, 1, request->author, -1, NULL) == SQLITE_OK && sqlite3_bind_int64(statement, 2, request->sequence) == SQLITE_OK &&
@@ -828,7 +827,8 @@ static void _tf_ssb_connection_send_history_stream_work(tf_ssb_connection_t* con
 			JSRuntime* runtime = JS_NewRuntime2(&funcs, NULL);
 			JSContext* context = JS_NewContext(runtime);
 
-			while (sqlite3_step(statement) == SQLITE_ROW)
+			int r = SQLITE_OK;
+			while ((r = sqlite3_step(statement)) == SQLITE_ROW)
 			{
 				JSValue message = JS_UNDEFINED;
 				request->out_max_sequence_seen = sqlite3_column_int64(statement, 3);
@@ -1003,10 +1003,10 @@ static void _tf_ssb_rpc_ebt_replicate_send_clock_work(tf_ssb_connection_t* conne
 	tf_ssb_t* ssb = tf_ssb_connection_get_ssb(connection);
 	JSValue full_clock = JS_NewObject(context);
 
-	int64_t depth = _get_global_setting_int64(ssb, "replication_hops", -1);
+	int64_t depth = _get_global_setting_int64(ssb, "replication_hops", 2);
 
 	/* Ask for every identity we know is being followed from local accounts. */
-	const char** visible = tf_ssb_db_get_all_visible_identities(ssb, depth - 1);
+	const char** visible = tf_ssb_db_get_all_visible_identities(ssb, depth);
 	for (int i = 0; visible[i]; i++)
 	{
 		int64_t sequence = 0;
@@ -1173,6 +1173,20 @@ static void _tf_ssb_rpc_ebt_replicate_store_callback(const char* id, bool verifi
 	tf_ssb_connection_adjust_read_backpressure(connection, -1);
 }
 
+typedef struct _resend_clock_t
+{
+	tf_ssb_connection_t* connection;
+	int32_t request_number;
+} resend_clock_t;
+
+static void _tf_ssb_rpc_ebt_replicate_resend_clock(tf_ssb_connection_t* connection, void* user_data)
+{
+	resend_clock_t* resend = user_data;
+	_tf_ssb_rpc_ebt_replicate_send_clock(resend->connection, resend->request_number, JS_UNDEFINED);
+	tf_ssb_connection_set_sent_clock(resend->connection, true);
+	tf_free(user_data);
+}
+
 static void _tf_ssb_rpc_ebt_replicate(tf_ssb_connection_t* connection, uint8_t flags, int32_t request_number, JSValue args, const uint8_t* message, size_t size, void* user_data)
 {
 	tf_ssb_t* ssb = tf_ssb_connection_get_ssb(connection);
@@ -1198,6 +1212,17 @@ static void _tf_ssb_rpc_ebt_replicate(tf_ssb_connection_t* connection, uint8_t f
 		/* Looks like a message. */
 		tf_ssb_connection_adjust_read_backpressure(connection, 1);
 		tf_ssb_verify_strip_and_store_message(ssb, args, _tf_ssb_rpc_ebt_replicate_store_callback, connection);
+
+		if (tf_ssb_connection_get_sent_clock(connection))
+		{
+			tf_ssb_connection_set_sent_clock(connection, false);
+			resend_clock_t* resend = tf_malloc(sizeof(resend_clock_t));
+			*resend = (resend_clock_t) {
+				.connection = connection,
+				.request_number = request_number,
+			};
+			tf_ssb_connection_schedule_idle(connection, _tf_ssb_rpc_ebt_replicate_resend_clock, resend);
+		}
 	}
 	else
 	{

src/ssb.tests.c

@@ -16,6 +16,8 @@
 #include <time.h>
 #include <unistd.h>
 
+#include "sodium/crypto_sign.h"
+
 #if !defined(_WIN32)
 #include <sys/wait.h>
 #endif
@@ -900,8 +902,7 @@ static void _write_file(const char* path, const char* contents)
 	FILE* file = fopen(path, "w");
 	if (!file)
 	{
-		printf("Unable to write %s: %s.\n", path, strerror(errno));
-		fflush(stdout);
+		tf_printf("Unable to write %s: %s.\n", path, strerror(errno));
 		abort();
 	}
 	fputs(contents, file);
@@ -931,7 +932,7 @@ void tf_ssb_test_encrypt(const tf_test_options_t* options)
 	int result = system(command);
 	(void)result;
 	assert(WIFEXITED(result));
-	printf("returned %d\n", WEXITSTATUS(result));
+	tf_printf("returned %d\n", WEXITSTATUS(result));
 	assert(WEXITSTATUS(result) == 0);
 }
 
@@ -1026,17 +1027,198 @@ void tf_ssb_test_publish(const tf_test_options_t* options)
 	int result = system(command);
 	(void)result;
 	assert(WIFEXITED(result));
-	printf("returned %d\n", WEXITSTATUS(result));
+	tf_printf("returned %d\n", WEXITSTATUS(result));
 	assert(WEXITSTATUS(result) == 0);
 
 	snprintf(command, sizeof(command), "%s publish -d out/test_db0.sqlite -u :admin -i %s -c '{\"type\": \"post\", \"text\": \"Two.\"}'", executable, id);
 	result = system(command);
 	assert(WIFEXITED(result));
-	printf("returned %d\n", WEXITSTATUS(result));
+	tf_printf("returned %d\n", WEXITSTATUS(result));
 	assert(WEXITSTATUS(result) == 0);
 
 	uv_run(&loop, UV_RUN_DEFAULT);
 	uv_loop_close(&loop);
 }
 
+static void _test_print_identity(const char* identity, void* user_data)
+{
+	tf_ssb_t* ssb = user_data;
+	int64_t sequence = -1;
+	char id[k_id_base64_len] = { 0 };
+	snprintf(id, sizeof(id), "@%s", identity);
+	tf_ssb_db_get_latest_message_by_author(ssb, id, &sequence, NULL, 0);
+	tf_printf("IDENTITY %s: %d\n", id, (int)sequence);
+}
+
+void tf_ssb_test_replicate(const tf_test_options_t* options)
+{
+	tf_printf("Testing replication.\n");
+
+	uv_loop_t loop = { 0 };
+	uv_loop_init(&loop);
+
+	unlink("out/test_db0.sqlite");
+	tf_ssb_t* ssb0 = tf_ssb_create(&loop, NULL, "file:out/test_db0.sqlite", NULL);
+	tf_ssb_register(tf_ssb_get_context(ssb0), ssb0);
+	unlink("out/test_db1.sqlite");
+	tf_ssb_t* ssb1 = tf_ssb_create(&loop, NULL, "file:out/test_db1.sqlite", NULL);
+	tf_ssb_register(tf_ssb_get_context(ssb1), ssb1);
+
+	uv_idle_t idle0 = { .data = ssb0 };
+	uv_idle_init(&loop, &idle0);
+	uv_idle_start(&idle0, _ssb_test_idle);
+
+	uv_idle_t idle1 = { .data = ssb1 };
+	uv_idle_init(&loop, &idle1);
+	uv_idle_start(&idle1, _ssb_test_idle);
+
+	test_t test = {
+		.ssb0 = ssb0,
+		.ssb1 = ssb1,
+	};
+
+	tf_ssb_add_connections_changed_callback(ssb0, _ssb_test_connections_changed, NULL, &test);
+	tf_ssb_add_connections_changed_callback(ssb1, _ssb_test_connections_changed, NULL, &test);
+
+	tf_ssb_generate_keys(ssb0);
+	tf_ssb_generate_keys(ssb1);
+
+	uint8_t priv0[crypto_sign_SECRETKEYBYTES] = { 0 };
+	uint8_t priv1[crypto_sign_SECRETKEYBYTES] = { 0 };
+	tf_ssb_get_private_key(ssb0, priv0, sizeof(priv0));
+	tf_ssb_get_private_key(ssb1, priv1, sizeof(priv1));
+
+	char id0[k_id_base64_len] = { 0 };
+	char id1[k_id_base64_len] = { 0 };
+	bool b = tf_ssb_whoami(ssb0, id0, sizeof(id0));
+	(void)b;
+	assert(b);
+	b = tf_ssb_whoami(ssb1, id1, sizeof(id1));
+	assert(b);
+	tf_printf("ID %s and %s\n", id0, id1);
+
+	char priv0_str[512] = { 0 };
+	char priv1_str[512] = { 0 };
+	tf_base64_encode(priv0, sizeof(priv0), priv0_str, sizeof(priv0_str));
+	tf_base64_encode(priv1, sizeof(priv0), priv1_str, sizeof(priv1_str));
+	tf_ssb_db_identity_add(ssb0, "test", id0 + 1, priv0_str);
+	tf_ssb_db_identity_add(ssb1, "test", id1 + 1, priv1_str);
+
+	static const int k_key_count = 5;
+	char public[k_key_count][k_id_base64_len - 1];
+	char private[k_key_count][512];
+	for (int i = 0; i < k_key_count; i++)
+	{
+		tf_ssb_generate_keys_buffer(public[i], sizeof(public[i]), private[i], sizeof(private[i]));
+		bool added = tf_ssb_db_identity_add(ssb0, "test", public[i], private[i]);
+		tf_printf("%s user %d = %s private=%s\n", added ? "added" : "failed", i, public[i], private[i]);
+	}
+
+	JSContext* context0 = tf_ssb_get_context(ssb0);
+	for (int i = 0; i < k_key_count - 1; i++)
+	{
+		JSValue obj = JS_NewObject(context0);
+		JS_SetPropertyStr(context0, obj, "type", JS_NewString(context0, "contact"));
+		char self[k_id_base64_len];
+		snprintf(self, sizeof(self), "@%s", public[i]);
+		char contact[k_id_base64_len];
+		snprintf(contact, sizeof(contact), "@%s", public[i + 1]);
+		JS_SetPropertyStr(context0, obj, "contact", JS_NewString(context0, contact));
+		JS_SetPropertyStr(context0, obj, "following", JS_TRUE);
+		bool stored = false;
+		uint8_t private_bin[512] = { 0 };
+		tf_base64_decode(private[i], strlen(private[i]) - strlen(".ed25519"), private_bin, sizeof(private_bin));
+		tf_printf("ssb0 %s following %s\n", self, contact);
+		JSValue signed_message = tf_ssb_sign_message(ssb0, self, private_bin, obj, NULL, 0);
+		tf_ssb_verify_strip_and_store_message(ssb0, signed_message, _message_stored, &stored);
+		JS_FreeValue(context0, signed_message);
+		_wait_stored(ssb0, &stored);
+		JS_FreeValue(context0, obj);
+
+		obj = JS_NewObject(context0);
+		JS_SetPropertyStr(context0, obj, "type", JS_NewString(context0, "post"));
+		JS_SetPropertyStr(context0, obj, "text", JS_NewString(context0, "Hello, world!"));
+		stored = false;
+		signed_message = tf_ssb_sign_message(ssb0, self, private_bin, obj, NULL, 0);
+		tf_ssb_verify_strip_and_store_message(ssb0, signed_message, _message_stored, &stored);
+		JS_FreeValue(context0, signed_message);
+		_wait_stored(ssb0, &stored);
+		JS_FreeValue(context0, obj);
+	}
+
+	JSContext* context1 = tf_ssb_get_context(ssb1);
+	{
+		JSValue obj = JS_NewObject(context1);
+		JS_SetPropertyStr(context1, obj, "type", JS_NewString(context1, "contact"));
+		char self[k_id_base64_len];
+		snprintf(self, sizeof(self), "%s", id1);
+		char contact[k_id_base64_len];
+		snprintf(contact, sizeof(contact), "@%s", public[0]);
+		JS_SetPropertyStr(context1, obj, "contact", JS_NewString(context1, contact));
+		JS_SetPropertyStr(context1, obj, "following", JS_TRUE);
+		bool stored = false;
+		tf_printf("ssb1 %s following %s\n", self, contact);
+		JSValue signed_message = tf_ssb_sign_message(ssb1, self, priv1, obj, NULL, 0);
+		tf_ssb_verify_strip_and_store_message(ssb1, signed_message, _message_stored, &stored);
+		JS_FreeValue(context1, signed_message);
+		_wait_stored(ssb1, &stored);
+		JS_FreeValue(context1, obj);
+	}
+
+	tf_printf("ssb0\n");
+	tf_ssb_db_identity_visit_all(ssb0, _test_print_identity, ssb0);
+	tf_printf("ssb1\n");
+	tf_ssb_db_identity_visit_all(ssb1, _test_print_identity, ssb1);
+
+	tf_ssb_server_open(ssb0, 12347);
+
+	uint8_t id0bin[k_id_bin_len];
+	tf_ssb_id_str_to_bin(id0bin, id0);
+	tf_ssb_connect(ssb1, "127.0.0.1", 12347, id0bin, 0, NULL, NULL);
+
+	tf_printf("Waiting for connection.\n");
+	while (test.connection_count0 != 1 || test.connection_count1 != 1)
+	{
+		tf_ssb_set_main_thread(ssb0, true);
+		tf_ssb_set_main_thread(ssb1, true);
+		uv_run(&loop, UV_RUN_ONCE);
+		tf_ssb_set_main_thread(ssb0, false);
+		tf_ssb_set_main_thread(ssb1, false);
+	}
+	tf_ssb_server_close(ssb0);
+
+	int count1 = 0;
+	tf_ssb_add_message_added_callback(ssb1, _message_added, NULL, &count1);
+	tf_printf("Waiting for message from other.\n");
+	while (count1 != 4)
+	{
+		tf_ssb_set_main_thread(ssb1, true);
+		uv_run(&loop, UV_RUN_ONCE);
+		tf_ssb_set_main_thread(ssb1, false);
+	}
+	tf_ssb_remove_message_added_callback(ssb1, _message_added, &count1);
+	tf_printf("done\n");
+
+	tf_ssb_send_close(ssb1);
+
+	uv_close((uv_handle_t*)&idle0, NULL);
+	uv_close((uv_handle_t*)&idle1, NULL);
+
+	tf_printf("final run\n");
+	tf_ssb_set_main_thread(ssb0, true);
+	tf_ssb_set_main_thread(ssb1, true);
+	uv_run(&loop, UV_RUN_DEFAULT);
+	tf_ssb_set_main_thread(ssb0, false);
+	tf_ssb_set_main_thread(ssb1, false);
+	tf_printf("done\n");
+
+	tf_printf("destroy 0\n");
+	tf_ssb_destroy(ssb0);
+	tf_printf("destroy 1\n");
+	tf_ssb_destroy(ssb1);
+
+	tf_printf("close\n");
+	uv_loop_close(&loop);
+}
+
 #endif

src/ssb.tests.h

@@ -65,4 +65,10 @@ void tf_ssb_test_peer_exchange(const tf_test_options_t* options);
 */
 void tf_ssb_test_publish(const tf_test_options_t* options);
 
+/**
+** Test replication.
+** @param options The test options.
+*/
+void tf_ssb_test_replicate(const tf_test_options_t* options);
+
 /** @} */

src/task.c

@@ -1338,8 +1338,7 @@ void tf_task_resolve_promise(tf_task_t* task, promiseid_t promise, JSValue value
 	}
 	else
 	{
-		tf_printf("Didn't find promise %d to resolve.\n", promise);
-		abort();
+		tf_printf("WARNING: Didn't find promise %d to resolve.\n", promise);
 	}
 }
 
@@ -1368,8 +1367,7 @@ void tf_task_reject_promise(tf_task_t* task, promiseid_t promise, JSValue value)
 	}
 	else
 	{
-		tf_printf("Didn't find promise %d to reject.\n", promise);
-		abort();
+		tf_printf("WARNING: Didn't find promise %d to reject.\n", promise);
 	}
 }
 

src/taskstub.js.c

@@ -62,6 +62,7 @@ static JSValue _taskstub_set_on_print(JSContext* context, JSValueConst this_val,
 static JSValue _taskstub_loadFile(JSContext* context, JSValueConst this_val, int argc, JSValueConst* argv);
 static void _taskstub_on_process_exit(uv_process_t* process, int64_t status, int terminationSignal);
 static void _taskstub_finalizer(JSRuntime* runtime, JSValue value);
+static void _taskstub_cleanup(tf_taskstub_t* stub);
 
 static void _tf_taskstub_run_sandbox_thread(void* data)
 {
@@ -74,12 +75,60 @@ static void _tf_taskstub_run_sandbox_thread(void* data)
 	tf_task_destroy(task);
 }
 
+static void _taskstub_on_handle_close(uv_handle_t* handle)
+{
+	tf_taskstub_t* stub = handle->data;
+	tf_task_remove_child(stub->_owner, stub);
+	handle->data = NULL;
+	_taskstub_cleanup(stub);
+}
+
+static void _tf_taskstub_on_exit(tf_taskstub_t* stub, int64_t status, int termination_signal)
+{
+	JSContext* context = tf_task_get_context(stub->_owner);
+	if (!JS_IsUndefined(stub->_on_exit))
+	{
+		JSValue ref = JS_DupValue(context, stub->_on_exit);
+		JSValue argv[] = { JS_NewInt64(context, status), JS_NewInt32(context, termination_signal) };
+		JSValue result = JS_Call(context, stub->_on_exit, JS_NULL, 2, argv);
+		tf_util_report_error(context, result);
+		JS_FreeValue(context, result);
+		JS_FreeValue(context, argv[0]);
+		JS_FreeValue(context, argv[1]);
+		JS_FreeValue(context, ref);
+	}
+	if (stub->_stream)
+	{
+		tf_packetstream_destroy(stub->_stream);
+		stub->_stream = NULL;
+	}
+	tf_task_remove_child(stub->_owner, stub);
+	if (stub->_process.data)
+	{
+		uv_close((uv_handle_t*)&stub->_process, _taskstub_on_handle_close);
+	}
+	else
+	{
+		_taskstub_cleanup(stub);
+	}
+}
+
+static void _tf_taskstub_packetstream_close(void* user_data)
+{
+	tf_taskstub_t* stub = user_data;
+	if (!stub->_process.data)
+	{
+		_tf_taskstub_on_exit(stub, -1, -1);
+	}
+}
+
 static JSValue _taskstub_create(JSContext* context, JSValueConst this_val, int argc, JSValueConst* argv)
 {
 	tf_task_t* parent = tf_task_get(context);
 	tf_taskstub_t* stub = tf_malloc(sizeof(tf_taskstub_t));
 	memset(stub, 0, sizeof(*stub));
 	stub->_stream = tf_packetstream_create();
+	tf_packetstream_set_on_close(stub->_stream, _tf_taskstub_packetstream_close, stub);
 
 	JSValue taskObject = JS_NewObjectClass(context, _classId);
 	JS_SetOpaque(taskObject, stub);
@@ -314,35 +363,10 @@ static void _taskstub_finalizer(JSRuntime* runtime, JSValue value)
 	_taskstub_cleanup(stub);
 }
 
-static void _taskstub_on_handle_close(uv_handle_t* handle)
-{
-	tf_taskstub_t* stub = handle->data;
-	tf_task_remove_child(stub->_owner, stub);
-	handle->data = NULL;
-	_taskstub_cleanup(stub);
-}
-
-static void _taskstub_on_process_exit(uv_process_t* process, int64_t status, int terminationSignal)
+static void _taskstub_on_process_exit(uv_process_t* process, int64_t status, int termination_signal)
 {
 	tf_taskstub_t* stub = process->data;
-	JSContext* context = tf_task_get_context(stub->_owner);
-	if (!JS_IsUndefined(stub->_on_exit))
-	{
-		JSValue ref = JS_DupValue(context, stub->_on_exit);
-		JSValue argv[] = { JS_NewInt64(context, status), JS_NewInt32(context, terminationSignal) };
-		JSValue result = JS_Call(context, stub->_on_exit, JS_NULL, 2, argv);
-		tf_util_report_error(context, result);
-		JS_FreeValue(context, result);
-		JS_FreeValue(context, argv[0]);
-		JS_FreeValue(context, argv[1]);
-		JS_FreeValue(context, ref);
-	}
-	if (stub->_stream)
-	{
-		tf_packetstream_destroy(stub->_stream);
-		stub->_stream = NULL;
-	}
-	uv_close((uv_handle_t*)process, _taskstub_on_handle_close);
+	_tf_taskstub_on_exit(stub, status, termination_signal);
 }
 
 static JSValue _taskstub_getExports(JSContext* context, JSValueConst this_val, int argc, JSValueConst* argv)

src/tests.c

@@ -40,8 +40,7 @@ static void _write_file(const char* path, const char* contents)
 	FILE* file = fopen(path, "w");
 	if (!file)
 	{
-		printf("Unable to write %s: %s.\n", path, strerror(errno));
-		fflush(stdout);
+		tf_printf("Unable to write %s: %s.\n", path, strerror(errno));
 		abort();
 	}
 	fputs(contents, file);
@@ -1062,6 +1061,7 @@ void tf_tests(const tf_test_options_t* options)
 	_tf_test_run(options, "encrypt", tf_ssb_test_encrypt, false);
 	_tf_test_run(options, "peer_exchange", tf_ssb_test_peer_exchange, false);
 	_tf_test_run(options, "publish", tf_ssb_test_publish, false);
+	_tf_test_run(options, "replicate", tf_ssb_test_replicate, false);
 	tf_printf("Tests completed.\n");
 #endif
 }

src/util.js.c

@@ -401,6 +401,11 @@ void tf_util_register(JSContext* context)
 
 int tf_util_get_length(JSContext* context, JSValue value)
 {
+	if (JS_IsUndefined(value))
+	{
+		return 0;
+	}
+
 	JSValue length = JS_GetPropertyStr(context, value, "length");
 	int result = 0;
 	JS_ToInt32(context, &result, length);

src/version.h

@@ -1,2 +1,2 @@
-#define VERSION_NUMBER "0.0.25"
+#define VERSION_NUMBER "0.0.26"
 #define VERSION_NAME "This program kills fascists."

tools/autotest.py

@@ -9,14 +9,59 @@ if sys.platform == 'haiku1':
 	print('Automation tests are disabled on Haiku.')
 	exit(0)
 
+import selenium
 from selenium import webdriver
 from selenium.webdriver.firefox.service import Service
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions
 from selenium.webdriver.support.ui import WebDriverWait
 
-def exists_in_shadow_root(shadow_root, by, value):
-	return lambda driver: shadow_root.find_element(by, value)
+def select(driver, path, action = None, keep_trying = False):
+	start_time = time.time()
+	done = False
+	while True:
+		try:
+			driver.switch_to.default_content()
+			context = driver
+			for node in path:
+				if node.startswith('#'):
+					context = context.find_element(By.ID, node[1:])
+				elif node.startswith('/'):
+					context = context.find_element(By.XPATH, node)
+				elif node.startswith('.'):
+					context = context.find_element(By.CLASS_NAME, node[1:])
+				elif node.startswith('='):
+					context = context.find_element(By.LINK_TEXT, node[1:])
+				elif node == 'frame':
+					driver.switch_to.frame(context)
+					context = driver
+				elif node == 'shadow_root':
+					context = context.shadow_root
+				else:
+					context = context.find_element(By.TAG_NAME, node)
+			if action is not None:
+				if action[0] == 'click':
+					context.click()
+				elif action[0] == 'send_keys':
+					context.send_keys(action[1])
+				elif action[0] == 'clear':
+					context.clear()
+				else:
+					raise RuntimeError(f'Unexpected action: {action}.')
+				done = True
+				if not keep_trying:
+					break
+			else:
+				return context
+		except (selenium.common.exceptions.NoSuchElementException,
+				selenium.common.exceptions.NoSuchShadowRootException,
+				selenium.common.exceptions.StaleElementReferenceException,
+				selenium.common.exceptions.WebDriverException):
+			if done and keep_trying:
+				break
+			if time.time() - start_time < 5.0:
+				time.sleep(0.1)
+				pass
 
 success = False
 try:
@@ -27,290 +72,177 @@ try:
 	wait = WebDriverWait(driver, 10)
 
 	driver.get('http://localhost:8888')
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.LINK_TEXT, 'login').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'register_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('adminuser')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('admin_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'confirm').send_keys('admin_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-	driver.switch_to.frame(driver.find_element(By.ID, 'document'))
-	wait.until(expected_conditions.presence_of_element_located((By.LINK_TEXT, 'identity')))
-	driver.switch_to.default_content()
+	select(driver, ['tf-navigation', 'shadow_root', '=login'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#register_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'adminuser'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'admin_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#confirm'], ('send_keys', 'admin_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['#document', 'frame', '=identity'])
 
 	driver.get('http://localhost:8888/~core/admin/')
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-	driver.switch_to.frame(driver.find_element(By.ID, 'document'))
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'gs_room_name'))).send_keys('test room')
-	wait.until(expected_conditions.presence_of_element_located((By.XPATH, '//*[@id="gs_room_name"]/following-sibling::button'))).click()
+	select(driver, ['#document', 'frame', '#gs_room_name'], ('send_keys', 'test room'))
+	select(driver, ['#document', 'frame', '//*[@id="gs_room_name"]/following-sibling::button'], ('click',))
+	select(driver, ['//button[text()="✅ Allow"]'], ('click',))
 	driver.switch_to.alert.accept()
-	driver.switch_to.default_content()
 
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'identity').click()
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'logout').click()
+	select(driver, ['tf-navigation', 'shadow_root', '#identity'], ('click',))
+	select(driver, ['tf-navigation', 'shadow_root', '#logout'], ('click',))
 
 	driver.get('http://localhost:8888')
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.LINK_TEXT, 'login').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'register_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('testuser')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'confirm').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
+	select(driver, ['tf-navigation', 'shadow_root', '=login'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#register_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'testuser'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#confirm'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['#document'])
 
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-
-	driver.switch_to.default_content()
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'create_identity').click()
+	select(driver, ['tf-navigation', 'shadow_root', '#create_identity'], ('click',))
 	wait.until(expected_conditions.alert_is_present()).accept()
 
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
+	select(driver, ['tf-navigation', 'shadow_root', '#identity'], ('click',))
+	select(driver, ['tf-navigation', 'shadow_root', '#id_dropdown', '//button[position()=2]'], ('click',))
 
-	driver.switch_to.default_content()
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'identity').click()
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'id_dropdown').find_element(By.XPATH, '//button[position()=2]').click()
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-	driver.switch_to.frame(driver.find_element(By.ID, 'document'))
-	# NoSuchShadowRootException
-	while True:
-		try:
-			tf_app = wait.until(expected_conditions.presence_of_element_located((By.TAG_NAME, 'tf-app'))).shadow_root
-			break
-		except:
-			pass
-	wait.until(exists_in_shadow_root(wait.until(exists_in_shadow_root(tf_app, By.ID, 'tf-tab-news')).shadow_root, By.CLASS_NAME, 'tf-profile')).shadow_root.find_element(By.ID, 'edit_profile').click()
-	wait.until(exists_in_shadow_root(wait.until(exists_in_shadow_root(tf_app, By.ID, 'tf-tab-news')).shadow_root, By.CLASS_NAME, 'tf-profile')).shadow_root.find_element(By.ID, 'name').send_keys('user')
-	wait.until(exists_in_shadow_root(wait.until(exists_in_shadow_root(tf_app, By.ID, 'tf-tab-news')).shadow_root, By.CLASS_NAME, 'tf-profile')).shadow_root.find_element(By.ID, 'save_profile').click()
-	driver.switch_to.default_content()
-	wait.until(expected_conditions.presence_of_element_located((By.XPATH, '//button[text()="✅ Allow"]'))).click()
+	select(driver, ['#document', 'frame', 'tf-app', 'shadow_root', '#tf-tab-news', 'shadow_root', '.tf-profile', 'shadow_root', '#edit_profile'], ('click',), keep_trying = True)
+	select(driver, ['#document', 'frame', 'tf-app', 'shadow_root', '#tf-tab-news', 'shadow_root', '.tf-profile', 'shadow_root', '#name'], ('send_keys', 'user'))
+	select(driver, ['#document', 'frame', 'tf-app', 'shadow_root', '#tf-tab-news', 'shadow_root', '.tf-profile', 'shadow_root', '#save_profile'], ('click',))
+
+	select(driver, ['//button[text()="✅ Allow"]'], ('click',))
 
-	driver.switch_to.default_content()
 	driver.get('http://localhost:8888/~testuser/test/')
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-	while True:
-		try:
-			wait.until(exists_in_shadow_root(driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root, By.ID, 'close_error')).click()
-			break
-		except:
-			pass
+	select(driver, ['#document'])
+	select(driver, ['tf-navigation', 'shadow_root', '#close_error'], ('click',))
+	select(driver, ['tf-navigation', 'shadow_root', '=edit'], ('click',))
+	select(driver, ['#editor', '.cm-content'], ('click',))
+	select(driver, ['#editor', '.cm-content'], ('send_keys', 'app.setDocument(\n\t"<div id=\'test-div\'>Hello, world!</div>"\n);'))
+	select(driver, ['#save'], ('click',))
 
-	driver.switch_to.default_content()
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.LINK_TEXT, 'edit').click()
-	editor = driver.find_element(By.ID, 'editor').find_element(By.CLASS_NAME, 'cm-content')
-	editor.click()
-	editor.send_keys('app.setDocument(\n\t"<div id=\'test-div\'>Hello, world!</div>"\n);');
-	driver.find_element(By.ID, 'save').click()
-
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-	driver.switch_to.frame(driver.find_element(By.ID, 'document'))
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'test-div')))
+	select(driver, ['#document', 'frame', '#test-div'])
 
 	size = driver.get_window_size()
 	driver.set_window_size(1200, 540)
 	driver.save_screenshot('out/screenshot0.png')
 	driver.set_window_size(size['width'], size['height'])
 
-	driver.switch_to.default_content()
-	editor = driver.find_element(By.ID, 'editor').find_element(By.CLASS_NAME, 'cm-content')
-	editor.click()
-	editor.clear()
-	editor.send_keys('app.setDocument("<div id=\'test-div2\'>Hello, world, again!</div>")');
-	driver.find_element(By.ID, 'save').click()
+	select(driver, ['#editor', '.cm-content'], ('click',))
+	select(driver, ['#editor', '.cm-content'], ('clear',))
+	select(driver, ['#editor', '.cm-content'], ('send_keys', 'app.setDocument("<div id=\'test-div2\'>Hello, world, again!</div>")'))
+	select(driver, ['#save'], ('click',))
+	select(driver, ['#document', 'frame', '#test-div2'])
 
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-	driver.switch_to.frame(driver.find_element(By.ID, 'document'))
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'test-div2')))
-
-	driver.switch_to.default_content()
-	driver.find_element(By.ID, 'delete').click()
+	select(driver, ['#delete'], ('click',))
 	wait.until(expected_conditions.alert_is_present()).accept()
 	wait.until(expected_conditions.alert_is_present()).dismiss()
 	driver.get('http://localhost:8888/~testuser/test/')
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-	while True:
-		try:
-			wait.until(exists_in_shadow_root(driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root, By.ID, 'close_error')).click()
-			break
-		except:
-			pass
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.LINK_TEXT, 'edit').click()
+
+	select(driver, ['#document'])
+	select(driver, ['tf-navigation', 'shadow_root', '#close_error'], ('click',))
+	select(driver, ['tf-navigation', 'shadow_root', '=edit'], ('click',))
 
 	driver.get('http://localhost:8888')
 
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-	driver.switch_to.frame(driver.find_element(By.ID, 'document'))
-
-	wait.until(expected_conditions.presence_of_element_located((By.LINK_TEXT, 'identity')))
+	select(driver, ['#document', 'frame', '=identity'])
 
 	size = driver.get_window_size()
 	driver.set_window_size(540, 1200)
 	driver.save_screenshot('out/screenshot1.png')
 	driver.set_window_size(size['width'], size['height'])
 
-	wait.until(expected_conditions.presence_of_element_located((By.LINK_TEXT, 'identity'))).click()
+	select(driver, ['#document', 'frame', '=identity'], ('click',))
 
-	# StaleElementReferenceException
-	while True:
-		try:
-			driver.switch_to.default_content()
-			wait.until(expected_conditions.presence_of_element_located((By.ID, 'content')))
-			driver.switch_to.frame(wait.until(expected_conditions.presence_of_element_located((By.ID, 'document'))))
-			wait.until(expected_conditions.presence_of_element_located((By.ID, 'create_id'))).click()
-			driver.switch_to.alert.accept()
-			break
-		except:
-			pass
+	select(driver, ['#document', 'frame', '#create_id'], ('click',))
+	id0 = select(driver, ['#document', 'frame', 'li']).text.split(' ')[-1]
 
-	# StaleElementReferenceException
-	while True:
-		try:
-			id0 = wait.until(expected_conditions.presence_of_element_located((By.TAG_NAME, 'li'))).text.split(' ')[-1]
-			break
-		except:
-			pass
-	wait.until(expected_conditions.presence_of_element_located((By.XPATH, '//li/button[text()="Export Identity"]'))).click()
-	driver.switch_to.default_content()
-	wait.until(expected_conditions.presence_of_element_located((By.XPATH, '//button[text()="✅ Allow"]'))).click()
-	driver.switch_to.frame(wait.until(expected_conditions.presence_of_element_located((By.ID, 'document'))))
-	words = wait.until(expected_conditions.presence_of_element_located((By.XPATH, '//li//textarea'))).get_attribute('value')
-	wait.until(expected_conditions.presence_of_element_located((By.XPATH, '//li/button[text()="Delete Identity"]'))).click()
+	select(driver, ['#document', 'frame', '//li/button[text()="Export Identity"]'], ('click',))
+	select(driver, ['//button[text()="✅ Allow"]'], ('click',))
+	words = select(driver, ['#document', 'frame', '//li//textarea']).get_attribute('value')
+	select(driver, ['#document', 'frame', '//li/button[text()="Delete Identity"]'], ('click',))
 	driver.switch_to.alert.send_keys('DELETE')
 	driver.switch_to.alert.accept()
-	driver.switch_to.default_content()
-	wait.until(expected_conditions.presence_of_element_located((By.XPATH, '//button[text()="✅ Allow"]'))).click()
+	select(driver, ['//button[text()="✅ Allow"]'], ('click',))
+	driver.switch_to.alert.accept()
+	words = select(driver, ['#document', 'frame', '//textarea'], ('send_keys', words))
+	select(driver, ['#document', 'frame', '//button[text()="Import Identity"]'], ('click',))
+	select(driver, ['//button[text()="✅ Allow"]'], ('click',))
 	driver.switch_to.alert.accept()
 	driver.switch_to.frame(wait.until(expected_conditions.presence_of_element_located((By.ID, 'document'))))
-	words = wait.until(expected_conditions.presence_of_element_located((By.XPATH, '//textarea'))).send_keys(words)
-	wait.until(expected_conditions.presence_of_element_located((By.XPATH, '//button[text()="Import Identity"]'))).click()
-	driver.switch_to.default_content()
-	wait.until(expected_conditions.presence_of_element_located((By.XPATH, '//button[text()="✅ Allow"]'))).click()
-	driver.switch_to.alert.accept()
-	driver.switch_to.frame(wait.until(expected_conditions.presence_of_element_located((By.ID, 'document'))))
-	id1 = wait.until(expected_conditions.presence_of_element_located((By.TAG_NAME, 'li'))).text.split(' ')[-1]
+	id1 = select(driver, ['#document', 'frame', 'li']).text.split(' ')[-1]
+	assert id0 == id1
 
 	driver.get('http://localhost:8888')
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-	driver.switch_to.frame(driver.find_element(By.ID, 'document'))
-	wait.until(expected_conditions.presence_of_element_located((By.LINK_TEXT, 'ssb'))).click()
-	driver.switch_to.default_content()
+	select(driver, ['#document', 'frame', '=ssb'], ('click',))
+	select(driver, ['#document', 'frame', 'tf-app', 'shadow_root', '#tf-tab-news', 'shadow_root', '#tf-compose', 'shadow_root', '#edit'], ('send_keys', 'Hello, world!'))
+	select(driver, ['#document', 'frame', 'tf-app', 'shadow_root', '#tf-tab-news', 'shadow_root', '#tf-compose', 'shadow_root', '#submit'], ('click',))
+	select(driver, ['//button[text()="✅ Allow"]'], ('click',))
 
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'content')))
+	select(driver, ['tf-navigation', 'shadow_root', '#identity'], ('click',))
+	select(driver, ['tf-navigation', 'shadow_root', '#logout'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#login_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'testuser'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
 
-	# StaleElementReferenceException
-	while True:
-		try:
-			driver.switch_to.frame(wait.until(expected_conditions.presence_of_element_located((By.ID, 'document'))))
-			break
-		except:
-			pass
-	# NoSuchShadowRootException
-	while True:
-		try:
-			tf_app = wait.until(expected_conditions.presence_of_element_located((By.TAG_NAME, 'tf-app'))).shadow_root
-			break
-		except:
-			pass
+	select(driver, ['tf-navigation', 'shadow_root', '#identity'], ('click',))
+	select(driver, ['tf-navigation', 'shadow_root', '#logout'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#guest_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#guestButton'], ('click',))
+	select(driver, ['#document', 'frame', 'tf-app', 'shadow_root'])
 
-	# WebDriverException (shadow root is detached)
-	while True:
-		try:
-			tf_tab_news = wait.until(exists_in_shadow_root(tf_app, By.ID, 'tf-tab-news')).shadow_root
-			tf_tab_news.find_element(By.ID, 'tf-compose').shadow_root.find_element(By.ID, 'edit').send_keys('Hello, world!')
-			tf_tab_news.find_element(By.ID, 'tf-compose').shadow_root.find_element(By.ID, 'submit').click()
-			break
-		except:
-			pass
+	select(driver, ['tf-navigation', 'shadow_root', '#logout'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#login_label'], ('click',))
 
-	driver.switch_to.default_content()
-	driver.find_element(By.ID, 'allow').click()
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'testuser'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'wrong_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#error'])
 
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'identity').click()
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'logout').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'login_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('testuser')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'wrong_user'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#error'])
 
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'content')))
+	select(driver, ['tf-auth', 'shadow_root', '#register_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'testuser'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'wrong_test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#confirm'], ('send_keys', 'wrong_test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#error'])
 
-	while True:
-		try:
-			driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'identity').click()
-			break
-		except:
-			pass
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'logout').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'guest_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'guestButton').click()
+	select(driver, ['tf-auth', 'shadow_root', '#register_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'testuser'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#confirm'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#error'])
 
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'content')))
-	driver.switch_to.frame(wait.until(expected_conditions.presence_of_element_located((By.ID, 'document'))))
-	# NoSuchShadowRootException
-	while True:
-		try:
-			tf_app = wait.until(expected_conditions.presence_of_element_located((By.TAG_NAME, 'tf-app'))).shadow_root
-			break
-		except:
-			pass
-	driver.switch_to.default_content()
+	select(driver, ['tf-auth', 'shadow_root', '#register_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', '😁'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#confirm'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#error'])
 
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'logout').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'login_label').click()
+	select(driver, ['tf-auth', 'shadow_root', '#change_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'testuser'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#new_password'], ('send_keys', 'new_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#confirm'], ('send_keys', 'new_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['#document', 'frame'])
 
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('testuser')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('wrong_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'error')
-
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('wrong_user')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'error')
-
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'register_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('testuser')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('wrong_test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'confirm').send_keys('wrong_test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'error')
-
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'register_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('1invalid')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'confirm').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'error')
-
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'register_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('😁')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'confirm').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'error')
-
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'change_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('testuser')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'new_password').send_keys('new_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'confirm').send_keys('new_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-	while True:
-		try:
-			driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'identity').click()
-			break
-		except:
-			pass
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'logout').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'login_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('testuser')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'error')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'login_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('testuser')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('new_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
+	select(driver, ['tf-navigation', 'shadow_root', '#identity'], ('click',))
+	select(driver, ['tf-navigation', 'shadow_root', '#logout'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#login_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'testuser'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#error'])
+	select(driver, ['tf-auth', 'shadow_root', '#login_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'testuser'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'new_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['#document', 'frame'])
 
 	success = True
 finally:

tools/buttfeed.py

@@ -34,6 +34,11 @@ def get_entries():
                 if m:
                     results.append((time.mktime(entry.get('updated_parsed')), name, entry.link, f'new issue #{m.group(1)}: {m.group(2)}'))
                     continue
+            elif '/releases/' in entry.link:
+                m = re.match(r'(.*) released <a href=".*?">(.*?)</a>', entry.title)
+                if m:
+                    results.append((time.mktime(entry.get('updated_parsed')), name, entry.link, f'{m.group(1)} released {m.group(2)}'))
+                    continue
             if entry.summary.startswith('<a href='):
                 for m in re.findall(r'<a href="(.*?)">.*?</a>$\s*^([^\n]+)$', entry.summary, re.S | re.M):
                     results.append((time.mktime(entry.get('updated_parsed')), name, m[0], m[1]))

tools/emojis.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import re
+import urllib.request
+
+if not os.path.exists('out/emoji-test.txt'):
+    urllib.request.urlretrieve('https://unicode.org/Public/emoji/latest/emoji-test.txt', 'out/emoji-test.txt')
+
+doc = {}
+
+with open('out/emoji-test.txt', 'r') as f:
+    for line in f:
+        line = line.strip()
+        if line.startswith('# group: '):
+            group = line[len('# group: '):]
+        elif line.startswith('# subgroup: '):
+            subgroup = line[len('# subgroup: '):]
+        else:
+            m = re.match(r'((?:\s?[0-9A-F]+)+)\s+; (\S+).*#.*E\d+\.\d+ (.*)', line)
+            if m:
+                emoji = ''.join(chr(int(g, 16)) for g in m.group(1).split(' '))
+                qualified = m.group(2)
+                if qualified == 'fully-qualified':
+                    name = m.group(3)
+                    if not group in doc:
+                        doc[group] = {}
+                    doc[group][name] = emoji
+
+with open('apps/ssb/emojis.json', 'w') as f:
+    json.dump(doc, f, ensure_ascii = False)