build: Let's release 0.0.26.

This reverts commit 7f4e2617ee.

.gitmodules

@@ -26,3 +26,6 @@
 [submodule "deps/c-ares"]
 	path = deps/c-ares
 	url = https://github.com/c-ares/c-ares.git
+[submodule "docs"]
+	path = docs
+	url = https://dev.tildefriends.net/cory/tildefriends.wiki.git

GNUmakefile

@@ -3,11 +3,24 @@
 MAKEFLAGS += --warn-undefined-variables
 MAKEFLAGS += --no-builtin-rules
 
-VERSION_CODE := 29
-VERSION_NUMBER := 0.0.24
-VERSION_NAME := Honey bunches of boats.
+## == Tilde Friends build. ==
+##
+## This is a list of all supported build targets.
+##
+## Consider passing -j$(nproc) or adding it to your $MAKEFLAGS to build in
+## parallel (faster).
+##
+## Useful variables to override:
+## CC := Compiler.
+## AS := Assembler.
+## LD := Linker.
+## ANDROID_SDK := Path to the Android SDK.
 
-SQLITE_URL := https://www.sqlite.org/2024/sqlite-amalgamation-3470000.zip
+VERSION_CODE := 31
+VERSION_NUMBER := 0.0.26
+VERSION_NAME := This program kills fascists.
+
+SQLITE_URL := https://www.sqlite.org/2024/sqlite-amalgamation-3470200.zip
 BUNDLETOOL_URL := https://github.com/google/bundletool/releases/download/1.17.0/bundletool-all-1.17.0.jar
 APPIMAGETOOL_URL := https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-x86_64.AppImage
 APPIMAGETOOL_MD5 := e989fadfc4d685fd3d6aeeb9b525d74d  out/appimagetool
@@ -747,11 +760,28 @@ $(IOS_TARGETS) $(IOSSIM_TARGETS): LDFLAGS += \
 	-framework UIKit \
 	-framework WebKit
 
-unix: debug release
-win: windebug winrelease
-all: $(BUILD_TYPES)
+##
+## Common targets:
+##
+debug: ## Build a debug executable for the current platform.
+release: ## Build a release executable for the current platform.
+all: $(BUILD_TYPES) ## Build all targets that appear possible to build on this machine.
+unix: debug release ## Build all UNIX targets.
+win: windebug winrelease ## Build all Windows targets.
 .PHONY: all win unix
 
+##
+## Windows targets:
+##
+windebug: ## Build a debug win32 executable.
+winrelease: ## Build a release win32 executable.
+
+##
+## MacOS targets:
+##
+macosdebug: ## Build a MacOS debug executable.
+macosrelease: ## Build a MacOS release executable.
+
 ALL_APP_OBJS := \
 	$(APP_OBJS) \
 	$(ARES_OBJS) \
@@ -807,7 +837,18 @@ src/android/AndroidManifest.xml : $(firstword $(MAKEFILE_LIST))
 		-e 's/android:targetSdkVersion="[[:digit:]]*"/android:targetSdkVersion="$(ANDROID_TARGET_SDK_VERSION)"/' \
 		$@
 
-# Android support.
+##
+## Android targets:
+##
+androiddebug: ## Build a debug 64-bit ARM Android APK.
+androidrelease: ## Build a release 64-bit ARM Android APK.
+androiddebug-armv7a: ## Build a debug 32-bit ARM Android APK.
+androidrelease-armv7a: ## Build a release 32-bit ARM Android APK.
+androiddebug-x86: ## Build a debug x86 Android APK.
+androidrelease-x86: ## Build a release x86 Android APK.
+androiddebug-x86_64: ## Build a debug x86_64 Android APK.
+androidrelease-x86_64: ## Build a release x86_64 Android APK.
+
 out/res/layout_activity_main.xml.flat: src/android/res/layout/activity_main.xml
 	@mkdir -p $(dir $@)
 	@echo "[aapt2] $@"
@@ -914,7 +955,7 @@ out/TildeFriends.aab: out/apk/classes.dex $(filter-out %debug%, $(ANDROID_TARGET
 	@java -jar $(BUNDLETOOL) build-bundle --overwrite --config=src/android/BundleConfig.json --modules=out/aab/base.zip --output=$@
 	@jarsigner -keystore .keys/android.jks $@ androidKey -storepass android
 
-aab: out/TildeFriends.aab
+aab: out/TildeFriends.aab ## Build an Android App Bundle.
 .PHONY: aab
 
 out/TildeFriends.apks: out/TildeFriends.aab $(BUNDLETOOL)
@@ -981,20 +1022,32 @@ out/%.zopfli.apk: out/%.apk
 	$(ANDROID_BUILD_TOOLS)/zipalign -f -z 4 $< $@.zopfli
 	@$(ANDROID_BUILD_TOOLS)/apksigner sign --ks .keys/android.jks --ks-key-alias androidKey --ks-pass pass:android --key-pass pass:android --min-sdk-version $(ANDROID_MIN_SDK_VERSION) --out $@ $@.zopfli
 
-release-apk: out/TildeFriends-arm-release.zopfli.apk out/TildeFriends-x86-release.zopfli.apk
+release-apk: out/TildeFriends-arm-release.zopfli.apk out/TildeFriends-x86-release.zopfli.apk ## Build an Android release APK.
 .PHONY: release-apk
 
-apkgo: out/TildeFriends-arm-debug.apk
+fdroid: out/apk/TildeFriends-release.fdroid.unsigned.apk ## Build Android APK for distribution on F-Droid.
+.PHONY: fdroid
+
+apkgo: out/TildeFriends-arm-debug.apk ## Build, install, and run a debug Android APK.
 	@adb install -r $<
 	@adb shell am start com.unprompted.tildefriends/.TildeFriendsActivity
 .PHONY: apkgo
 
-releaseapkgo: out/TildeFriends-arm-release.apk
+releaseapkgo: out/TildeFriends-arm-release.apk ## Build, install, and run a release Android APK.
 	@adb install -r $<
 	@adb shell am start com.unprompted.tildefriends/.TildeFriendsActivity
 .PHONY: releaseapkgo
 
-# iOS Support
+apklog: ## Display Android log output.
+	@adb logcat *:S tildefriends
+.PHONY: apklog
+
+##
+## iPhoneOS targets:
+##
+iosdebug: ## Build a debug iPhoneOS executable.
+iosrelease: ## Build a release iPhoneOS executable.
+
 out/%.app/Info.plist: src/ios/Info.plist
 	@mkdir -p $(dir $@)
 	@cp -v $< $@
@@ -1031,39 +1084,23 @@ out/%/tildefriends.standalone.exe: out/%/tildefriends.exe out/data.zip
 	@cat $< out/data.zip > $@
 	@chmod +x $@
 
-iossimdebug-app: out/tildefriends-iossimdebug.app/tildefriends
-iossimrelease-app: out/tildefriends-iossimrelease.app/tildefriends
-iosdebug-app: out/tildefriends-iosdebug.app/tildefriends
-iosrelease-app: out/tildefriends-iosrelease.app/tildefriends
+iossimdebug-app: out/tildefriends-iossimdebug.app/tildefriends ## Build a debug iOS Simulator .app directory.
+iossimrelease-app: out/tildefriends-iossimrelease.app/tildefriends ## Build a release iOS Simulator .app directory.
+iosdebug-app: out/tildefriends-iosdebug.app/tildefriends ## Build a debug iOS .app directory.
+iosrelease-app: out/tildefriends-iosrelease.app/tildefriends ## Build a release iOS .app directory.
 
-iosdebug-ipa: out/tildefriends-debug.ipa
-iosrelease-ipa: out/tildefriends-release.ipa
+iosdebug-ipa: out/tildefriends-debug.ipa ## Build a debug iOS .ipa.
+iosrelease-ipa: out/tildefriends-release.ipa ## Build a release iOS .ipa.
 .PHONY: iossimdebug-app iossimrelease-app iosdebug-app iosrelease-app
 
 ios%go: out/tildefriends-ios%.app/tildefriends
 	ideviceinstaller -i $(realpath $(dir $<))
 
-iossimdebuggo: out/tildefriends-iossimdebug.app/tildefriends
+iossimdebuggo: out/tildefriends-iossimdebug.app/tildefriends ## Build, install, and run an iOS debug build.
 	xcrun simctl install booted out/tildefriends-iossimdebug.app/
 	xcrun simctl launch booted com.unprompted.tildefriends
 .PHONY: iossimdebuggo
 
-apklog:
-	@adb logcat *:S tildefriends
-.PHONY: apklog
-
-fetchdeps:
-	@echo "[fetch] sqlite"
-	@test -f out/deps/sqlite.zip && test "$$(cat out/deps/sqlite.txt 2>/dev/null)" = $(SQLITE_URL) || (mkdir -p out/deps/ && curl -q $(SQLITE_URL) -o out/deps/sqlite.zip)
-	@test -d deps/sqlite/ && test "$$(cat out/deps/sqlite.txt 2>/dev/null)" = $(SQLITE_URL) || (mkdir -p deps/sqlite/ && unzip -qDjo -d deps/sqlite/ out/deps/sqlite.zip)
-	@echo -n $(SQLITE_URL) > out/deps/sqlite.txt
-	@echo "[fetch] prettier"
-	@test -f deps/prettier/standalone.mjs || curl -q --create-dirs -O --output-dir deps/prettier/ https://cdn.jsdelivr.net/npm/prettier@3.2.5/standalone.mjs
-	@test -f deps/prettier/html.mjs || curl -q --create-dirs -O --output-dir deps/prettier/ https://cdn.jsdelivr.net/npm/prettier@3.2.5/plugins/html.mjs
-	@test -f deps/prettier/babel.mjs || curl -q --create-dirs -O --output-dir deps/prettier/ https://cdn.jsdelivr.net/npm/prettier@3.2.5/plugins/babel.mjs
-	@test -f deps/prettier/estree.mjs || curl -q --create-dirs -O --output-dir deps/prettier/ https://cdn.jsdelivr.net/npm/prettier@3.2.5/plugins/estree.mjs
-.PHONY: fetchdeps
-
 ANDROID_DEPS := deps/openssl/android/arm64-v8a/usr/local/lib/libssl.a
 $(ANDROID_DEPS):
 	+@ANDROID_NDK_ROOT=$(ANDROID_NDK) tools/ssl-android
@@ -1083,6 +1120,10 @@ $(IOS_DEPS):
 $(filter $(BUILD_DIR)/ios%,$(APP_OBJS)): | $(IOS_DEPS)
 endif
 
+##
+## Linux package targets:
+##
+
 out/tildefriends-x86_64.AppImage: out/release/tildefriends out/data.zip
 	@echo "[appimage] $$@"
 	@rm -rf out/tildefriends.AppDir
@@ -1102,19 +1143,36 @@ out/tildefriends-x86_64.AppImage: out/release/tildefriends out/data.zip
 	@cd out; ./appimagetool --appimage-extract; cd ..
 	@cd out; unset SOURCE_DATE_EPOCH; PATH=$$PATH:squashfs-root/usr/bin ARCH=x86_64 squashfs-root/usr/bin/appimagetool -u 'zsync|https://dev.tildefriends.net/releases/tildefriends-x86_64.AppImage.zsync' tildefriends.AppDir tildefriends-x86_64.AppImage; cd ..
 
-appimage: out/tildefriends-x86_64.AppImage
+appimage: out/tildefriends-x86_64.AppImage ## Build an AppImage.
 .PHONY: appimage
 
-flatpak: out/
+flatpak: out/ ## Build a flatpak.
 	flatpak-builder --force-clean --user --install-deps-from=flathub --install --repo=out/flatpak-repo out/flatpak src/com.unprompted.tildefriends.yml
 	flatpak build-bundle out/flatpak-repo out/tildefriends.flatpak com.unprompted.tildefriends
 .PHONY: flatpak
 
-clean:
-	rm -rf $(BUILD_DIR)
-.PHONY: clean
+##
+## Targets for release management:
+##
 
-tarball:
+fetchdeps: ## Update various external sources that live in the tree that can't be pulled in as git submodules.
+	@echo "[fetch] sqlite"
+	@test -f out/deps/sqlite.zip && test "$$(cat out/deps/sqlite.txt 2>/dev/null)" = $(SQLITE_URL) || (mkdir -p out/deps/ && curl -q $(SQLITE_URL) -o out/deps/sqlite.zip)
+	@test -d deps/sqlite/ && test "$$(cat out/deps/sqlite.txt 2>/dev/null)" = $(SQLITE_URL) || (mkdir -p deps/sqlite/ && unzip -qDjo -d deps/sqlite/ out/deps/sqlite.zip)
+	@echo -n $(SQLITE_URL) > out/deps/sqlite.txt
+	@echo "[fetch] prettier"
+	@test -f deps/prettier/standalone.mjs || curl -q --create-dirs -O --output-dir deps/prettier/ https://cdn.jsdelivr.net/npm/prettier@3.2.5/standalone.mjs
+	@test -f deps/prettier/html.mjs || curl -q --create-dirs -O --output-dir deps/prettier/ https://cdn.jsdelivr.net/npm/prettier@3.2.5/plugins/html.mjs
+	@test -f deps/prettier/babel.mjs || curl -q --create-dirs -O --output-dir deps/prettier/ https://cdn.jsdelivr.net/npm/prettier@3.2.5/plugins/babel.mjs
+	@test -f deps/prettier/estree.mjs || curl -q --create-dirs -O --output-dir deps/prettier/ https://cdn.jsdelivr.net/npm/prettier@3.2.5/plugins/estree.mjs
+.PHONY: fetchdeps
+
+shots: ## Copy generated screenshots from `tildefriends test -t=auto` into place in the metadata/ directory.
+	@echo [shots] $(wildcard out/screenshot*.png)
+	@cp -f out/screenshot*.png metadata/en-US/images/phoneScreenshots/
+.PHONY: shots
+
+tarball: ## Build an all-inclusive source tarball (.tar.xz).
 	@echo [archive] out/tildefriends-$(VERSION_NUMBER).tar.xz
 	@rm -rf out/tildefriends-$(VERSION_NUMBER)
 	@mkdir -p out/tildefriends-$(VERSION_NUMBER)
@@ -1139,6 +1197,7 @@ tarball:
 		tildefriends-$(VERSION_NUMBER)
 .PHONY: tarball
 
+dist: ## Build versions of all distributables for release.
 dist: release-apk iosrelease-ipa aab $(if $(HAVE_WIN), out/winrelease/tildefriends.standalone.exe) out/TildeFriends-release.fdroid.apk appimage tarball
 	@mkdir -p dist/
 	@echo "[cp] tildefriends-$(VERSION_NUMBER).tar.xz"
@@ -1159,24 +1218,53 @@ dist: release-apk iosrelease-ipa aab $(if $(HAVE_WIN), out/winrelease/tildefrien
 	@cp out/tildefriends-x86_64.AppImage dist/TildeFriends-x86_64-$(VERSION_NUMBER).AppImage
 .PHONY: dist
 
-dist-test: dist
+dist-test: dist ## Exercise some built distributable files, making sure they work as intended.
 	@tar -xf tildefriends-$(VERSION_NUMBER).tar.xz
 	@$(MAKE) -C tildefriends-$(VERSION_NUMBER)/ debug release
 	@docker build tildefriends-$(VERSION_NUMBER)/
 	@rm -rf tildefriends-$(VERSION_NUMBER)
 .PHONY: dist-test
 
-format:
+##
+## Targets for tidying up:
+##
+
+format: ## Standardize formatting of C source.
 	@clang-format -i $(wildcard src/*.c src/*.h src/*.m)
 .PHONY: format
 
-prettier:
+prettier: ## Standardize formatting of JavaScript and Markdown source.
 	@npm run prettier
 .PHONY: prettier
 
-docs:
+clean: ## Clean all generated files from the out/ directory.
+	rm -rf $(BUILD_DIR)
+.PHONY: clean
+
+##
+## Documentation:
+##
+help: ## Display this help message.
+	@awk \
+	-F: \
+	-vG=$$(tput setaf 2) \
+	-vO=$$(tput setaf 3) \
+	-vB=$$(tput setaf 4) \
+	-vM=$$(tput setaf 5) \
+	-vC=$$(tput setaf 6) \
+	-vR=$$(tput sgr0) ' \
+	/^## ==.*==$$/ { sub(/^## ?/, ""); printf "%s%s%s\n", C, $$0, R } \
+	/^##.*:=.*/ { sub(/^## ?/, ""); sub(/:=/, ":"); printf "  %s%-20s%s %s%s%s\n", M, $$1, R, O, $$2, R } \
+	/^##/ { sub(/^## ?/, ""); print $$0 } \
+	/^[[:alnum:]-]+:.*##/ { \
+		sub(/:.*##\s?/, ":"); \
+		printf "  %s%-20s%s %s%s%s\n", G, $$1, R, O, $$2, R \
+	} \
+	' < $(filter-out %.d,$(MAKEFILE_LIST))
+	@echo "" # Blank line.
+.PHONY: help
+.DEFAULT_GOAL := help
+
+docs: ## Build HTML docs.
 	@doxygen
 .PHONY: docs
-
-fdroid: out/apk/TildeFriends-release.fdroid.unsigned.apk
-.PHONY: fdroid

README.md

@@ -14,10 +14,7 @@ Scuttlebutt, as well as a platform for writing and running web applications.
 3. Make creating and sharing web applications accessible to anyone with a
    browser.
 
-## Building
-
-Builds on Linux (x86_64 and aarch64), MacOS, OpenBSD, and Haiku. Builds for
-all of those host platforms plus mingw64, iOS, and android.
+## Getting the Source
 
 Tilde Friends uses git submodules, so either:
 
@@ -35,20 +32,35 @@ git submodule update --init --recursive
 
 The `.tar.xz` source releases are all-inclusive.
 
-1. On Linux only, system OpenSSL libraries (`libssl-dev`, in debian-speak) are
-   assumed to be available.
-2. To build, run `make debug` or `make release`. An executable will be
-   generated in a subdirectory of `out/`.
-3. It's possible to build for Android, iOS, and Windows on Linux, if you have
-   the right dependencies in the right places. `make windebug winrelease
-iosdebug-ipa iosrelease-ipa release-apk`.
-4. To build in docker, `docker build .`.
-5. `make format` will normalize formatting to the coding standard.
+## Building
+
+Builds on Linux (x86_64 and aarch64), MacOS, OpenBSD, and Haiku. It's possible
+to build for Android, iOS, and Windows on Linux, if you have the right
+dependencies in the right places.
+
+### Requirements
+
+On Linux only, system OpenSSL libraries (`libssl-dev`, in debian-speak) are
+assumed to be available.
+
+On MacOS, Xcode's command-line tools are expected to be available.
+
+### Build Commands
+
+Run `make` with no arguments to see available build targets and options. `make
+debug` is a good place to start.
+
+To build in docker, `docker build .`.
+
+`make format` and `make prettier` will normalize formatting to the coding
+standard.
 
 ## Running
 
-By default, running the built `tildefriends` executable will start a web server
-at <http://localhost:12345/>. `tildefriends -h` lists further options.
+By default, running the built `out/debug/tildefriends` executable will start a
+web server at <http://localhost:12345/>. It expects to be run with the
+repository root as the current working directory. `tildefriends -h` lists
+further options.
 
 The first user to create an account and log in will be granted administrative
 privileges. Further administration can be done at
@@ -57,7 +69,7 @@ privileges. Further administration can be done at
 ## Documentation
 
 Docs are a work in progress:
-<https://www.tildefriends.net/~cory/wiki/#test-wiki/tf-app-quick-reference>.
+<https://dev.tildefriends.net/cory/tildefriends/wiki>.
 
 ## License
 

apps/api.json

@@ -1,5 +1,5 @@
 {
 	"type": "tildefriends-app",
 	"emoji": "📜",
-	"previous": "&miGORZ8BwjHg2YO0t4bms6SI28XWPYqnqOZ8u9zsbZc=.sha256"
+	"previous": "&BEf0nraBdHk/+PWqx6tOSu5rheWVaxaL7orAOz3285M=.sha256"
 }

apps/api/app.js

@@ -21,7 +21,7 @@ function* treeify(prefix, o) {
 
 function markdown(md) {
 	let parsed = new commonmark.Parser().parse(md ?? '*undocumented*');
-	return new commonmark.HtmlRenderer().render(parsed);
+	return new commonmark.HtmlRenderer({safe: true}).render(parsed);
 }
 
 function document(api) {

apps/blog.json

@@ -1,5 +1,5 @@
 {
 	"type": "tildefriends-app",
 	"emoji": "🪵",
-	"previous": "&TIrBnpN3iz3O9L9MCCteAcVJZjA83EKdcfu4SCM76VE=.sha256"
+	"previous": "&3jabNEk6W2uolzTvfXX6fcWF50N3501vtgZ6ZxFVJ1s=.sha256"
 }

apps/blog/blog.js

@@ -52,8 +52,8 @@ export async function get_blog_message(id) {
 }
 
 export function markdown(md) {
-	let reader = new commonmark.Parser({safe: true});
-	let writer = new commonmark.HtmlRenderer();
+	let reader = new commonmark.Parser();
+	let writer = new commonmark.HtmlRenderer({safe: true});
 	let parsed = reader.parse(md || '');
 	let walker = parsed.walker();
 	let event, node;

apps/issues.json

@@ -1,5 +1,5 @@
 {
 	"type": "tildefriends-app",
 	"emoji": "🦟",
-	"previous": "&cUqvSDUls3jn0haD85LPFAGdkc8wFuy347TtATNcJgg=.sha256"
+	"previous": "&O0huuEgL/UQC9bkUfhPOyZFo9eRiz+koOkba6QwCGNA=.sha256"
 }

apps/issues/tf-utils.js

@@ -1,5 +1,11 @@
 import * as linkify from './commonmark-linkify.js';
 
+var reUnsafeProtocol = /^javascript:|vbscript:|file:|data:/i;
+var reSafeDataProtocol = /^data:image\/(?:png|gif|jpeg|webp)/i;
+var potentiallyUnsafe = function (url) {
+	return reUnsafeProtocol.test(url) && !reSafeDataProtocol.test(url);
+};
+
 function image(node, entering) {
 	if (
 		node.firstChild?.type === 'text' &&
@@ -61,8 +67,8 @@ function image(node, entering) {
 }
 
 export function markdown(md) {
-	var reader = new commonmark.Parser({safe: true});
-	var writer = new commonmark.HtmlRenderer();
+	var reader = new commonmark.Parser();
+	var writer = new commonmark.HtmlRenderer({safe: true});
 	writer.image = image;
 	var parsed = reader.parse(md || '');
 	parsed = linkify.transform(parsed);

apps/journal.json

@@ -1,5 +1,5 @@
 {
 	"type": "tildefriends-app",
 	"emoji": "📝",
-	"previous": "&b//KqE4Vx6kOSBRODK1p/8wjOLKZJ+CBB5IkaBt5YsM=.sha256"
+	"previous": "&5LpOTEnor/rYFk3axyfmmehAoq9aEwNQRH4jwNhRQ7o=.sha256"
 }

apps/journal/tf-journal-entry.js

@@ -18,8 +18,8 @@ class TfJournalEntryElement extends LitElement {
 	}
 
 	markdown(md) {
-		var reader = new commonmark.Parser({safe: true});
-		var writer = new commonmark.HtmlRenderer();
+		var reader = new commonmark.Parser();
+		var writer = new commonmark.HtmlRenderer({safe: true});
 		var parsed = reader.parse(md || '');
 		return writer.render(parsed);
 	}

apps/ssb.json

@@ -1,5 +1,5 @@
 {
 	"type": "tildefriends-app",
 	"emoji": "🐌",
-	"previous": "&IH3DadMNF785idnMI/LuCpIJQxzvpg1PDp8BI7m1Nx0=.sha256"
+	"previous": "&q/1uGp0jMvsYGW7Gj8E33kf6UFo/uNYDXg3zo1sVKQg=.sha256"
 }

apps/ssb/commonmark-hashtag.js

@@ -7,7 +7,7 @@ function textNode(text) {
 function linkNode(text, link) {
 	const linkNode = new commonmark.Node('link', undefined);
 	if (link.startsWith('#')) {
-		linkNode.destination = `#q=${encodeURIComponent(link)}`;
+		linkNode.destination = `#${encodeURIComponent('#' + link)}`;
 	} else {
 		linkNode.destination = link;
 	}

apps/ssb/emojis.js

@@ -37,10 +37,12 @@ export async function picker(callback, anchor, author) {
 	div.style.color = '#000';
 	div.style.background = '#fff';
 	div.style.border = '1px solid #000';
-	div.style.display = 'block';
+	div.style.display = 'flex';
 	div.style.overflow = 'scroll';
 	div.style.fontWeight = 'bold';
 	div.style.fontSize = 'xx-large';
+	div.style.flex = '1 1';
+	div.style.flexDirection = 'column';
 	let input = document.createElement('input');
 	input.type = 'text';
 	input.style.display = 'block';
@@ -50,6 +52,7 @@ export async function picker(callback, anchor, author) {
 	input.style.position = 'relative';
 	div.appendChild(input);
 	let list = document.createElement('div');
+	list.style.overflow = 'scroll';
 	div.appendChild(list);
 	div.addEventListener('mousedown', function (event) {
 		event.stopPropagation();
@@ -142,21 +145,40 @@ export async function picker(callback, anchor, author) {
 	}
 	refresh();
 	input.oninput = refresh;
-	let modal = html`
-		<style>
-			${styles}
-		</style>
-		<div class="w3-modal" style="display: block">
-			<div class="w3-modal-content w3-card-4">${div}</div>
-		</div>
-	`;
 	let parent = document.createElement('div');
-	document.body.appendChild(parent);
 	function cleanup() {
 		parent.parentElement.removeChild(parent);
 		window.removeEventListener('keydown', key_down);
 		document.body.removeEventListener('mousedown', cleanup);
 	}
+	let modal = html`
+		<style>
+			${styles}
+		</style>
+		<div
+			class="w3-modal"
+			style="display: block; box-sizing: border-box; z-index: 10"
+		>
+			<div class="w3-modal-content w3-card-4">
+				<div
+					class="w3-content w3-theme-d1"
+					style="display: flex; flex-direction: column; max-height: 50vh"
+				>
+					<header class="w3-container" style="flex: 0 0">
+						<h1>Choose a Reaction</h1>
+						<span class="w3-button w3-display-topright" @click=${cleanup}
+							>&times;</span
+						>
+					</header>
+					${div}
+					<footer class="w3-container w3-padding" style="flex: 0 0">
+						<button class="w3-button" @click=${cleanup}>Close</button>
+					</footer>
+				</div>
+			</div>
+		</div>
+	`;
+	document.body.appendChild(parent);
 	render(modal, parent);
 	input.focus();
 	document.body.addEventListener('mousedown', cleanup);

apps/ssb/script.js

@@ -8,10 +8,16 @@ import * as tf_compose from './tf-compose.js';
 import * as tf_news from './tf-news.js';
 import * as tf_profile from './tf-profile.js';
 import * as tf_reactions_modal from './tf-reactions-modal.js';
-import * as tf_tab_mentions from './tf-tab-mentions.js';
 import * as tf_tab_news from './tf-tab-news.js';
 import * as tf_tab_news_feed from './tf-tab-news-feed.js';
 import * as tf_tab_search from './tf-tab-search.js';
 import * as tf_tab_connections from './tf-tab-connections.js';
 import * as tf_tab_query from './tf-tab-query.js';
 import * as tf_tag from './tf-tag.js';
+import * as tf_styles from './tf-styles.js';
+
+window.addEventListener('load', function () {
+	let style = document.createElement('style');
+	style.innerText = tf_styles.styles;
+	document.body.appendChild(style);
+});

apps/ssb/tf-app.js

@@ -16,7 +16,9 @@ class TfElement extends LitElement {
 			following: {type: Array},
 			users: {type: Object},
 			ids: {type: Array},
-			tags: {type: Array},
+			channels: {type: Array},
+			channels_unread: {type: Object},
+			channels_latest: {type: Object},
 		};
 	}
 
@@ -33,7 +35,11 @@ class TfElement extends LitElement {
 		this.following = [];
 		this.users = {};
 		this.loaded = false;
-		this.tags = [];
+		this.channels = [];
+		this.channels_unread = {};
+		this.channels_latest = {};
+		this.loading_channels_latest = 0;
+		this.loading_channels_latest_scheduled = 0;
 		tfrpc.rpc.getBroadcasts().then((b) => {
 			self.broadcasts = b || [];
 		});
@@ -64,16 +70,76 @@ class TfElement extends LitElement {
 		let ids = (await tfrpc.rpc.getIdentities()) || [];
 		this.whoami = whoami ?? (ids.length ? ids[0] : undefined);
 		this.ids = ids;
+		await this.load_channels();
+	}
+
+	async load_channels() {
+		let channels = await tfrpc.rpc.query(
+			`
+			SELECT
+				content ->> 'channel' AS channel,
+				content ->> 'subscribed' AS subscribed
+			FROM
+				messages
+			WHERE
+				author = ? AND
+				content ->> 'type' = 'channel'
+			ORDER BY sequence
+		`,
+			[this.whoami]
+		);
+		let channel_map = {};
+		for (let row of channels) {
+			if (row.subscribed) {
+				channel_map[row.channel] = true;
+			} else {
+				delete channel_map[row.channel];
+			}
+		}
+		this.channels = Object.keys(channel_map).sort();
+	}
+
+	connectedCallback() {
+		super.connectedCallback();
+		this._keydown = this.keydown.bind(this);
+		window.addEventListener('keydown', this._keydown);
+	}
+
+	disconnectedCallback() {
+		super.disconnectedCallback();
+		window.removeEventListener('keydown', this._keydown);
+	}
+
+	keydown(event) {
+		if (event.altKey && event.key == 'ArrowUp') {
+			this.next_channel(1);
+			event.preventDefault();
+		} else if (event.altKey && event.key == 'ArrowDown') {
+			this.next_channel(-1);
+			event.preventDefault();
+		}
+	}
+
+	next_channel(delta) {
+		let channel_names = ['', '@'].concat(this.channels);
+		let index = channel_names.indexOf(this.hash.substring(1));
+		if (index != -1) {
+			index += delta;
+			this.set_hash(
+				'#' +
+					encodeURIComponent(
+						channel_names[(index + channel_names.length) % channel_names.length]
+					)
+			);
+		}
 	}
 
 	set_hash(hash) {
-		this.hash = hash || '#';
+		this.hash = decodeURIComponent(hash || '#');
 		if (this.hash.startsWith('#q=')) {
 			this.tab = 'search';
 		} else if (this.hash === '#connections') {
 			this.tab = 'connections';
-		} else if (this.hash === '#mentions') {
-			this.tab = 'mentions';
 		} else if (this.hash.startsWith('#sql=')) {
 			this.tab = 'query';
 		} else {
@@ -167,10 +233,19 @@ class TfElement extends LitElement {
 			`,
 			[JSON.stringify(this.following), id]
 		);
+		for (let message of messages) {
+			if (message.author == this.whoami) {
+				let content = JSON.parse(message.content);
+				if (content?.type == 'channel') {
+					this.load_channels();
+				}
+			}
+		}
 		if (messages && messages.length) {
 			this.unread = [...this.unread, ...messages];
 			this.unread = this.unread.slice(this.unread.length - 1024);
 		}
+		this.schedule_load_channels_latest();
 	}
 
 	async _handle_whoami_changed(event) {
@@ -195,32 +270,94 @@ class TfElement extends LitElement {
 		}
 	}
 
-	async load_recent_tags() {
-		let start = new Date();
-		this.tags = await tfrpc.rpc.query(
-			`
-			WITH
-				recent AS (SELECT id, json(content) AS content FROM messages
-					WHERE messages.timestamp > ? AND json_extract(content, '$.type') = 'post'
-					ORDER BY timestamp DESC LIMIT 1024),
-				recent_channels AS (SELECT recent.id, '#' || json_extract(content, '$.channel') AS tag
-					FROM recent
-					WHERE json_extract(content, '$.channel') IS NOT NULL),
-				recent_mentions AS (SELECT recent.id, json_extract(mention.value, '$.link') AS tag
-					FROM recent, json_each(recent.content, '$.mentions') AS mention
-					WHERE json_valid(mention.value) AND tag LIKE '#%'),
-				combined AS (SELECT id, tag FROM recent_channels UNION ALL SELECT id, tag FROM recent_mentions),
-				by_message AS (SELECT DISTINCT id, tag FROM combined)
-			SELECT tag, COUNT(*) AS count FROM by_message GROUP BY tag ORDER BY count DESC LIMIT 10
-		`,
-			[new Date() - 7 * 24 * 60 * 60 * 1000]
-		);
-		console.log('tags took', (new Date() - start) / 1000.0, 'seconds');
+	async get_latest_private(following) {
+		let latest = (
+			await tfrpc.rpc.query('SELECT MAX(rowid) AS latest FROM messages')
+		)[0].latest;
+		const k_chunk_count = 256;
+		while (latest - k_chunk_count >= 0) {
+			let messages = await tfrpc.rpc.query(
+				`
+					SELECT messages.rowid, messages.id, previous, author, sequence, timestamp, hash, json(content) AS content, signature
+						FROM messages
+						JOIN json_each(?1) AS following ON messages.author = following.value
+						WHERE
+							messages.rowid > ?2 AND
+							messages.rowid <= ?3 AND
+							json(messages.content) LIKE '"%'
+						ORDER BY sequence DESC
+					`,
+				[JSON.stringify(following), latest - k_chunk_count, latest]
+			);
+			messages = (await this.decrypt(messages)).filter((x) => x.decrypted);
+			if (messages.length) {
+				return Math.max(...messages.map((x) => x.rowid));
+			}
+			latest -= k_chunk_count;
+		}
+		return -1;
+	}
+
+	async load_channels_latest(following) {
+		this.loading_channels_latest++;
+		try {
+			let start_time = new Date();
+			let latest_private = this.get_latest_private(following);
+			let channels = await tfrpc.rpc.query(
+				`
+				SELECT channels.value AS channel, MAX(messages.rowid) AS rowid FROM messages
+				JOIN json_each(?1) AS channels ON messages.content ->> 'channel' = channels.value
+				JOIN json_each(?2) AS following ON messages.author = following.value
+				WHERE messages.content ->> 'type' = 'post' AND messages.content ->> 'root' IS NULL
+				GROUP by channel
+				UNION
+				SELECT '' AS channel, MAX(messages.rowid) AS rowid FROM messages
+				JOIN json_each(?2) AS following ON messages.author = following.value
+				UNION
+				SELECT '@' AS channel, MAX(messages.rowid) AS rowid FROM messages_fts(?3)
+				JOIN messages ON messages.rowid = messages_fts.rowid
+				JOIN json_each(?2) AS following ON messages.author = following.value
+			`,
+				[
+					JSON.stringify(this.channels),
+					JSON.stringify(following),
+					'"' + this.whoami.replace('"', '""') + '"',
+				]
+			);
+			this.channels_latest = Object.fromEntries(
+				channels.map((x) => [x.channel, x.rowid])
+			);
+			console.log('latest', this.channels_latest);
+			console.log('unread', this.channels_unread);
+			console.log('channels took', (new Date() - start_time) / 1000.0);
+			let self = this;
+			latest_private.then(function (latest) {
+				self.channels_latest = Object.assign({}, self.channels_latest, {
+					'🔐': latest,
+				});
+				console.log('private took', (new Date() - start_time) / 1000.0);
+			});
+		} finally {
+			this.loading_channels_latest--;
+		}
+	}
+
+	_schedule_load_channels_latest_timer() {
+		--this.loading_channels_latest_scheduled;
+		this.schedule_load_channels_latest();
+	}
+
+	schedule_load_channels_latest() {
+		if (!this.loading_channels_latest) {
+			this.load_channels_latest(this.following);
+		} else if (!this.loading_channels_latest_scheduled) {
+			this.loading_channels_latest_scheduled++;
+			setTimeout(this._schedule_load_channels_latest_timer, 5000);
+		}
 	}
 
 	async load() {
 		let whoami = this.whoami;
-		let tags = this.load_recent_tags();
 		let following = await tfrpc.rpc.following([whoami], 2);
 		let users = {};
 		let by_count = [];
@@ -233,7 +370,10 @@ class TfElement extends LitElement {
 			};
 			by_count.push({count: v.of, id: id});
 		}
-		console.log(by_count.sort((x, y) => y.count - x.count).slice(0, 20));
+		let channels_latest = this.load_channels_latest(Object.keys(following));
+		this.channels_unread = JSON.parse(
+			(await tfrpc.rpc.databaseGet('unread')) ?? '{}'
+		);
 		let start_time = new Date();
 		users = await this.fetch_about(Object.keys(following).sort(), users);
 		console.log(
@@ -243,14 +383,47 @@ class TfElement extends LitElement {
 			Object.keys(users).length,
 			'users'
 		);
+		start_time = new Date();
+		await channels_latest;
 		this.following = Object.keys(following);
 		this.users = users;
-		await tags;
 		console.log(`load finished ${whoami} => ${this.whoami}`);
 		this.whoami = whoami;
 		this.loaded = whoami;
 	}
 
+	channel_set_unread(event) {
+		this.channels_unread[event.detail.channel ?? ''] = event.detail.unread;
+		this.channels_unread = Object.assign({}, this.channels_unread);
+		tfrpc.rpc.databaseSet('unread', JSON.stringify(this.channels_unread));
+	}
+
+	async decrypt(messages) {
+		let whoami = this.whoami;
+		return Promise.all(
+			messages.map(async function (message) {
+				let content;
+				try {
+					content = JSON.parse(message?.content);
+				} catch {}
+				if (typeof content === 'string') {
+					let decrypted;
+					try {
+						decrypted = await tfrpc.rpc.try_decrypt(whoami, content);
+					} catch {}
+					if (decrypted) {
+						try {
+							message.decrypted = JSON.parse(decrypted);
+						} catch {
+							message.decrypted = decrypted;
+						}
+					}
+				}
+				return message;
+			})
+		);
+	}
+
 	render_tab() {
 		let following = this.following;
 		let users = this.users;
@@ -265,6 +438,10 @@ class TfElement extends LitElement {
 					.unread=${this.unread}
 					@refresh=${() => (this.unread = [])}
 					?loading=${this.loading}
+					.channels=${this.channels}
+					.channels_latest=${this.channels_latest}
+					.channels_unread=${this.channels_unread}
+					@channelsetunread=${this.channel_set_unread}
 				></tf-tab-news>
 			`;
 		} else if (this.tab === 'connections') {
@@ -275,14 +452,6 @@ class TfElement extends LitElement {
 					.broadcasts=${this.broadcasts}
 				></tf-tab-connections>
 			`;
-		} else if (this.tab === 'mentions') {
-			return html`
-				<tf-tab-mentions
-					.following=${this.following}
-					whoami=${this.whoami}
-					.users="${this.users}}"
-				></tf-tab-mentions>
-			`;
 		} else if (this.tab === 'search') {
 			return html`
 				<tf-tab-search
@@ -314,13 +483,15 @@ class TfElement extends LitElement {
 			await tfrpc.rpc.setHash('#');
 		} else if (tab === 'connections') {
 			await tfrpc.rpc.setHash('#connections');
-		} else if (tab === 'mentions') {
-			await tfrpc.rpc.setHash('#mentions');
 		} else if (tab === 'query') {
 			await tfrpc.rpc.setHash('#sql=');
 		}
 	}
 
+	refresh() {
+		tfrpc.rpc.sync();
+	}
+
 	render() {
 		let self = this;
 
@@ -334,13 +505,21 @@ class TfElement extends LitElement {
 		const k_tabs = {
 			'📰': 'news',
 			'📡': 'connections',
-			'@': 'mentions',
 			'🔍': 'search',
 			'👩‍💻': 'query',
 		};
 
 		let tabs = html`
-			<div class="w3-bar w3-theme-l1">
+			<div
+				class="w3-bar w3-theme-l1"
+				style="position: sticky; top: 0; z-index: 10"
+			>
+				<button
+					class="w3-bar-item w3-button w3-circle w3-ripple"
+					@click=${this.refresh}
+				>
+					↻
+				</button>
 				${Object.entries(k_tabs).map(
 					([k, v]) => html`
 						<button
@@ -374,13 +553,7 @@ class TfElement extends LitElement {
 				style="width: 100vw; min-height: 100vh; height: 100%"
 				class="w3-theme-dark"
 			>
-				${tabs}
-				<div style="padding: 8px">
-					${this.tags.map(
-						(x) => html`<tf-tag tag=${x.tag} count=${x.count}></tf-tag>`
-					)}
-					${contents}
-				</div>
+				${tabs} ${contents}
 			</div>
 		`;
 	}

apps/ssb/tf-compose.js

@@ -14,6 +14,7 @@ class TfComposeElement extends LitElement {
 			apps: {type: Object},
 			drafts: {type: Object},
 			author: {type: String},
+			channel: {type: String},
 		};
 	}
 
@@ -196,6 +197,7 @@ class TfComposeElement extends LitElement {
 		let message = {
 			type: 'post',
 			text: edit.innerText,
+			channel: this.channel,
 		};
 		if (this.root || this.branch) {
 			message.root = this.root;
@@ -535,6 +537,9 @@ class TfComposeElement extends LitElement {
 				class="w3-card-4 w3-theme-d4 w3-padding-small"
 				style="box-sizing: border-box"
 			>
+				${this.channel !== undefined
+					? html`<p>To #${this.channel}:</p>`
+					: undefined}
 				${this.render_encrypt()}
 				<div class="w3-container w3-padding-small">
 					<div class="w3-half">

apps/ssb/tf-message.js

@@ -14,6 +14,8 @@ class TfMessageElement extends LitElement {
 			format: {type: String},
 			blog_data: {type: String},
 			expanded: {type: Object},
+			channel: {type: String},
+			channel_unread: {type: Number},
 		};
 	}
 
@@ -28,6 +30,7 @@ class TfMessageElement extends LitElement {
 		this.drafts = {};
 		this.format = 'message';
 		this.expanded = {};
+		this.channel_unread = -1;
 	}
 
 	show_reply() {
@@ -228,7 +231,7 @@ class TfMessageElement extends LitElement {
 				>${mention.name}</a
 			>`;
 		} else if (mention.link?.startsWith('#')) {
-			return html` <a href=${'#q=' + encodeURIComponent(mention.link)}
+			return html` <a href=${'#' + encodeURIComponent('#' + mention.link)}
 				>${mention.link}</a
 			>`;
 		} else if (
@@ -312,12 +315,27 @@ ${JSON.stringify(mention, null, 2)}</pre
 								.users=${this.users}
 								.drafts=${this.drafts}
 								.expanded=${this.expanded}
+								channel=${this.channel}
+								channel_unread=${this.channel_unread}
 							></tf-message>`
 					)}`;
 			}
 		}
 	}
 
+	mark_unread() {
+		this.dispatchEvent(
+			new CustomEvent('channelsetunread', {
+				bubbles: true,
+				composed: true,
+				detail: {
+					channel: this.channel,
+					unread: this.message.rowid,
+				},
+			})
+		);
+	}
+
 	render_channels() {
 		let content = this.message?.content;
 		if (this?.messsage?.decrypted?.type == 'post') {
@@ -344,7 +362,9 @@ ${JSON.stringify(mention, null, 2)}</pre
 		}
 		let class_background = this.message?.decrypted
 			? 'w3-pale-red'
-			: 'w3-theme-d4';
+			: this.message?.rowid >= this.channel_unread
+				? 'w3-theme-d2'
+				: 'w3-theme-d4';
 		let self = this;
 		let raw_button;
 		switch (this.format) {
@@ -403,14 +423,15 @@ ${JSON.stringify(mention, null, 2)}</pre
 			let body;
 			return html`
 				<div
-					class="w3-card-4 w3-theme-d4 w3-border-theme"
+					class="w3-card-4 ${class_background} w3-border-theme"
 					style="margin-top: 8px; padding: 16px; display: inline-block; overflow-wrap: anywhere"
 				>
 					<tf-user id=${self.message.author} .users=${self.users}></tf-user>
 					<span style="padding-right: 8px"
-						><a tfarget="_top" href=${'#' + self.message.id}>%</a> ${new Date(
-							self.message.timestamp
-						).toLocaleString()}</span
+						><a tfarget="_top" href=${'#' + encodeURIComponent(self.message.id)}
+							>%</a
+						>
+						${new Date(self.message.timestamp).toLocaleString()}</span
 					>
 					${raw_button} ${self.format == 'raw' ? self.render_raw() : inner}
 					${self.render_votes()}
@@ -422,6 +443,8 @@ ${JSON.stringify(mention, null, 2)}</pre
 								.users=${self.users}
 								.drafts=${self.drafts}
 								.expanded=${self.expanded}
+								channel=${self.channel}
+								channel_unread=${self.channel_unread}
 							></tf-message>
 						`
 					)}
@@ -430,7 +453,7 @@ ${JSON.stringify(mention, null, 2)}</pre
 		}
 		if (this.message?.type === 'contact_group') {
 			return html` <div
-				class="w3-card-4 w3-theme-d4 w3-border-theme"
+				class="w3-card-4 ${class_background} w3-border-theme"
 				style="margin-top: 8px; padding: 16px; overflow-wrap: anywhere"
 			>
 				${this.message.messages.map(
@@ -441,15 +464,19 @@ ${JSON.stringify(mention, null, 2)}</pre
 							.users=${this.users}
 							.drafts=${this.drafts}
 							.expanded=${this.expanded}
+							channel=${this.channel}
+							channel_unread=${this.channel_unread}
 						></tf-message>`
 				)}
 			</div>`;
 		} else if (this.message.placeholder) {
 			return html` <div
-				class="w3-card-4 w3-theme-d4 w3-border-theme"
+				class="w3-card-4 ${class_background} w3-border-theme"
 				style="margin-top: 8px; padding: 16px; overflow-wrap: anywhere"
 			>
-				<a target="_top" href=${'#' + this.message.id}>${this.message.id}</a>
+				<a target="_top" href=${'#' + encodeURIComponent(this.message.id)}
+					>${this.message.id}</a
+				>
 				(placeholder)
 				<div>${this.render_votes()}</div>
 				${(this.message.child_messages || []).map(
@@ -460,6 +487,8 @@ ${JSON.stringify(mention, null, 2)}</pre
 							.users=${this.users}
 							.drafts=${this.drafts}
 							.expanded=${this.expanded}
+							channel=${this.channel}
+							channel_unread=${this.channel_unread}
 						></tf-message>
 					`
 				)}
@@ -600,7 +629,11 @@ ${JSON.stringify(content, null, 2)}</pre
 							${is_encrypted}
 							<span style="flex: 1"></span>
 							<span style="padding-right: 8px"
-								><a target="_top" href=${'#' + self.message.id}>%</a>
+								><a
+									target="_top"
+									href=${'#' + encodeURIComponent(self.message.id)}
+									>%</a
+								>
 								${new Date(this.message.timestamp).toLocaleString()}</span
 							>
 							<span>${raw_button}</span>
@@ -611,6 +644,16 @@ ${JSON.stringify(content, null, 2)}</pre
 							<button class="w3-button w3-theme-d1" @click=${this.react}>
 								React
 							</button>
+							${!content.root && this.message.rowid < this.channel_unread
+								? html`
+										<button
+											class="w3-button w3-theme-d1"
+											@click=${this.mark_unread}
+										>
+											Mark Unread
+										</button>
+									`
+								: undefined}
 						</p>
 						${this.render_children()}
 					</div>
@@ -643,7 +686,11 @@ ${JSON.stringify(content, null, 2)}</pre
 							${is_encrypted}
 							<span style="flex: 1"></span>
 							<span style="padding-right: 8px"
-								><a target="_top" href=${'#' + self.message.id}>%</a>
+								><a
+									target="_top"
+									href=${'#' + encodeURIComponent(self.message.id)}
+									>%</a
+								>
 								${new Date(this.message.timestamp).toLocaleString()}</span
 							>
 							<span>${raw_button}</span>
@@ -726,14 +773,18 @@ ${JSON.stringify(content, null, 2)}</pre
 						}
 					</style>
 					<div
-						class="w3-card-4 w3-theme-d4 w3-border-theme"
+						class="w3-card-4 ${class_background} w3-border-theme"
 						style="margin-top: 8px; padding: 16px"
 					>
 						<div style="display: flex; flex-direction: row">
 							<tf-user id=${this.message.author} .users=${this.users}></tf-user>
 							<span style="flex: 1"></span>
 							<span style="padding-right: 8px"
-								><a target="_top" href=${'#' + self.message.id}>%</a>
+								><a
+									target="_top"
+									href=${'#' + encodeURIComponent(self.message.id)}
+									>%</a
+								>
 								${new Date(this.message.timestamp).toLocaleString()}</span
 							>
 							<span>${raw_button}</span>
@@ -772,7 +823,7 @@ ${JSON.stringify(content, null, 2)}</pre
 				return small_frame(html`
 					<div>
 						${content.subscribed ? 'subscribed to' : 'unsubscribed from'}
-						<a href=${'#q=' + encodeURIComponent('#' + content.channel)}
+						<a href=${'#' + encodeURIComponent('#' + content.channel)}
 							>#${content.channel}</a
 						>
 					</div>

apps/ssb/tf-news.js

@@ -11,6 +11,8 @@ class TfNewsElement extends LitElement {
 			following: {type: Array},
 			drafts: {type: Object},
 			expanded: {type: Object},
+			channel: {type: String},
+			channel_unread: {type: Number},
 		};
 	}
 
@@ -25,6 +27,7 @@ class TfNewsElement extends LitElement {
 		this.following = [];
 		this.drafts = {};
 		this.expanded = {};
+		this.channel_unread = -1;
 	}
 
 	process_messages(messages) {
@@ -33,12 +36,13 @@ class TfNewsElement extends LitElement {
 
 		console.log('processing', messages.length, 'messages');
 
-		function ensure_message(id) {
+		function ensure_message(id, rowid) {
 			let found = messages_by_id[id];
 			if (found) {
 				return found;
 			} else {
 				let added = {
+					rowid: rowid,
 					id: id,
 					placeholder: true,
 					content: '"placeholder"',
@@ -53,7 +57,7 @@ class TfNewsElement extends LitElement {
 
 		function link_message(message) {
 			if (message.content.type === 'vote') {
-				let parent = ensure_message(message.content.vote.link);
+				let parent = ensure_message(message.content.vote.link, message.rowid);
 				if (!parent.votes) {
 					parent.votes = [];
 				}
@@ -62,14 +66,14 @@ class TfNewsElement extends LitElement {
 			} else if (message.content.type == 'post') {
 				if (message.content.root) {
 					if (typeof message.content.root === 'string') {
-						let m = ensure_message(message.content.root);
+						let m = ensure_message(message.content.root, message.rowid);
 						if (!m.child_messages) {
 							m.child_messages = [];
 						}
 						m.child_messages.push(message);
 						message.parent_message = message.content.root;
 					} else {
-						let m = ensure_message(message.content.root[0]);
+						let m = ensure_message(message.content.root[0], message.rowid);
 						if (!m.child_messages) {
 							m.child_messages = [];
 						}
@@ -162,6 +166,7 @@ class TfNewsElement extends LitElement {
 			} else {
 				if (group.length > 0) {
 					result.push({
+						rowid: Math.max(...group.map((x) => x.rowid)),
 						type: 'contact_group',
 						messages: group,
 					});
@@ -170,6 +175,13 @@ class TfNewsElement extends LitElement {
 				result.push(message);
 			}
 		}
+		if (group.length > 0) {
+			result.push({
+				rowid: Math.max(...group.map((x) => x.rowid)),
+				type: 'contact_group',
+				messages: group,
+			});
+		}
 		return result;
 	}
 
@@ -178,18 +190,38 @@ class TfNewsElement extends LitElement {
 		let final_messages = this.group_following(
 			this.finalize_messages(messages_by_id)
 		);
+		let unread_rowid = -1;
+		for (let message of final_messages) {
+			if (message.rowid >= this.channel_unread) {
+				unread_rowid = message.rowid;
+			}
+		}
 		return html`
-			<div style="display: flex; flex-direction: column">
+			<div>
 				${final_messages.map(
-					(x) =>
-						html`<tf-message
+					(x) => html`
+						<tf-message
 							.message=${x}
 							whoami=${this.whoami}
 							.users=${this.users}
 							.drafts=${this.drafts}
 							.expanded=${this.expanded}
 							collapsed="true"
-						></tf-message>`
+							channel=${this.channel}
+							channel_unread=${this.channel_unread}
+						></tf-message>
+						${x.rowid == unread_rowid && x != final_messages[0]
+							? html`<div style="display: flex; flex-direction: row">
+									<div
+										style="border-bottom: 1px solid #f00; flex: 1; align-self: center; height: 1px"
+									></div>
+									<div style="color: #f00; padding: 8px">unread</div>
+									<div
+										style="border-bottom: 1px solid #f00; flex: 1; align-self: center; height: 1px"
+									></div>
+								</div>`
+							: undefined}
+					`
 				)}
 			</div>
 		`;

apps/ssb/tf-profile.js

@@ -233,7 +233,11 @@ class TfProfileElement extends LitElement {
 					</button>`;
 				}
 				edit = html`
-					<button class="w3-button w3-theme-d1" @click=${this.save_edits}>
+					<button
+						id="save_profile"
+						class="w3-button w3-theme-d1"
+						@click=${this.save_edits}
+					>
 						Save Profile
 					</button>
 					<button class="w3-button w3-theme-d1" @click=${this.discard_edits}>
@@ -242,7 +246,11 @@ class TfProfileElement extends LitElement {
 					${server_follow}
 				`;
 			} else {
-				edit = html`<button class="w3-button w3-theme-d1" @click=${this.edit}>
+				edit = html`<button
+					id="edit_profile"
+					class="w3-button w3-theme-d1"
+					@click=${this.edit}
+				>
 					Edit Profile
 				</button>`;
 			}
@@ -289,16 +297,10 @@ class TfProfileElement extends LitElement {
 			typeof profile.image == 'string' ? profile.image : profile.image?.link;
 		image = this.editing?.image ?? image;
 		let description = this.editing?.description ?? profile.description;
-		return html`<div style="border: 2px solid black; background-color: rgba(255, 255, 255, 0.2); padding: 16px">
-			<tf-user id=${this.id} .users=${this.users}></tf-user> (${tfutils.human_readable_size(this.size)})
-			<div class="w3-row">
-				<div class="w3-col s1 w3-container w3-right">
-					<button class="w3-button w3-theme-d1 w3-ripple" @click=${this.copy_id}>Copy</button>
-				</div>
-				<div class="w3-rest w3-container">
-					<input type="text" class="w3-theme-d1" style="width: 100%; vertical-align: middle" readonly value=${this.id}></input>
-				</div>
-			</div>
+		return html`<div class="w3-container" style="box-sizing: border-box; border: 2px solid black; background-color: rgba(255, 255, 255, 0.2)">
+			<p><tf-user id=${this.id} .users=${this.users}></tf-user> (${tfutils.human_readable_size(this.size)})
+			<input type="text" class="w3-input w3-border w3-theme-d1" readonly value=${this.id}></input>
+			<button class="w3-button w3-theme-d1 w3-ripple" @click=${this.copy_id}>Copy</button>
 			<div style="display: flex; flex-direction: row; gap: 1em">
 				${edit_profile}
 				<div style="flex: 1 0 50%">
@@ -312,11 +314,11 @@ class TfProfileElement extends LitElement {
 				Blocking ${profile.blocking} identities.
 				Blocked by ${profile.blocked} identities.
 			</div>
-			<div>
+			<p>
 				${edit}
 				${follow}
 				${block}
-			</div>
+			</p>
 		</div>`;
 	}
 }

apps/ssb/tf-reactions-modal.js

@@ -26,7 +26,7 @@ class TfReactionsModalElement extends LitElement {
 		return this.votes?.length
 			? html` <div
 					class="w3-modal w3-animate-opacity"
-					style="display: block; box-sizing: border-box"
+					style="display: block; box-sizing: border-box; z-index: 10"
 					@click=${this.clear}
 				>
 					<div

apps/ssb/tf-styles.js

@@ -286,29 +286,29 @@ hr{border:0;border-top:1px solid #eee;margin:20px 0}
 `;
 
 // prettier-ignore
-const w3_2016_riverside = css`
-.w3-theme-l5 {color:#000 !important; background-color:#f4f6f9 !important}
-.w3-theme-l4 {color:#000 !important; background-color:#d9e1ec !important}
-.w3-theme-l3 {color:#000 !important; background-color:#b4c3d8 !important}
-.w3-theme-l2 {color:#fff !important; background-color:#8ea6c5 !important}
-.w3-theme-l1 {color:#fff !important; background-color:#6888b1 !important}
-.w3-theme-d1 {color:#fff !important; background-color:#456185 !important}
-.w3-theme-d2 {color:#fff !important; background-color:#3d5676 !important}
-.w3-theme-d3 {color:#fff !important; background-color:#354b68 !important}
-.w3-theme-d4 {color:#fff !important; background-color:#2e4059 !important}
-.w3-theme-d5 {color:#fff !important; background-color:#26364a !important}
+const w3_2016_snorkel_blue = css`
+.w3-theme-l5 {color:#000 !important; background-color:#e9f5ff !important}
+.w3-theme-l4 {color:#000 !important; background-color:#b5dffd !important}
+.w3-theme-l3 {color:#000 !important; background-color:#6bc0fc !important}
+.w3-theme-l2 {color:#fff !important; background-color:#21a0fa !important}
+.w3-theme-l1 {color:#fff !important; background-color:#0479cc !important}
+.w3-theme-d1 {color:#fff !important; background-color:#024575 !important}
+.w3-theme-d2 {color:#fff !important; background-color:#023e68 !important}
+.w3-theme-d3 {color:#fff !important; background-color:#02365b !important}
+.w3-theme-d4 {color:#fff !important; background-color:#022e4e !important}
+.w3-theme-d5 {color:#fff !important; background-color:#012641 !important}
 
-.w3-theme-light {color:#000 !important; background-color:#f4f6f9 !important}
-.w3-theme-dark {color:#fff !important; background-color:#26364a !important}
-.w3-theme-action {color:#fff !important; background-color:#26364a !important}
+.w3-theme-light {color:#000 !important; background-color:#e9f5ff !important}
+.w3-theme-dark {color:#fff !important; background-color:#012641 !important}
+.w3-theme-action {color:#fff !important; background-color:#012641 !important}
 
-.w3-theme {color:#fff !important; background-color:#4c6a92 !important}
-.w3-text-theme {color:#4c6a92 !important}
-.w3-border-theme {border-color:#4c6a92 !important}
+.w3-theme {color:#fff !important; background-color:#034f84 !important}
+.w3-text-theme {color:#034f84 !important}
+.w3-border-theme {border-color:#034f84 !important}
 
-.w3-hover-theme:hover {color:#fff !important; background-color:#4c6a92 !important}
-.w3-hover-text-theme:hover {color:#4c6a92 !important}
-.w3-hover-border-theme:hover {border-color:#4c6a92 !important}
+.w3-hover-theme:hover {color:#fff !important; background-color:#034f84 !important}
+.w3-hover-text-theme:hover {color:#034f84 !important}
+.w3-hover-border-theme:hover {border-color:#034f84 !important}
 `;
 
-export let styles = [tf, w3, w3_2016_riverside];
+export let styles = [tf, w3, w3_2016_snorkel_blue];

apps/ssb/tf-tab-connections.js

@@ -12,6 +12,9 @@ class TfTabConnectionsElement extends LitElement {
 			stored_connections: {type: Array},
 			users: {type: Object},
 			server_identity: {type: String},
+			connect_attempt: {type: Object},
+			connect_message: {type: String},
+			connect_success: {type: Boolean},
 		};
 	}
 
@@ -88,20 +91,36 @@ class TfTabConnectionsElement extends LitElement {
 		`;
 	}
 
+	render_message(connection) {
+		return html`<div
+			?hidden=${this.connect_message === undefined ||
+			this.connect_attempt != connection}
+			style="cursor: pointer"
+			class=${'w3-panel ' + (this.connect_success ? 'w3-green' : 'w3-red')}
+			@click=${() => (this.connect_attempt = undefined)}
+		>
+			<p>${this.connect_message}</p>
+		</div>`;
+	}
+
 	render_broadcast(connection) {
+		let self = this;
 		return html`
-			<li class="w3-bar" style="overflow: hidden; overflow-wrap: nowrap">
-				<button
-					class="w3-bar-item w3-button w3-theme-d1"
-					@click=${() => tfrpc.rpc.connect(connection)}
-				>
-					Connect
-				</button>
-				<div class="w3-bar-item">
-					${TfTabConnectionsElement.k_broadcast_emojis[connection.origin]}
-					<tf-user id=${connection.pubkey} .users=${this.users}></tf-user>
-					${this.render_connection_summary(connection)}
+			<li>
+				<div class="w3-bar" style="overflow: hidden; overflow-wrap: nowrap">
+					<button
+						class="w3-bar-item w3-button w3-theme-d1"
+						@click=${() => self.connect(connection)}
+					>
+						Connect
+					</button>
+					<div class="w3-bar-item">
+						${TfTabConnectionsElement.k_broadcast_emojis[connection.origin]}
+						<tf-user id=${connection.pubkey} .users=${this.users}></tf-user>
+						${this.render_connection_summary(connection)}
+					</div>
 				</div>
+				${this.render_message(connection)}
 			</li>
 		`;
 	}
@@ -159,26 +178,38 @@ class TfTabConnectionsElement extends LitElement {
 		`;
 	}
 
-	refresh() {
-		tfrpc.rpc.sync();
+	connect(address) {
+		let self = this;
+		self.connect_attempt = address;
+		self.connect_message = undefined;
+		self.connect_success = false;
+		tfrpc.rpc
+			.connect(address)
+			.then(function () {
+				if (self.connect_attempt == address) {
+					self.connect_message = 'Connected.';
+					self.connect_success = true;
+				}
+			})
+			.catch(function (error) {
+				if (self.connect_attempt == address) {
+					self.connect_message = 'Error: ' + error;
+					self.connect_success = false;
+				}
+			});
 	}
 
 	render() {
 		let self = this;
 		return html`
 			<div class="w3-container" style="box-sizing: border-box">
-				<button
-					class="w3-button w3-theme-l3 w3-circle w3-ripple w3-large"
-					@click=${this.refresh}
-				>
-					🔃
-				</button>
 				<h2>New Connection</h2>
 				<textarea class="w3-input w3-theme-d1" id="code"></textarea>
+				${this.render_message(this.renderRoot.getElementById('code')?.value)}
 				<button
 					class="w3-button w3-theme-d1"
 					@click=${() =>
-						tfrpc.rpc.connect(self.renderRoot.getElementById('code').value)}
+						self.connect(self.renderRoot.getElementById('code')?.value)}
 				>
 					Connect
 				</button>
@@ -186,6 +217,9 @@ class TfTabConnectionsElement extends LitElement {
 				<ul class="w3-ul w3-border">
 					${this.broadcasts
 						.filter((x) => x.address)
+						.filter(
+							(x) => self.connections.map((c) => c.id).indexOf(x.pubkey) == -1
+						)
 						.map((x) => self.render_broadcast(x))}
 				</ul>
 				<h2>Connections</h2>
@@ -202,23 +236,26 @@ class TfTabConnectionsElement extends LitElement {
 				<ul class="w3-ul w3-border">
 					${this.stored_connections.map(
 						(x) => html`
-							<li class="w3-bar">
-								<button
-									class="w3-bar-item w3-button w3-theme-d1"
-									@click=${() => self.forget_stored_connection(x)}
-								>
-									Forget
-								</button>
-								<button
-									class="w3-bar-item w3-button w3-theme-d1"
-									@click=${() => tfrpc.rpc.connect(x)}
-								>
-									Connect
-								</button>
-								<div class="w3-bar-item">
-									<tf-user id=${x.pubkey} .users=${self.users}></tf-user>
-									<div><small>${x.address}:${x.port}</small></div>
+							<li>
+								<div class="w3-bar">
+									<button
+										class="w3-bar-item w3-button w3-theme-d1"
+										@click=${() => self.forget_stored_connection(x)}
+									>
+										Forget
+									</button>
+									<button
+										class="w3-bar-item w3-button w3-theme-d1"
+										@click=${() => this.connect(x)}
+									>
+										Connect
+									</button>
+									<div class="w3-bar-item">
+										<tf-user id=${x.pubkey} .users=${self.users}></tf-user>
+										<div><small>${x.address}:${x.port}</small></div>
+									</div>
 								</div>
+								${this.render_message(x)}
 							</li>
 						`
 					)}

apps/ssb/tf-tab-mentions.js

@@ -1,78 +0,0 @@
-import {LitElement, html, unsafeHTML} from './lit-all.min.js';
-import * as tfrpc from '/static/tfrpc.js';
-import {styles} from './tf-styles.js';
-
-class TfTabMentionsElement extends LitElement {
-	static get properties() {
-		return {
-			whoami: {type: String},
-			users: {type: Object},
-			following: {type: Array},
-			expanded: {type: Object},
-			messages: {type: Array},
-		};
-	}
-
-	static styles = styles;
-
-	constructor() {
-		super();
-		let self = this;
-		this.whoami = null;
-		this.users = {};
-		this.following = [];
-		this.expanded = {};
-		this.messages = [];
-	}
-
-	async load() {
-		console.log('Loading...', this.whoami);
-		let results = await tfrpc.rpc.query(
-			`
-				SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
-				FROM messages_fts(?)
-				JOIN messages ON messages.rowid = messages_fts.rowid
-				JOIN json_each(?) AS following ON messages.author = following.value
-				WHERE messages.author != ?
-				ORDER BY timestamp DESC limit 20
-			`,
-			[
-				'"' + this.whoami.replace('"', '""') + '"',
-				JSON.stringify(this.following),
-				this.whoami,
-			]
-		);
-		console.log('Done.');
-		this.messages = results;
-	}
-
-	on_expand(event) {
-		if (event.detail.expanded) {
-			let expand = {};
-			expand[event.detail.id] = true;
-			this.expanded = Object.assign({}, this.expanded, expand);
-		} else {
-			delete this.expanded[event.detail.id];
-			this.expanded = Object.assign({}, this.expanded);
-		}
-	}
-
-	render() {
-		let self = this;
-		if (!this.loading) {
-			this.loading = true;
-			this.load();
-		}
-		return html`
-			<tf-news
-				id="news"
-				whoami=${this.whoami}
-				.messages=${this.messages}
-				.users=${this.users}
-				.expanded=${this.expanded}
-				@tf-expand=${this.on_expand}
-			></tf-news>
-		`;
-	}
-}
-customElements.define('tf-tab-mentions', TfTabMentionsElement);

apps/ssb/tf-tab-news-feed.js

@@ -12,6 +12,11 @@ class TfTabNewsFeedElement extends LitElement {
 			messages: {type: Array},
 			drafts: {type: Object},
 			expanded: {type: Object},
+			channels_unread: {type: Object},
+			channels_latest: {type: Object},
+			loading: {type: Number},
+			time_range: {type: Array},
+			time_loading: {type: Array},
 		};
 	}
 
@@ -26,30 +31,67 @@ class TfTabNewsFeedElement extends LitElement {
 		this.following = [];
 		this.drafts = {};
 		this.expanded = {};
-		this.start_time = new Date().valueOf() - 24 * 60 * 60 * 1000;
+		this.channels_unread = {};
+		this.channels_latest = {};
+		this.start_time = new Date().valueOf();
+		this.time_range = [0, 0];
+		this.time_loading = undefined;
+		this.loading = 0;
 	}
 
-	async fetch_messages() {
-		if (this.hash.startsWith('#@')) {
-			let r = await tfrpc.rpc.query(
+	channel() {
+		return this.hash.startsWith('##')
+			? this.hash.substring(2)
+			: this.hash.substring(1);
+	}
+
+	async fetch_messages(start_time, end_time) {
+		this.time_loading = [start_time, end_time];
+		let result;
+		if (this.hash == '#@') {
+			result = await tfrpc.rpc.query(
 				`
-					WITH mine AS (SELECT id, previous, author, sequence, timestamp, hash, json(content) AS content, signature
+					SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
+					FROM messages_fts(?1)
+					JOIN messages ON messages.rowid = messages_fts.rowid
+					JOIN json_each(?2) AS following ON messages.author = following.value
+					WHERE
+						messages.author != ?1 AND
+						messages.timestamp >= ?3 AND
+						messages.timestamp < ?4
+					ORDER BY timestamp DESC limit 20
+				`,
+				[
+					'"' + this.whoami.replace('"', '""') + '"',
+					JSON.stringify(this.following),
+					start_time,
+					end_time,
+				]
+			);
+		} else if (this.hash.startsWith('#@')) {
+			result = await tfrpc.rpc.query(
+				`
+					WITH mine AS (SELECT rowid, id, previous, author, sequence, timestamp, hash, json(content) AS content, signature
 						FROM messages
 						WHERE messages.author = ?
-						ORDER BY sequence DESC
-						LIMIT 20)
-					SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
+						ORDER BY sequence DESC)
+					SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
 						FROM mine
 						JOIN messages_refs ON mine.id = messages_refs.ref
 						JOIN messages ON messages_refs.message = messages.id
+						WHERE
+							mine.timestamp >= ?2 AND
+							mine.timestamp < ?3
 					UNION
 					SELECT * FROM mine
+					WHERE
+						mine.timestamp >= ?2 AND
+						mine.timestamp < ?3
 				`,
-				[this.hash.substring(1)]
+				[this.hash.substring(1), start_time, end_time]
 			);
-			return r;
 		} else if (this.hash.startsWith('#%')) {
-			return await tfrpc.rpc.query(
+			result = await tfrpc.rpc.query(
 				`
 					SELECT id, previous, author, sequence, timestamp, hash, json(content) AS content, signature
 					FROM messages
@@ -62,6 +104,68 @@ class TfTabNewsFeedElement extends LitElement {
 				`,
 				[this.hash.substring(1)]
 			);
+		} else if (this.hash.startsWith('##')) {
+			let promises = [];
+			const k_following_limit = 256;
+			for (let i = 0; i < this.following.length; i += k_following_limit) {
+				promises.push(
+					tfrpc.rpc.query(
+						`
+						WITH news AS (SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
+							FROM messages
+							JOIN json_each(?) AS following ON messages.author = following.value
+							WHERE
+								messages.timestamp >= ? AND
+								messages.timestamp < ? AND
+								messages.content ->> 'channel' = ?
+							ORDER BY messages.timestamp DESC)
+						SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
+							FROM news
+							JOIN messages_refs ON news.id = messages_refs.ref
+							JOIN messages ON messages_refs.message = messages.id
+						UNION
+						SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
+							FROM news
+							JOIN messages_refs ON news.id = messages_refs.message
+							JOIN messages ON messages_refs.ref = messages.id
+						UNION
+						SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
+							FROM messages_fts(?5)
+							JOIN messages ON messages.rowid = messages_fts.rowid
+							JOIN json_each(?1) AS following ON messages.author = following.value
+							JOIN json_tree(messages.content, '$.mentions') AS mention ON mention.value = '#' || ?4
+							WHERE
+								messages.timestamp >= ?2 AND
+								messages.timestamp < ?3
+						UNION
+						SELECT news.* FROM news
+					`,
+						[
+							JSON.stringify(this.following.slice(i, i + k_following_limit)),
+							start_time,
+							end_time,
+							this.hash.substring(2),
+							'"#' + this.hash.substring(2).replace('"', '""') + '"',
+						]
+					)
+				);
+			}
+			result = [].concat(...(await Promise.all(promises)));
+		} else if (this.hash == '#🔐') {
+			result = await tfrpc.rpc.query(
+				`
+					SELECT messages.rowid, messages.id, previous, author, sequence, timestamp, hash, json(content) AS content, signature
+						FROM messages
+						JOIN json_each(?1) AS following ON messages.author = following.value
+						WHERE
+							messages.timestamp >= ?2 AND
+							messages.timestamp < ?3 AND
+							json(messages.content) LIKE '"%'
+						ORDER BY sequence DESC
+				`,
+				[JSON.stringify(this.following), start_time, end_time]
+			);
+			result = (await this.decrypt(result)).filter((x) => x.decrypted);
 		} else {
 			let promises = [];
 			const k_following_limit = 256;
@@ -69,17 +173,17 @@ class TfTabNewsFeedElement extends LitElement {
 				promises.push(
 					tfrpc.rpc.query(
 						`
-						WITH news AS (SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
+						WITH news AS (SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
 						FROM messages
 						JOIN json_each(?) AS following ON messages.author = following.value
-						WHERE messages.timestamp > ? AND messages.timestamp < ?
+						WHERE messages.timestamp >= ? AND messages.timestamp < ?
 						ORDER BY messages.timestamp DESC)
-						SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
+						SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
 							FROM news
 							JOIN messages_refs ON news.id = messages_refs.ref
 							JOIN messages ON messages_refs.message = messages.id
 						UNION
-						SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
+						SELECT messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
 							FROM news
 							JOIN messages_refs ON news.id = messages_refs.message
 							JOIN messages ON messages_refs.ref = messages.id
@@ -88,50 +192,58 @@ class TfTabNewsFeedElement extends LitElement {
 					`,
 						[
 							JSON.stringify(this.following.slice(i, i + k_following_limit)),
-							this.start_time,
-							/*
-							 ** Don't show messages more than a day into the future to prevent
-							 ** messages with far-future timestamps from staying at the top forever.
-							 */
-							new Date().valueOf() + 24 * 60 * 60 * 1000,
+							start_time,
+							end_time,
 						]
 					)
 				);
 			}
-			return [].concat(...(await Promise.all(promises)));
+			result = [].concat(...(await Promise.all(promises)));
 		}
+		this.time_loading = undefined;
+		return result;
+	}
+
+	update_time_range_from_messages(messages) {
+		this.time_range = [
+			messages.reduce(
+				(accumulator, current) => Math.min(accumulator, current.timestamp),
+				this.time_range[0]
+			),
+			messages.reduce(
+				(accumulator, current) => Math.max(accumulator, current.timestamp),
+				this.time_range[1]
+			),
+		];
 	}
 
 	async load_more() {
-		let last_start_time = this.start_time;
-		this.start_time = last_start_time - 24 * 60 * 60 * 1000;
-		let more = await tfrpc.rpc.query(
-			`
-				WITH news AS (SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
-				FROM messages
-				JOIN json_each(?) AS following ON messages.author = following.value
-				WHERE messages.timestamp > ?
-				AND messages.timestamp <= ?
-				ORDER BY messages.timestamp DESC)
-				SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
-					FROM news
-					JOIN messages_refs ON news.id = messages_refs.ref
-					JOIN messages ON messages_refs.message = messages.id
-				UNION
-				SELECT messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
-					FROM news
-					JOIN messages_refs ON news.id = messages_refs.message
-					JOIN messages ON messages_refs.ref = messages.id
-				UNION
-				SELECT news.* FROM news
-			`,
-			[JSON.stringify(this.following), this.start_time, last_start_time]
-		);
-		this.messages = await this.decrypt([...more, ...this.messages]);
+		this.loading++;
+		this.loading_canceled = false;
+		try {
+			let more = [];
+			while (!more.length && !this.loading_canceled) {
+				let last_start_time = this.start_time;
+				this.start_time = last_start_time - 7 * 24 * 60 * 60 * 1000;
+				more = await this.fetch_messages(this.start_time, last_start_time);
+				this.update_time_range_from_messages(
+					more.filter(
+						(x) =>
+							x.timestamp >= this.start_time && x.timestamp < last_start_time
+					)
+				);
+			}
+			this.messages = await this.decrypt([...more, ...this.messages]);
+		} finally {
+			this.loading--;
+		}
+	}
+
+	cancel_load() {
+		this.loading_canceled = true;
 	}
 
 	async decrypt(messages) {
-		console.log('decrypt');
 		let result = [];
 		for (let message of messages) {
 			let content;
@@ -156,8 +268,94 @@ class TfTabNewsFeedElement extends LitElement {
 		return result;
 	}
 
-	async add_messages(messages) {
-		this.messages = await this.decrypt([...messages, ...this.messages]);
+	async load_latest() {
+		this.loading++;
+		let now = new Date().valueOf();
+		let end_time = now + 24 * 60 * 60 * 1000;
+		let messages = [];
+		try {
+			messages = await this.fetch_messages(
+				this.time_range[1] - 24 * 60 * 60 * 1000,
+				end_time
+			);
+			messages = await this.decrypt(messages);
+			this.update_time_range_from_messages(
+				messages.filter(
+					(x) => x.timestamp >= this.time_range[1] && x.timestamp < end_time
+				)
+			);
+		} finally {
+			this.loading--;
+		}
+		this.messages = Object.values(
+			Object.fromEntries([...this.messages, ...messages].map((x) => [x.id, x]))
+		);
+		console.log('done loading latest messages.');
+	}
+
+	async load_messages() {
+		let self = this;
+		this.loading++;
+		let messages = [];
+		try {
+			this.messages = [];
+			this._messages_hash = this.hash;
+			this._messages_following = this.following;
+			let now = new Date().valueOf();
+			let start_time = now - 24 * 60 * 60 * 1000;
+			this.start_time = start_time;
+			this.time_range = [this.start_time, now + 24 * 60 * 60 * 1000];
+			messages = await this.fetch_messages(
+				this.time_range[0],
+				this.time_range[1]
+			);
+			this.update_time_range_from_messages(
+				messages.filter(
+					(x) =>
+						x.timestamp >= this.time_range[0] &&
+						x.timestamp < this.time_range[1]
+				)
+			);
+			messages = await this.decrypt(messages);
+			if (!messages.length) {
+				let more = [];
+				while (!more.length && start_time >= 0) {
+					let last_start_time = start_time;
+					start_time = last_start_time - 7 * 24 * 60 * 60 * 1000;
+					more = await this.fetch_messages(start_time, last_start_time);
+					this.update_time_range_from_messages(
+						more.filter(
+							(x) => x.timestamp >= start_time && x.timestamp < last_start_time
+						)
+					);
+				}
+				messages = await this.decrypt([...more, ...this.messages]);
+			}
+		} finally {
+			this.loading--;
+		}
+		this.messages = messages;
+		this.time_loading = undefined;
+		console.log(`loading messages done for ${self.whoami}`);
+	}
+
+	mark_all_read() {
+		let newest = this.messages.reduce(
+			(accumulator, current) => Math.max(accumulator, current.rowid),
+			this.channels_latest[this.channel()] ?? -1
+		);
+		if (newest >= 0) {
+			this.dispatchEvent(
+				new CustomEvent('channelsetunread', {
+					bubbles: true,
+					composed: true,
+					detail: {
+						channel: this.channel(),
+						unread: newest + 1,
+					},
+				})
+			);
+		}
 	}
 
 	render() {
@@ -169,31 +367,49 @@ class TfTabNewsFeedElement extends LitElement {
 			console.log(
 				`loading messages for ${this.whoami} (following ${this.following.length})`
 			);
-			let self = this;
-			this.messages = [];
-			this._messages_hash = this.hash;
-			this._messages_following = this.following;
-			this.fetch_messages()
-				.then(this.decrypt.bind(this))
-				.then(function (messages) {
-					self.messages = messages;
-					console.log(`loading mesages done for ${self.whoami}`);
-				})
-				.catch(function (error) {
-					alert(JSON.stringify(error, null, 2));
-				});
+			this.load_messages();
 		}
 		let more;
-		if (!this.hash.startsWith('#@') && !this.hash.startsWith('#%')) {
+		if (!this.hash.startsWith('#%')) {
 			more = html`
 				<p>
-					<button class="w3-button w3-theme-d1" @click=${this.load_more}>
+					<button class="w3-button w3-theme-d1" @click=${this.mark_all_read}>
+						Mark All Read
+					</button>
+					<button
+						?disabled=${this.loading}
+						class="w3-button w3-theme-d1"
+						@click=${this.load_more}
+					>
 						Load More
 					</button>
+					<button
+						class=${'w3-button w3-theme-d1' + (this.loading ? '' : ' w3-hide')}
+						@click=${this.cancel_load}
+					>
+						Cancel
+					</button>
+					<span
+						>Showing
+						${new Date(
+							this.time_loading
+								? Math.min(this.time_loading[0], this.time_range[0])
+								: this.time_range[0]
+						).toLocaleDateString()}
+						-
+						${new Date(
+							this.time_loading
+								? Math.max(this.time_loading[1], this.time_range[1])
+								: this.time_range[1]
+						).toLocaleDateString()}.</span
+					>
 				</p>
 			`;
 		}
 		return html`
+			<button class="w3-button w3-theme-d1" @click=${this.mark_all_read}>
+				Mark All Read
+			</button>
 			<tf-news
 				id="news"
 				whoami=${this.whoami}
@@ -202,6 +418,8 @@ class TfTabNewsFeedElement extends LitElement {
 				.following=${this.following}
 				.drafts=${this.drafts}
 				.expanded=${this.expanded}
+				channel=${this.channel()}
+				channel_unread=${this.channels_unread?.[this.channel()]}
 			></tf-news>
 			${more}
 		`;

apps/ssb/tf-tab-news.js

@@ -13,6 +13,9 @@ class TfTabNewsElement extends LitElement {
 			drafts: {type: Object},
 			expanded: {type: Object},
 			loading: {type: Boolean},
+			channels: {type: Array},
+			channels_unread: {type: Object},
+			channels_latest: {type: Object},
 		};
 	}
 
@@ -29,6 +32,9 @@ class TfTabNewsElement extends LitElement {
 		this.cache = {};
 		this.drafts = {};
 		this.expanded = {};
+		this.channels_unread = {};
+		this.channels_latest = {};
+		this.channels = [];
 		tfrpc.rpc.localStorageGet('drafts').then(function (d) {
 			self.drafts = JSON.parse(d || '{}');
 		});
@@ -48,10 +54,7 @@ class TfTabNewsElement extends LitElement {
 		let unread = this.unread;
 		let news = this.shadowRoot?.getElementById('news');
 		if (news) {
-			console.log('injecting messages', news.messages);
-			news.add_messages(
-				Object.values(Object.fromEntries(this.unread.map((x) => [x.id, x])))
-			);
+			news.load_latest();
 			this.dispatchEvent(new CustomEvent('refresh'));
 		}
 	}
@@ -106,14 +109,57 @@ class TfTabNewsElement extends LitElement {
 		}
 	}
 
+	unread_status(channel) {
+		if (
+			this.channels_latest[channel] &&
+			(this.channels_unread[channel] === undefined ||
+				this.channels_unread[channel] <= this.channels_latest[channel])
+		) {
+			return '🔵';
+		}
+	}
+
+	show_sidebar() {
+		this.renderRoot.getElementById('sidebar').style.display = 'block';
+		this.renderRoot.getElementById('sidebar_overlay').style.display = 'block';
+	}
+
+	hide_sidebar() {
+		this.renderRoot.getElementById('sidebar').style.display = 'none';
+		this.renderRoot.getElementById('sidebar_overlay').style.display = 'none';
+	}
+
+	async channel_toggle_subscribed() {
+		let channel = this.hash.substring(2);
+		let subscribed = this.channels.indexOf(channel) != -1;
+		subscribed = !subscribed;
+
+		await tfrpc.rpc.appendMessage(this.whoami, {
+			type: 'channel',
+			channel: channel,
+			subscribed: subscribed,
+		});
+		if (subscribed) {
+			this.channels = [].concat([channel], this.channels).sort();
+		} else {
+			this.channels = this.channels.filter((x) => x != channel);
+		}
+	}
+
+	channel() {
+		return this.hash.startsWith('##') ? this.hash.substring(2) : undefined;
+	}
+
 	render() {
-		let profile = this.hash.startsWith('#@')
-			? html`<tf-profile
-					id=${this.hash.substring(1)}
-					whoami=${this.whoami}
-					.users=${this.users}
-				></tf-profile>`
-			: undefined;
+		let profile =
+			this.hash.startsWith('#@') && this.hash != '#@'
+				? html`<tf-profile
+						class="tf-profile"
+						id=${this.hash.substring(1)}
+						whoami=${this.whoami}
+						.users=${this.users}
+					></tf-profile>`
+				: undefined;
 		let edit_profile;
 		if (
 			!this.loading &&
@@ -128,39 +174,118 @@ class TfTabNewsElement extends LitElement {
 			</div>`;
 		}
 		return html`
-			<p class="w3-bar">
-				<button
-					class="w3-bar-item w3-button w3-theme-d1"
-					@click=${this.show_more}
+			<div
+				class="w3-sidebar w3-bar-block w3-theme-d1 w3-collapse w3-animate-left"
+				style="width: 2in; left: 0; z-index: 5"
+				id="sidebar"
+			>
+				<div
+					class="w3-right w3-button w3-hide-large"
+					@click=${this.hide_sidebar}
 				>
-					${this.new_messages_text()}
-				</button>
-			</p>
-			<div class="w3-bar">
-				Welcome, <tf-user id=${this.whoami} .users=${this.users}></tf-user>!
-				${edit_profile}
+					&times;
+				</div>
+				${this.hash.startsWith('##') &&
+				this.channels.indexOf(this.hash.substring(2)) == -1
+					? html`
+							<div class="w3-bar-item w3-theme-d2">Viewing</div>
+							<a
+								href="#"
+								class="w3-bar-item w3-button"
+								style="font-weight: bold"
+								>${this.hash.substring(2)}</a
+							>
+						`
+					: undefined}
+				<div class="w3-bar-item w3-theme-d2">Channels</div>
+				<a
+					href="#"
+					class="w3-bar-item w3-button"
+					style=${this.hash == '#' ? 'font-weight: bold' : undefined}
+					>general ${this.unread_status('')}</a
+				>
+				<a
+					href="#@"
+					class="w3-bar-item w3-button"
+					style=${this.hash == '#@' ? 'font-weight: bold' : undefined}
+					>@mentions ${this.unread_status('@')}</a
+				>
+				<a
+					href="#🔐"
+					class="w3-bar-item w3-button"
+					style=${this.hash == '#🔐' ? 'font-weight: bold' : undefined}
+					>🔐private ${this.unread_status('🔐')}</a
+				>
+				${this.channels.map(
+					(x) => html`
+						<a
+							href=${'#' + encodeURIComponent('#' + x)}
+							class="w3-bar-item w3-button"
+							style=${this.hash == '##' + x ? 'font-weight: bold' : undefined}
+							>#${x} ${this.unread_status(x)}</a
+						>
+					`
+				)}
 			</div>
-			<div>
-				<tf-compose
-					id="tf-compose"
+			<div
+				class="w3-overlay"
+				id="sidebar_overlay"
+				@click=${this.hide_sidebar}
+			></div>
+			<div style="margin-left: 2in; padding: 8px" id="main" class="w3-main">
+				<div
+					id="show_sidebar"
+					class="w3-left w3-button w3-hide-large"
+					@click=${this.show_sidebar}
+				>
+					&#9776;
+				</div>
+				<p>
+					<button class="w3-button w3-theme-d1" @click=${this.show_more}>
+						${this.new_messages_text()}
+					</button>
+					${this.hash.startsWith('##')
+						? html`
+								<button
+									class="w3-button w3-theme-d1"
+									@click=${this.channel_toggle_subscribed}
+								>
+									${this.channels.indexOf(this.hash.substring(2)) != -1
+										? 'Unsubscribe from #'
+										: 'Subscribe to #'}${this.hash.substring(2)}
+								</button>
+							`
+						: undefined}
+				</p>
+				<div class="w3-bar">
+					Welcome, <tf-user id=${this.whoami} .users=${this.users}></tf-user>!
+					${edit_profile}
+				</div>
+				<div>
+					<tf-compose
+						id="tf-compose"
+						whoami=${this.whoami}
+						.users=${this.users}
+						.drafts=${this.drafts}
+						@tf-draft=${this.draft}
+						.channel=${this.channel()}
+					></tf-compose>
+				</div>
+				${profile}
+				<tf-tab-news-feed
+					id="news"
 					whoami=${this.whoami}
 					.users=${this.users}
+					.following=${this.following}
+					hash=${this.hash}
 					.drafts=${this.drafts}
+					.expanded=${this.expanded}
 					@tf-draft=${this.draft}
-				></tf-compose>
+					@tf-expand=${this.on_expand}
+					.channels_unread=${this.channels_unread}
+					.channels_latest=${this.channels_latest}
+				></tf-tab-news-feed>
 			</div>
-			${profile}
-			<tf-tab-news-feed
-				id="news"
-				whoami=${this.whoami}
-				.users=${this.users}
-				.following=${this.following}
-				hash=${this.hash}
-				.drafts=${this.drafts}
-				.expanded=${this.expanded}
-				@tf-draft=${this.draft}
-				@tf-expand=${this.on_expand}
-			></tf-tab-news-feed>
 		`;
 	}
 }

apps/ssb/tf-tag.js

@@ -18,7 +18,7 @@ class TfTagElement extends LitElement {
 	render() {
 		let number = this.count ? html` (${this.count})` : undefined;
 		return html`<a
-			href="#q=${this.tag}"
+			href=${'#' + encodeURIComponent(this.tag)}
 			style="display: inline-block; margin: 3px; border: 1px solid black; background-color: #444; padding: 4px; border-radius: 3px"
 			>${this.tag}${number}</a
 		>`;

apps/ssb/tf-utils.js

@@ -2,6 +2,12 @@ import * as hashtagify from './commonmark-hashtag.js';
 
 const k_code_classes = 'w3-theme-l4 w3-theme-border w3-round';
 
+var reUnsafeProtocol = /^javascript:|vbscript:|file:|data:/i;
+var reSafeDataProtocol = /^data:image\/(?:png|gif|jpeg|webp)/i;
+var potentiallyUnsafe = function (url) {
+	return reUnsafeProtocol.test(url) && !reSafeDataProtocol.test(url);
+};
+
 function image(node, entering) {
 	if (
 		node.firstChild?.type === 'text' &&
@@ -81,8 +87,8 @@ function attrs(node) {
 }
 
 export function markdown(md) {
-	let reader = new commonmark.Parser({safe: true});
-	let writer = new commonmark.HtmlRenderer();
+	let reader = new commonmark.Parser();
+	let writer = new commonmark.HtmlRenderer({safe: true});
 	writer.image = image;
 	writer.code = code;
 	writer.attrs = attrs;

apps/storage.json

@@ -0,0 +1,5 @@
+{
+	"type": "tildefriends-app",
+	"emoji": "💾",
+	"previous": "&mvGTlWKFR5QM/3nb4fJ2WQq0n/gNKvBmhGDkAvb8ki8=.sha256"
+}

apps/storage/app.js

@@ -0,0 +1,127 @@
+async function query(sql, args) {
+	let rows = [];
+	await ssb.sqlAsync(sql, args ?? [], function (row) {
+		rows.push(row);
+	});
+	return rows;
+}
+
+async function get_biggest() {
+	return query(`
+		select author, sum(length(content)) as size from messages group by author order by size desc limit 10;
+	`);
+}
+
+async function get_total() {
+	return (
+		await query(`
+		select sum(length(content)) as size, count(distinct author) as count from messages;
+	`)
+	)[0];
+}
+
+async function get_names(identities) {
+	return query(
+		`
+		SELECT author, name FROM (
+			SELECT
+				messages.author,
+				RANK() OVER (PARTITION BY messages.author ORDER BY messages.sequence DESC) AS author_rank,
+				messages.content ->> 'name' AS name
+			FROM messages
+			JOIN json_each(?) AS identities ON identities.value = messages.author
+			WHERE
+				json_extract(messages.content, '$.type') = 'about' AND
+				content ->> 'about' = messages.author AND name IS NOT NULL)
+		WHERE author_rank = 1
+	`,
+		[JSON.stringify(identities)]
+	);
+}
+
+async function get_most_follows() {
+	return query(`
+		select author, count(*) as count
+		from messages
+		where content ->> 'type' = 'contact' and content ->> 'following' = true
+		group by author
+		order by count desc
+		limit 10;
+	`);
+}
+
+function nice_size(bytes) {
+	let value = bytes;
+	let index = 0;
+	let units = ['B', 'kB', 'MB', 'GB'];
+	while (value > 1024 && index < units.length - 1) {
+		value /= 1024;
+		index++;
+	}
+	return `${Math.round(value * 10) / 10} ${units[index]}`;
+}
+
+async function main() {
+	await app.setDocument(
+		'<p style="color: #fff">Finding the top 10 largest feeds...</p>'
+	);
+	let most_follows = await get_most_follows();
+	let total = await get_total();
+	let identities = await ssb.getAllIdentities();
+	let following1 = await ssb.following(identities, 1);
+	let following2 = await ssb.following(identities, 2);
+	let biggest = await get_biggest();
+	let names = await get_names(
+		[].concat(
+			biggest.map((x) => x.author),
+			most_follows.map((x) => x.author)
+		)
+	);
+	names = Object.fromEntries(names.map((x) => [x.author, x.name]));
+	for (let item of biggest) {
+		item.name = names[item.author];
+		item.following =
+			identities.indexOf(item.author) != -1
+				? 0
+				: following1[item.author] !== undefined
+					? 1
+					: following2[item.author] !== undefined
+						? 2
+						: undefined;
+	}
+	for (let item of most_follows) {
+		item.name = names[item.author];
+	}
+	let html = `<body style="color: #000; background-color: #ddd">\n
+		<h1>Storage Summary</h1>
+		<h2>Top 10 Accounts by Size</h2>
+		<ol>`;
+	for (let item of biggest) {
+		html += `<li>
+			<span style="color: #888">${nice_size(item.size)}</span>
+			<a target="_top" href="/~core/ssb/#${encodeURI(item.author)}">${item.name ?? item.author}</a>
+		</li>
+		\n`;
+	}
+	html += `
+		</ol>
+		<h2>Top 10 Accounts by Follows</h2>
+		<ol>`;
+	for (let item of most_follows) {
+		html += `<li>
+			<span style="color: #888">${item.count}</span>
+			${following2[item.author] ? '✅' : '🚫'}
+			<a target="_top" href="/~core/ssb/#${encodeURI(item.author)}">${item.name ?? item.author}</a>
+		</li>
+		\n`;
+	}
+	html += `
+		</ol>
+		<p>Total <span style="color: #888">${nice_size(total.size)}</span> in ${total.count} accounts.</p>
+	`;
+	await app.setDocument(html);
+}
+
+main().catch(function (e) {
+	print(e);
+});

apps/test.json

@@ -0,0 +1,4 @@
+{
+	"type": "tildefriends-app",
+	"emoji": "📦"
+}

apps/test/app.js

@@ -0,0 +1,3 @@
+app.setDocument(
+	'<p style="color: #fff">Maybe one day this app will run tests, but for now there is nothing to see here.</p>'
+);

apps/test/hello.txt

@@ -0,0 +1 @@
+Hello, world!

apps/wiki.json

@@ -1,5 +1,5 @@
 {
 	"type": "tildefriends-app",
 	"emoji": "📝",
-	"previous": "&DaYqKHRBKhjFGaOzbKZ1+/pLspJeEkDJYTF2B50tH6k=.sha256"
+	"previous": "&4UHlsfQJvSh7L3D86uFtr7KUKCMRVBBTFxRIMqIc5as=.sha256"
 }

apps/wiki/handler.js

@@ -2,8 +2,8 @@ import * as utils from './utils.js';
 import * as commonmark from './commonmark.min.js';
 
 function markdown(md) {
-	let reader = new commonmark.Parser({safe: true});
-	let writer = new commonmark.HtmlRenderer();
+	let reader = new commonmark.Parser();
+	let writer = new commonmark.HtmlRenderer({safe: true});
 	let parsed = reader.parse(md || '');
 	let walker = parsed.walker();
 	let event;

apps/wiki/tf-wiki-doc.js

@@ -20,8 +20,8 @@ class TfWikiDocElement extends LitElement {
 	}
 
 	markdown(md) {
-		let reader = new commonmark.Parser({safe: true});
-		let writer = new commonmark.HtmlRenderer();
+		let reader = new commonmark.Parser();
+		let writer = new commonmark.HtmlRenderer({safe: true});
 		let parsed = reader.parse(md || '');
 		let walker = parsed.walker();
 		let event;

core/client.js

@@ -56,7 +56,7 @@ class TfNavigationElement extends LitElement {
 			status: {type: Object},
 			spark_lines: {type: Object},
 			version: {type: Object},
-			show_version: {type: Boolean},
+			show_expanded: {type: Boolean},
 			identity: {type: String},
 			identities: {type: Array},
 			names: {type: Object},
@@ -105,7 +105,6 @@ class TfNavigationElement extends LitElement {
 			let spark_line = document.createElement('tf-sparkline');
 			spark_line.title = key;
 			spark_line.classList.add('w3-bar-item');
-			spark_line.classList.add('w3-hide-small');
 			spark_line.style.paddingRight = '0';
 			if (options) {
 				if (options.max) {
@@ -162,6 +161,10 @@ class TfNavigationElement extends LitElement {
 							class="w3-dropdown-content w3-bar-block w3-card-4"
 							style="max-width: 100%; right: 0"
 						>
+							<div
+								style="position: fixed; left: 0; right: 0; top: 0; bottom: 0; background-color: rgba(0, 0, 0, 0.25); z-index: -100"
+								@click=${self.toggle_id_dropdown}
+							></div>
 							<button
 								class="w3-bar-item w3-button w3-border"
 								@click=${() => (window.location.href = '/~core/identity')}
@@ -169,6 +172,7 @@ class TfNavigationElement extends LitElement {
 								Manage Identities...
 							</button>
 							<button
+								id="edit_profile"
 								class="w3-bar-item w3-button w3-border"
 								@click=${self.edit_profile}
 							>
@@ -311,13 +315,13 @@ class TfNavigationElement extends LitElement {
 				<span
 					class="w3-bar-item"
 					style="cursor: pointer"
-					@click=${() => (this.show_version = !this.show_version)}
+					@click=${() => (this.show_expanded = !this.show_expanded)}
 					>😎</span
 				>
 				<span
 					class="w3-bar-item"
 					style=${'white-space: nowrap' +
-					(this.show_version ? '' : '; display: none')}
+					(this.show_expanded ? '' : '; display: none')}
 					title=${this.version?.name +
 					' ' +
 					Object.entries(this.version || {})
@@ -372,9 +376,11 @@ class TfNavigationElement extends LitElement {
 							</div>
 						`
 					: undefined}
-				${Object.keys(this.spark_lines)
-					.sort()
-					.map((x) => this.spark_lines[x])}
+				<span class=${this.show_expanded ? '' : 'w3-hide-small'}>
+					${Object.keys(this.spark_lines)
+						.sort()
+						.map((x) => this.spark_lines[x])}
+				</span>
 				${this.render_identity()}
 			</div>
 			${this.status?.is_error
@@ -1174,7 +1180,7 @@ function api_requestPermission(permission, id) {
 
 	let div = document.createElement('div');
 	div.appendChild(
-		document.createTextNode('This app is requesting the following permission:')
+		document.createTextNode('This app is requesting the following permission: ')
 	);
 	let span = document.createElement('span');
 	span.style = 'font-weight: bold';
@@ -1190,6 +1196,7 @@ function api_requestPermission(permission, id) {
 	check.classList.add('w3-check');
 	check.classList.add('w3-blue');
 	div.appendChild(check);
+	div.appendChild(document.createTextNode(' '));
 	let label = document.createElement('label');
 	label.htmlFor = check.id;
 	label.appendChild(document.createTextNode('Remember this decision.'));

core/core.js

@@ -4,89 +4,6 @@ import * as http from './http.js';
 
 let gProcesses = {};
 let gStatsTimer = false;
-
-const k_content_security_policy =
-	'sandbox allow-downloads allow-top-navigation-by-user-activation';
-
-const k_global_settings = {
-	index: {
-		type: 'string',
-		default_value: '/~core/apps/',
-		description: 'Default path.',
-	},
-	index_map: {
-		type: 'textarea',
-		default_value: undefined,
-		description:
-			'Mappings from hostname to redirect path, one per line, as in: "www.tildefriends.net=/~core/index/"',
-	},
-	room: {
-		type: 'boolean',
-		default_value: true,
-		description: 'Enable peers to tunnel through this instance as a room.',
-	},
-	room_name: {
-		type: 'string',
-		default_value: 'tilde friends tunnel',
-		description: 'Name of the room.',
-	},
-	replicator: {
-		type: 'boolean',
-		default_value: true,
-		description: 'Enable message and blob replication.',
-	},
-	code_of_conduct: {
-		type: 'textarea',
-		default_value: undefined,
-		description: 'Code of conduct presented at sign-in.',
-	},
-	http_redirect: {
-		type: 'string',
-		default_value: undefined,
-		description:
-			'If connecting by HTTP and HTTPS is configured, Location header prefix (ie, "https://example.com")',
-	},
-	fetch_hosts: {
-		type: 'string',
-		default_value: undefined,
-		description:
-			'Comma-separated list of host names to which HTTP fetch requests are allowed.  None if empty.',
-	},
-	blob_fetch_age_seconds: {
-		type: 'integer',
-		default_value:
-			platform() == 'android' || platform() == 'iphone'
-				? 0.5 * 365 * 24 * 60 * 60
-				: undefined,
-		description:
-			'Only blobs mentioned more recently than this age will be automatically fetched.',
-	},
-	blob_expire_age_seconds: {
-		type: 'integer',
-		default_value:
-			platform() == 'android' || platform() == 'iphone'
-				? 1.0 * 365 * 24 * 60 * 60
-				: undefined,
-		description: 'Blobs older than this will be automatically deleted.',
-	},
-	seeds_host: {
-		type: 'string',
-		default_value: 'seeds.tildefriends.net',
-		description: 'Hostname for seed connections.',
-	},
-	peer_exchange: {
-		type: 'boolean',
-		default_value: false,
-		description:
-			'Enable discovery of, sharing of, and connecting to internet peer strangers, including announcing this instance.',
-	},
-	account_registration: {
-		type: 'boolean',
-		default_value: true,
-		description: 'Allow registration of new accounts.',
-	},
-};
-
 let kPingInterval = 60 * 1000;
 
 /**
@@ -489,7 +406,7 @@ async function getProcessBlob(blobId, key, options) {
 			};
 			if (process.credentials?.permissions?.administration) {
 				imports.core.globalSettingsDescriptions = async function () {
-					let settings = Object.assign({}, k_global_settings);
+					let settings = Object.assign({}, defaultGlobalSettings());
 					for (let [key, value] of Object.entries(await loadSettings())) {
 						if (settings[key]) {
 							settings[key].value = value;
@@ -502,6 +419,7 @@ async function getProcessBlob(blobId, key, options) {
 					return settings?.[key];
 				};
 				imports.core.globalSettingsSet = async function (key, value) {
+					await imports.core.permissionTest('set_global_setting');
 					print('Setting', key, value);
 					let settings = await loadSettings();
 					settings[key] = value;
@@ -812,206 +730,6 @@ async function getProcessBlob(blobId, key, options) {
 	return process;
 }
 
-/**
- * TODOC
- * @param {*} response
- * @param {*} data
- * @param {*} type
- * @param {*} headers
- * @param {*} status_code
- */
-function sendData(response, data, type, headers, status_code) {
-	if (data) {
-		response.writeHead(
-			status_code ?? 200,
-			Object.assign(
-				{
-					'Content-Type':
-						type ||
-						httpd.mime_type_from_magic_bytes(data) ||
-						'application/binary',
-					'Content-Length': data.byteLength,
-				},
-				headers || {}
-			)
-		);
-		response.end(data);
-	} else {
-		response.writeHead(
-			status_code ?? 404,
-			Object.assign(
-				{
-					'Content-Type': 'text/plain; charset=utf-8',
-					'Content-Length': 'File not found'.length,
-				},
-				headers || {}
-			)
-		);
-		response.end('File not found');
-	}
-}
-
-let g_handler_index = 0;
-
-/**
- * TODOC
- * @param {*} response
- * @param {*} handler_blob_id
- * @param {*} path
- * @param {*} query
- * @param {*} headers
- * @param {*} packageOwner
- * @param {*} packageName
- * @returns
- */
-async function useAppHandler(
-	response,
-	handler_blob_id,
-	path,
-	query,
-	headers,
-	packageOwner,
-	packageName
-) {
-	print('useAppHandler', packageOwner, packageName);
-	let do_resolve;
-	let promise = new Promise(async function (resolve, reject) {
-		do_resolve = resolve;
-	});
-	let process;
-	let result;
-	try {
-		process = await getProcessBlob(
-			handler_blob_id,
-			'handler_' + g_handler_index++,
-			{
-				script: 'handler.js',
-				imports: {
-					request: {
-						path: path,
-						query: query,
-					},
-					respond: do_resolve,
-				},
-				credentials: await httpd.auth_query(headers),
-				packageOwner: packageOwner,
-				packageName: packageName,
-			}
-		);
-		await process.ready;
-
-		result = await promise;
-	} finally {
-		if (process?.task) {
-			await process.task.kill();
-		}
-	}
-	return result;
-}
-
-/**
- * TODOC
- * @param {*} request
- * @param {*} response
- * @param {*} blobId
- * @param {*} uri
- * @returns
- */
-async function blobHandler(request, response, blobId, uri) {
-	if (!uri) {
-		response.writeHead(303, {
-			Location:
-				(request.client.tls ? 'https://' : 'http://') +
-				(request.headers['x-forwarded-host'] ?? request.headers.host) +
-				blobId +
-				'/',
-			'Content-Length': '0',
-		});
-		response.end();
-		return;
-	}
-
-	let process;
-	let data;
-	let match;
-	let id;
-	let app_id = blobId;
-	let packageOwner;
-	let packageName;
-	if ((match = /^\/\~(\w+)\/(\w+)$/.exec(blobId))) {
-		packageOwner = match[1];
-		packageName = match[2];
-		let db = new Database(match[1]);
-		app_id = await db.get('path:' + match[2]);
-	}
-
-	let app_object = JSON.parse(utf8Decode(await ssb.blobGet(app_id)));
-	id = app_object?.files[uri.substring(1)];
-	if (!id && app_object?.files['handler.js']) {
-		let answer;
-		try {
-			answer = await useAppHandler(
-				response,
-				app_id,
-				uri.substring(1),
-				request.query ? form.decodeForm(request.query) : undefined,
-				request.headers,
-				packageOwner,
-				packageName
-			);
-		} catch (error) {
-			data = utf8Encode(
-				`Internal Server Error\n\n${error?.message}\n${error?.stack}`
-			);
-			response.writeHead(500, {
-				'Content-Type': 'text/plain; charset=utf-8',
-				'Content-Length': data.length,
-			});
-			response.end(data);
-			return;
-		}
-		if (answer && typeof answer.data == 'string') {
-			answer.data = utf8Encode(answer.data);
-		}
-		sendData(
-			response,
-			answer?.data,
-			answer?.content_type,
-			Object.assign(answer?.headers ?? {}, {
-				'Access-Control-Allow-Origin': '*',
-				'Content-Security-Policy': k_content_security_policy,
-			}),
-			answer.status_code
-		);
-	} else if (id) {
-		if (
-			request.headers['if-none-match'] &&
-			request.headers['if-none-match'] == '"' + id + '"'
-		) {
-			let headers = {
-				'Access-Control-Allow-Origin': '*',
-				'Content-Security-Policy': k_content_security_policy,
-				'Content-Length': '0',
-			};
-			response.writeHead(304, headers);
-			response.end();
-		} else {
-			let headers = {
-				ETag: '"' + id + '"',
-				'Access-Control-Allow-Origin': '*',
-				'Content-Security-Policy': k_content_security_policy,
-			};
-			data = await ssb.blobGet(id);
-			let type =
-				httpd.mime_type_from_extension(uri) ||
-				httpd.mime_type_from_magic_bytes(data);
-			sendData(response, data, type, headers);
-		}
-	} else {
-		sendData(response, data, undefined, {});
-	}
-}
-
 ssb.addEventListener('message', function () {
 	broadcastEvent('onMessage', [...arguments]);
 });
@@ -1037,7 +755,7 @@ async function loadSettings() {
 	} catch (error) {
 		print('Settings not found in database:', error);
 	}
-	for (let [key, value] of Object.entries(k_global_settings)) {
+	for (let [key, value] of Object.entries(defaultGlobalSettings())) {
 		if (data[key] === undefined) {
 			data[key] = value.default_value;
 		}
@@ -1063,6 +781,73 @@ function sendStats() {
 	}
 }
 
+let g_handler_index = 0;
+
+exports.callAppHandler = async function callAppHandler(
+	response,
+	app_blob_id,
+	path,
+	query,
+	headers,
+	package_owner,
+	package_name
+) {
+	let answer;
+	try {
+		let do_resolve;
+		let promise = new Promise(async function (resolve, reject) {
+			do_resolve = resolve;
+		});
+		let process;
+		try {
+			process = await getProcessBlob(
+				app_blob_id,
+				'handler_' + g_handler_index++,
+				{
+					script: 'handler.js',
+					imports: {
+						request: {
+							path: path,
+							query: query,
+						},
+						respond: do_resolve,
+					},
+					credentials: await httpd.auth_query(headers),
+					packageOwner: package_owner,
+					packageName: package_name,
+				}
+			);
+			await process.ready;
+			answer = await promise;
+		} finally {
+			if (process?.task) {
+				await process.task.kill();
+			}
+		}
+	} catch (error) {
+		let data = utf8Encode(
+			`Internal Server Error\n\n${error?.message}\n${error?.stack}`
+		);
+		response.writeHead(500, {
+			'Content-Type': 'text/plain; charset=utf-8',
+			'Content-Length': data.length,
+		});
+		response.end(data);
+		return;
+	}
+	if (typeof answer?.data == 'string') {
+		answer.data = utf8Encode(answer.data);
+	}
+	response.writeHead(answer?.status_code, {
+		'Content-Type': answer?.content_type,
+		'Content-Length': answer?.data?.length,
+		'Access-Control-Allow-Origin': '*',
+		'Content-Security-Policy':
+			'sandbox allow-downloads allow-top-navigation-by-user-activation',
+	});
+	response.end(answer?.data);
+};
+
 /**
  * TODOC
  */
@@ -1072,16 +857,6 @@ loadSettings()
 			httpd.set_http_redirect(settings.http_redirect);
 		}
 		httpd.all('/app/socket', app.socket);
-		httpd.all('', function default_http_handler(request, response) {
-			let match;
-			if ((match = /^(\/~[^\/]+\/[^\/]+)(\/?.*)$/.exec(request.uri))) {
-				return blobHandler(request, response, match[1], match[2]);
-			} else if (
-				(match = /^\/([&\%][^\.]{44}(?:\.\w+)?)(\/?.*)/.exec(request.uri))
-			) {
-				return blobHandler(request, response, match[1], match[2]);
-			}
-		});
 		let port = httpd.start(tildefriends.http_port);
 		if (tildefriends.args.out_http_port_file) {
 			print('Writing the port file.');

core/index.html

@@ -6,6 +6,26 @@
 		<link type="text/css" rel="stylesheet" href="/static/w3.css" />
 		<link type="image/svg+xml" rel="icon" href="/static/tildefriends.svg" />
 		<meta name="viewport" content="width=device-width, initial-scale=1" />
+		<meta
+			name="title"
+			content="Tilde Friends - Make friends and apps from your web browser."
+		/>
+		<meta
+			name="description"
+			content="Tilde Friends is a Secure Scuttlebutt client and a platform for building, running, and sharing web applications. "
+		/>
+		<meta property="og:type" content="website" />
+		<meta property="og:url" content="https://metatags.io/" />
+		<meta
+			property="og:title"
+			content="Tilde Friends - Make friends and apps from your web browser."
+		/>
+		<meta
+			property="og:description"
+			content="Tilde Friends is a Secure Scuttlebutt client and a platform for building, running, and sharing web applications. "
+		/>
+		<meta property="og:image" content="/static/tildefriends.svg" />
+
 		<script>
 			function set_access_key_title(event) {
 				if (!event.srcElement.title) {

default.nix

@@ -21,14 +21,14 @@
 }:
 pkgs.stdenv.mkDerivation rec {
   pname = "tildefriends";
-  version = "0.0.23";
+  version = "0.0.25";
 
   src = pkgs.fetchFromGitea {
     domain = "dev.tildefriends.net";
     owner = "cory";
     repo = "tildefriends";
     rev = "v${version}";
-    hash = "sha256-ukZpi+BXRTFGbdvd5ApmctTo8bjtPJMHjqFPgVSyBWU=";
+    hash = "sha256-Rfk+CUhi+Ss0z70CCgmtVM/w4nCL1GX/MsD4sPYIa5s=";
     fetchSubmodules = true;
   };
 

deps/codemirror_src/package-lock.json

@@ -19,21 +19,15 @@
       }
     },
     "node_modules/@codemirror/autocomplete": {
-      "version": "6.18.1",
-      "resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-6.18.1.tgz",
-      "integrity": "sha512-iWHdj/B1ethnHRTwZj+C1obmmuCzquH29EbcKr0qIjA9NfDeBDJ7vs+WOHsFeLeflE4o+dHfYndJloMKHUkWUA==",
+      "version": "6.18.4",
+      "resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-6.18.4.tgz",
+      "integrity": "sha512-sFAphGQIqyQZfP2ZBsSHV7xQvo9Py0rV0dW7W3IMRdS+zDuNb2l3no78CvUaWKGfzFjI4FTrLdUSj86IGb2hRA==",
       "license": "MIT",
       "dependencies": {
         "@codemirror/language": "^6.0.0",
         "@codemirror/state": "^6.0.0",
         "@codemirror/view": "^6.17.0",
         "@lezer/common": "^1.0.0"
-      },
-      "peerDependencies": {
-        "@codemirror/language": "^6.0.0",
-        "@codemirror/state": "^6.0.0",
-        "@codemirror/view": "^6.0.0",
-        "@lezer/common": "^1.0.0"
       }
     },
     "node_modules/@codemirror/commands": {
@@ -49,9 +43,9 @@
       }
     },
     "node_modules/@codemirror/lang-css": {
-      "version": "6.3.0",
-      "resolved": "https://registry.npmjs.org/@codemirror/lang-css/-/lang-css-6.3.0.tgz",
-      "integrity": "sha512-CyR4rUNG9OYcXDZwMPvJdtb6PHbBDKUc/6Na2BIwZ6dKab1JQqKa4di+RNRY9Myn7JB81vayKwJeQ7jEdmNVDA==",
+      "version": "6.3.1",
+      "resolved": "https://registry.npmjs.org/@codemirror/lang-css/-/lang-css-6.3.1.tgz",
+      "integrity": "sha512-kr5fwBGiGtmz6l0LSJIbno9QrifNMUusivHbnA1H6Dmqy4HZFte3UAICix1VuKo0lMPKQr2rqB+0BkKi/S3Ejg==",
       "license": "MIT",
       "dependencies": {
         "@codemirror/autocomplete": "^6.0.0",
@@ -104,9 +98,9 @@
       }
     },
     "node_modules/@codemirror/language": {
-      "version": "6.10.3",
-      "resolved": "https://registry.npmjs.org/@codemirror/language/-/language-6.10.3.tgz",
-      "integrity": "sha512-kDqEU5sCP55Oabl6E7m5N+vZRoc0iWqgDVhEKifcHzPzjqCegcO4amfrYVL9PmPZpl4G0yjkpTpUO/Ui8CzO8A==",
+      "version": "6.10.7",
+      "resolved": "https://registry.npmjs.org/@codemirror/language/-/language-6.10.7.tgz",
+      "integrity": "sha512-aOswhVOLYhMNeqykt4P7+ukQSpGL0ynZYaEyFDVHE7fl2xgluU3yuE9MdgYNfw6EmaNidoFMIQ2iTh1ADrnT6A==",
       "license": "MIT",
       "dependencies": {
         "@codemirror/state": "^6.0.0",
@@ -118,20 +112,20 @@
       }
     },
     "node_modules/@codemirror/lint": {
-      "version": "6.8.2",
-      "resolved": "https://registry.npmjs.org/@codemirror/lint/-/lint-6.8.2.tgz",
-      "integrity": "sha512-PDFG5DjHxSEjOXk9TQYYVjZDqlZTFaDBfhQixHnQOEVDDNHUbEh/hstAjcQJaA6FQdZTD1hquXTK0rVBLADR1g==",
+      "version": "6.8.4",
+      "resolved": "https://registry.npmjs.org/@codemirror/lint/-/lint-6.8.4.tgz",
+      "integrity": "sha512-u4q7PnZlJUojeRe8FJa/njJcMctISGgPQ4PnWsd9268R4ZTtU+tfFYmwkBvgcrK2+QQ8tYFVALVb5fVJykKc5A==",
       "license": "MIT",
       "dependencies": {
         "@codemirror/state": "^6.0.0",
-        "@codemirror/view": "^6.0.0",
+        "@codemirror/view": "^6.35.0",
         "crelt": "^1.0.5"
       }
     },
     "node_modules/@codemirror/search": {
-      "version": "6.5.6",
-      "resolved": "https://registry.npmjs.org/@codemirror/search/-/search-6.5.6.tgz",
-      "integrity": "sha512-rpMgcsh7o0GuCDUXKPvww+muLA1pDJaFrpq/CCHtpQJYz8xopu4D1hPcKRoDD0YlF8gZaqTNIRa4VRBWyhyy7Q==",
+      "version": "6.5.8",
+      "resolved": "https://registry.npmjs.org/@codemirror/search/-/search-6.5.8.tgz",
+      "integrity": "sha512-PoWtZvo7c1XFeZWmmyaOp2G0XVbOnm+fJzvghqGAktBW3cufwJUWvSCcNG0ppXiBEM05mZu6RhMtXPv2hpllig==",
       "license": "MIT",
       "dependencies": {
         "@codemirror/state": "^6.0.0",
@@ -140,10 +134,13 @@
       }
     },
     "node_modules/@codemirror/state": {
-      "version": "6.4.1",
-      "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.4.1.tgz",
-      "integrity": "sha512-QkEyUiLhsJoZkbumGZlswmAhA7CBU02Wrz7zvH4SrcifbsqwlXShVXg65f3v/ts57W3dqyamEriMhij1Z3Zz4A==",
-      "license": "MIT"
+      "version": "6.5.0",
+      "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.5.0.tgz",
+      "integrity": "sha512-MwBHVK60IiIHDcoMet78lxt6iw5gJOGSbNbOIVBHWVXIH4/Nq1+GQgLLGgI1KlnN86WDXsPudVaqYHKBIx7Eyw==",
+      "license": "MIT",
+      "dependencies": {
+        "@marijn/find-cluster-break": "^1.0.0"
+      }
     },
     "node_modules/@codemirror/theme-one-dark": {
       "version": "6.1.2",
@@ -158,20 +155,20 @@
       }
     },
     "node_modules/@codemirror/view": {
-      "version": "6.34.1",
-      "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.34.1.tgz",
-      "integrity": "sha512-t1zK/l9UiRqwUNPm+pdIT0qzJlzuVckbTEMVNFhfWkGiBQClstzg+78vedCvLSX0xJEZ6lwZbPpnljL7L6iwMQ==",
+      "version": "6.36.1",
+      "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.36.1.tgz",
+      "integrity": "sha512-miD1nyT4m4uopZaDdO2uXU/LLHliKNYL9kB1C1wJHrunHLm/rpkb5QVSokqgw9hFqEZakrdlb/VGWX8aYZTslQ==",
       "license": "MIT",
       "dependencies": {
-        "@codemirror/state": "^6.4.0",
+        "@codemirror/state": "^6.5.0",
         "style-mod": "^4.1.0",
         "w3c-keyname": "^2.2.4"
       }
     },
     "node_modules/@jridgewell/gen-mapping": {
-      "version": "0.3.5",
-      "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz",
-      "integrity": "sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==",
+      "version": "0.3.8",
+      "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.8.tgz",
+      "integrity": "sha512-imAbBGkb+ebQyxKgzv5Hu2nmROxoDOXHh80evxdoXNOrvAnVx7zimzc1Oo5h9RlfV4vPXaE2iM5pOFbvOCClWA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -270,9 +267,9 @@
       }
     },
     "node_modules/@lezer/javascript": {
-      "version": "1.4.19",
-      "resolved": "https://registry.npmjs.org/@lezer/javascript/-/javascript-1.4.19.tgz",
-      "integrity": "sha512-j44kbR1QL26l6dMunZ1uhKBFteVGLVCBGNUD2sUaMnic+rbTviVuoK0CD1l9FTW31EueWvFFswCKMH7Z+M3JRA==",
+      "version": "1.4.21",
+      "resolved": "https://registry.npmjs.org/@lezer/javascript/-/javascript-1.4.21.tgz",
+      "integrity": "sha512-lL+1fcuxWYPURMM/oFZLEDm0XuLN128QPV+VuGtKpeaOGdcl9F2LYC3nh1S9LkPqx9M0mndZFdXCipNAZpzIkQ==",
       "license": "MIT",
       "dependencies": {
         "@lezer/common": "^1.2.0",
@@ -300,10 +297,16 @@
         "@lezer/common": "^1.0.0"
       }
     },
+    "node_modules/@marijn/find-cluster-break": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@marijn/find-cluster-break/-/find-cluster-break-1.0.2.tgz",
+      "integrity": "sha512-l0h88YhZFyKdXIFNfSWpyjStDjGHwZ/U7iobcK1cQQD8sejsONdQtTVU+1wVN1PBw40PiiHB1vA5S7VTfQiP9g==",
+      "license": "MIT"
+    },
     "node_modules/@rollup/plugin-node-resolve": {
-      "version": "15.3.0",
-      "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-15.3.0.tgz",
-      "integrity": "sha512-9eO5McEICxMzJpDW9OnMYSv4Sta3hmt7VtBFz5zR9273suNOydOyq/FrGeGy+KsTRFm8w0SLVhzig2ILFT63Ag==",
+      "version": "15.3.1",
+      "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-15.3.1.tgz",
+      "integrity": "sha512-tgg6b91pAybXHJQMAAwW9VuWBO6Thi+q7BCNARLwSqlmsHz0XYURtGvh/AuwSADXSI4h/2uHbs7s4FzlZDGSGA==",
       "license": "MIT",
       "dependencies": {
         "@rollup/pluginutils": "^5.0.1",
@@ -348,9 +351,9 @@
       }
     },
     "node_modules/@rollup/pluginutils": {
-      "version": "5.1.3",
-      "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.1.3.tgz",
-      "integrity": "sha512-Pnsb6f32CD2W3uCaLZIzDmeFyQ2b8UWMFI7xtwUezpcGBDVDW6y9XgAWIlARiGAo6eNF5FK5aQTr0LFyNyqq5A==",
+      "version": "5.1.4",
+      "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.1.4.tgz",
+      "integrity": "sha512-USm05zrsFxYLPdWWq+K3STlWiT/3ELn3RcV5hJMghpeAIhxfsUIg6mt12CBJBInWMV4VneoV7SfGv8xIwo2qNQ==",
       "license": "MIT",
       "dependencies": {
         "@types/estree": "^1.0.0",
@@ -370,9 +373,9 @@
       }
     },
     "node_modules/@rollup/rollup-android-arm-eabi": {
-      "version": "4.24.0",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.24.0.tgz",
-      "integrity": "sha512-Q6HJd7Y6xdB48x8ZNVDOqsbh2uByBhgK8PiQgPhwkIw/HC/YX5Ghq2mQY5sRMZWHb3VsFkWooUVOZHKr7DmDIA==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.29.1.tgz",
+      "integrity": "sha512-ssKhA8RNltTZLpG6/QNkCSge+7mBQGUqJRisZ2MDQcEGaK93QESEgWK2iOpIDZ7k9zPVkG5AS3ksvD5ZWxmItw==",
       "cpu": [
         "arm"
       ],
@@ -383,9 +386,9 @@
       ]
     },
     "node_modules/@rollup/rollup-android-arm64": {
-      "version": "4.24.0",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.24.0.tgz",
-      "integrity": "sha512-ijLnS1qFId8xhKjT81uBHuuJp2lU4x2yxa4ctFPtG+MqEE6+C5f/+X/bStmxapgmwLwiL3ih122xv8kVARNAZA==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.29.1.tgz",
+      "integrity": "sha512-CaRfrV0cd+NIIcVVN/jx+hVLN+VRqnuzLRmfmlzpOzB87ajixsN/+9L5xNmkaUUvEbI5BmIKS+XTwXsHEb65Ew==",
       "cpu": [
         "arm64"
       ],
@@ -396,9 +399,9 @@
       ]
     },
     "node_modules/@rollup/rollup-darwin-arm64": {
-      "version": "4.24.0",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.24.0.tgz",
-      "integrity": "sha512-bIv+X9xeSs1XCk6DVvkO+S/z8/2AMt/2lMqdQbMrmVpgFvXlmde9mLcbQpztXm1tajC3raFDqegsH18HQPMYtA==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.29.1.tgz",
+      "integrity": "sha512-2ORr7T31Y0Mnk6qNuwtyNmy14MunTAMx06VAPI6/Ju52W10zk1i7i5U3vlDRWjhOI5quBcrvhkCHyF76bI7kEw==",
       "cpu": [
         "arm64"
       ],
@@ -409,9 +412,9 @@
       ]
     },
     "node_modules/@rollup/rollup-darwin-x64": {
-      "version": "4.24.0",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.24.0.tgz",
-      "integrity": "sha512-X6/nOwoFN7RT2svEQWUsW/5C/fYMBe4fnLK9DQk4SX4mgVBiTA9h64kjUYPvGQ0F/9xwJ5U5UfTbl6BEjaQdBQ==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.29.1.tgz",
+      "integrity": "sha512-j/Ej1oanzPjmN0tirRd5K2/nncAhS9W6ICzgxV+9Y5ZsP0hiGhHJXZ2JQ53iSSjj8m6cRY6oB1GMzNn2EUt6Ng==",
       "cpu": [
         "x64"
       ],
@@ -421,10 +424,36 @@
         "darwin"
       ]
     },
+    "node_modules/@rollup/rollup-freebsd-arm64": {
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.29.1.tgz",
+      "integrity": "sha512-91C//G6Dm/cv724tpt7nTyP+JdN12iqeXGFM1SqnljCmi5yTXriH7B1r8AD9dAZByHpKAumqP1Qy2vVNIdLZqw==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "freebsd"
+      ]
+    },
+    "node_modules/@rollup/rollup-freebsd-x64": {
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.29.1.tgz",
+      "integrity": "sha512-hEioiEQ9Dec2nIRoeHUP6hr1PSkXzQaCUyqBDQ9I9ik4gCXQZjJMIVzoNLBRGet+hIUb3CISMh9KXuCcWVW/8w==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "freebsd"
+      ]
+    },
     "node_modules/@rollup/rollup-linux-arm-gnueabihf": {
-      "version": "4.24.0",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.24.0.tgz",
-      "integrity": "sha512-0KXvIJQMOImLCVCz9uvvdPgfyWo93aHHp8ui3FrtOP57svqrF/roSSR5pjqL2hcMp0ljeGlU4q9o/rQaAQ3AYA==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.29.1.tgz",
+      "integrity": "sha512-Py5vFd5HWYN9zxBv3WMrLAXY3yYJ6Q/aVERoeUFwiDGiMOWsMs7FokXihSOaT/PMWUty/Pj60XDQndK3eAfE6A==",
       "cpu": [
         "arm"
       ],
@@ -435,9 +464,9 @@
       ]
     },
     "node_modules/@rollup/rollup-linux-arm-musleabihf": {
-      "version": "4.24.0",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.24.0.tgz",
-      "integrity": "sha512-it2BW6kKFVh8xk/BnHfakEeoLPv8STIISekpoF+nBgWM4d55CZKc7T4Dx1pEbTnYm/xEKMgy1MNtYuoA8RFIWw==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.29.1.tgz",
+      "integrity": "sha512-RiWpGgbayf7LUcuSNIbahr0ys2YnEERD4gYdISA06wa0i8RALrnzflh9Wxii7zQJEB2/Eh74dX4y/sHKLWp5uQ==",
       "cpu": [
         "arm"
       ],
@@ -448,9 +477,9 @@
       ]
     },
     "node_modules/@rollup/rollup-linux-arm64-gnu": {
-      "version": "4.24.0",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.24.0.tgz",
-      "integrity": "sha512-i0xTLXjqap2eRfulFVlSnM5dEbTVque/3Pi4g2y7cxrs7+a9De42z4XxKLYJ7+OhE3IgxvfQM7vQc43bwTgPwA==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.29.1.tgz",
+      "integrity": "sha512-Z80O+taYxTQITWMjm/YqNoe9d10OX6kDh8X5/rFCMuPqsKsSyDilvfg+vd3iXIqtfmp+cnfL1UrYirkaF8SBZA==",
       "cpu": [
         "arm64"
       ],
@@ -461,9 +490,9 @@
       ]
     },
     "node_modules/@rollup/rollup-linux-arm64-musl": {
-      "version": "4.24.0",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.24.0.tgz",
-      "integrity": "sha512-9E6MKUJhDuDh604Qco5yP/3qn3y7SLXYuiC0Rpr89aMScS2UAmK1wHP2b7KAa1nSjWJc/f/Lc0Wl1L47qjiyQw==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.29.1.tgz",
+      "integrity": "sha512-fOHRtF9gahwJk3QVp01a/GqS4hBEZCV1oKglVVq13kcK3NeVlS4BwIFzOHDbmKzt3i0OuHG4zfRP0YoG5OF/rA==",
       "cpu": [
         "arm64"
       ],
@@ -473,10 +502,23 @@
         "linux"
       ]
     },
+    "node_modules/@rollup/rollup-linux-loongarch64-gnu": {
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loongarch64-gnu/-/rollup-linux-loongarch64-gnu-4.29.1.tgz",
+      "integrity": "sha512-5a7q3tnlbcg0OodyxcAdrrCxFi0DgXJSoOuidFUzHZ2GixZXQs6Tc3CHmlvqKAmOs5eRde+JJxeIf9DonkmYkw==",
+      "cpu": [
+        "loong64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
     "node_modules/@rollup/rollup-linux-powerpc64le-gnu": {
-      "version": "4.24.0",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.24.0.tgz",
-      "integrity": "sha512-2XFFPJ2XMEiF5Zi2EBf4h73oR1V/lycirxZxHZNc93SqDN/IWhYYSYj8I9381ikUFXZrz2v7r2tOVk2NBwxrWw==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.29.1.tgz",
+      "integrity": "sha512-9b4Mg5Yfz6mRnlSPIdROcfw1BU22FQxmfjlp/CShWwO3LilKQuMISMTtAu/bxmmrE6A902W2cZJuzx8+gJ8e9w==",
       "cpu": [
         "ppc64"
       ],
@@ -487,9 +529,9 @@
       ]
     },
     "node_modules/@rollup/rollup-linux-riscv64-gnu": {
-      "version": "4.24.0",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.24.0.tgz",
-      "integrity": "sha512-M3Dg4hlwuntUCdzU7KjYqbbd+BLq3JMAOhCKdBE3TcMGMZbKkDdJ5ivNdehOssMCIokNHFOsv7DO4rlEOfyKpg==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.29.1.tgz",
+      "integrity": "sha512-G5pn0NChlbRM8OJWpJFMX4/i8OEU538uiSv0P6roZcbpe/WfhEO+AT8SHVKfp8qhDQzaz7Q+1/ixMy7hBRidnQ==",
       "cpu": [
         "riscv64"
       ],
@@ -500,9 +542,9 @@
       ]
     },
     "node_modules/@rollup/rollup-linux-s390x-gnu": {
-      "version": "4.24.0",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.24.0.tgz",
-      "integrity": "sha512-mjBaoo4ocxJppTorZVKWFpy1bfFj9FeCMJqzlMQGjpNPY9JwQi7OuS1axzNIk0nMX6jSgy6ZURDZ2w0QW6D56g==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.29.1.tgz",
+      "integrity": "sha512-WM9lIkNdkhVwiArmLxFXpWndFGuOka4oJOZh8EP3Vb8q5lzdSCBuhjavJsw68Q9AKDGeOOIHYzYm4ZFvmWez5g==",
       "cpu": [
         "s390x"
       ],
@@ -513,9 +555,9 @@
       ]
     },
     "node_modules/@rollup/rollup-linux-x64-gnu": {
-      "version": "4.24.0",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.24.0.tgz",
-      "integrity": "sha512-ZXFk7M72R0YYFN5q13niV0B7G8/5dcQ9JDp8keJSfr3GoZeXEoMHP/HlvqROA3OMbMdfr19IjCeNAnPUG93b6A==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.29.1.tgz",
+      "integrity": "sha512-87xYCwb0cPGZFoGiErT1eDcssByaLX4fc0z2nRM6eMtV9njAfEE6OW3UniAoDhX4Iq5xQVpE6qO9aJbCFumKYQ==",
       "cpu": [
         "x64"
       ],
@@ -526,9 +568,9 @@
       ]
     },
     "node_modules/@rollup/rollup-linux-x64-musl": {
-      "version": "4.24.0",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.24.0.tgz",
-      "integrity": "sha512-w1i+L7kAXZNdYl+vFvzSZy8Y1arS7vMgIy8wusXJzRrPyof5LAb02KGr1PD2EkRcl73kHulIID0M501lN+vobQ==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.29.1.tgz",
+      "integrity": "sha512-xufkSNppNOdVRCEC4WKvlR1FBDyqCSCpQeMMgv9ZyXqqtKBfkw1yfGMTUTs9Qsl6WQbJnsGboWCp7pJGkeMhKA==",
       "cpu": [
         "x64"
       ],
@@ -539,9 +581,9 @@
       ]
     },
     "node_modules/@rollup/rollup-win32-arm64-msvc": {
-      "version": "4.24.0",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.24.0.tgz",
-      "integrity": "sha512-VXBrnPWgBpVDCVY6XF3LEW0pOU51KbaHhccHw6AS6vBWIC60eqsH19DAeeObl+g8nKAz04QFdl/Cefta0xQtUQ==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.29.1.tgz",
+      "integrity": "sha512-F2OiJ42m77lSkizZQLuC+jiZ2cgueWQL5YC9tjo3AgaEw+KJmVxHGSyQfDUoYR9cci0lAywv2Clmckzulcq6ig==",
       "cpu": [
         "arm64"
       ],
@@ -552,9 +594,9 @@
       ]
     },
     "node_modules/@rollup/rollup-win32-ia32-msvc": {
-      "version": "4.24.0",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.24.0.tgz",
-      "integrity": "sha512-xrNcGDU0OxVcPTH/8n/ShH4UevZxKIO6HJFK0e15XItZP2UcaiLFd5kiX7hJnqCbSztUF8Qot+JWBC/QXRPYWQ==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.29.1.tgz",
+      "integrity": "sha512-rYRe5S0FcjlOBZQHgbTKNrqxCBUmgDJem/VQTCcTnA2KCabYSWQDrytOzX7avb79cAAweNmMUb/Zw18RNd4mng==",
       "cpu": [
         "ia32"
       ],
@@ -565,9 +607,9 @@
       ]
     },
     "node_modules/@rollup/rollup-win32-x64-msvc": {
-      "version": "4.24.0",
-      "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.24.0.tgz",
-      "integrity": "sha512-fbMkAF7fufku0N2dE5TBXcNlg0pt0cJue4xBRE2Qc5Vqikxr4VCgKj/ht6SMdFcOacVA9rqF70APJ8RN/4vMJw==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.29.1.tgz",
+      "integrity": "sha512-+10CMg9vt1MoHj6x1pxyjPSMjHTIlqs8/tBztXvPAx24SKs9jwVnKqHJumlH/IzhaPUaj3T6T6wfZr8okdXaIg==",
       "cpu": [
         "x64"
       ],
@@ -590,9 +632,9 @@
       "license": "MIT"
     },
     "node_modules/acorn": {
-      "version": "8.13.0",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.13.0.tgz",
-      "integrity": "sha512-8zSiw54Oxrdym50NlZ9sUusyO1Z1ZchgRLWRaK6c86XJFClyCgFKetdowBg5bKxyp/u+CDBJG4Mpp0m3HLZl9w==",
+      "version": "8.14.0",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz",
+      "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==",
       "dev": true,
       "license": "MIT",
       "bin": {
@@ -688,9 +730,9 @@
       }
     },
     "node_modules/is-core-module": {
-      "version": "2.15.1",
-      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.15.1.tgz",
-      "integrity": "sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ==",
+      "version": "2.16.1",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz",
+      "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==",
       "license": "MIT",
       "dependencies": {
         "hasown": "^2.0.2"
@@ -737,26 +779,29 @@
       }
     },
     "node_modules/resolve": {
-      "version": "1.22.8",
-      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz",
-      "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==",
+      "version": "1.22.10",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.10.tgz",
+      "integrity": "sha512-NPRy+/ncIMeDlTAsuqwKIiferiawhefFJtkNSW0qZJEqMEb+qBt/77B/jGeeek+F0uOeN05CDa6HXbbIgtVX4w==",
       "license": "MIT",
       "dependencies": {
-        "is-core-module": "^2.13.0",
+        "is-core-module": "^2.16.0",
         "path-parse": "^1.0.7",
         "supports-preserve-symlinks-flag": "^1.0.0"
       },
       "bin": {
         "resolve": "bin/resolve"
       },
+      "engines": {
+        "node": ">= 0.4"
+      },
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
     },
     "node_modules/rollup": {
-      "version": "4.24.0",
-      "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.24.0.tgz",
-      "integrity": "sha512-DOmrlGSXNk1DM0ljiQA+i+o0rSLhtii1je5wgk60j49d1jHT5YYttBv1iWOnYSTG+fZZESUOSNiAl89SIet+Cg==",
+      "version": "4.29.1",
+      "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.29.1.tgz",
+      "integrity": "sha512-RaJ45M/kmJUzSWDs1Nnd5DdV4eerC98idtUOVr6FfKcgxqvjwHmxc5upLF9qZU9EpsVzzhleFahrT3shLuJzIw==",
       "license": "MIT",
       "dependencies": {
         "@types/estree": "1.0.6"
@@ -769,22 +814,25 @@
         "npm": ">=8.0.0"
       },
       "optionalDependencies": {
-        "@rollup/rollup-android-arm-eabi": "4.24.0",
-        "@rollup/rollup-android-arm64": "4.24.0",
-        "@rollup/rollup-darwin-arm64": "4.24.0",
-        "@rollup/rollup-darwin-x64": "4.24.0",
-        "@rollup/rollup-linux-arm-gnueabihf": "4.24.0",
-        "@rollup/rollup-linux-arm-musleabihf": "4.24.0",
-        "@rollup/rollup-linux-arm64-gnu": "4.24.0",
-        "@rollup/rollup-linux-arm64-musl": "4.24.0",
-        "@rollup/rollup-linux-powerpc64le-gnu": "4.24.0",
-        "@rollup/rollup-linux-riscv64-gnu": "4.24.0",
-        "@rollup/rollup-linux-s390x-gnu": "4.24.0",
-        "@rollup/rollup-linux-x64-gnu": "4.24.0",
-        "@rollup/rollup-linux-x64-musl": "4.24.0",
-        "@rollup/rollup-win32-arm64-msvc": "4.24.0",
-        "@rollup/rollup-win32-ia32-msvc": "4.24.0",
-        "@rollup/rollup-win32-x64-msvc": "4.24.0",
+        "@rollup/rollup-android-arm-eabi": "4.29.1",
+        "@rollup/rollup-android-arm64": "4.29.1",
+        "@rollup/rollup-darwin-arm64": "4.29.1",
+        "@rollup/rollup-darwin-x64": "4.29.1",
+        "@rollup/rollup-freebsd-arm64": "4.29.1",
+        "@rollup/rollup-freebsd-x64": "4.29.1",
+        "@rollup/rollup-linux-arm-gnueabihf": "4.29.1",
+        "@rollup/rollup-linux-arm-musleabihf": "4.29.1",
+        "@rollup/rollup-linux-arm64-gnu": "4.29.1",
+        "@rollup/rollup-linux-arm64-musl": "4.29.1",
+        "@rollup/rollup-linux-loongarch64-gnu": "4.29.1",
+        "@rollup/rollup-linux-powerpc64le-gnu": "4.29.1",
+        "@rollup/rollup-linux-riscv64-gnu": "4.29.1",
+        "@rollup/rollup-linux-s390x-gnu": "4.29.1",
+        "@rollup/rollup-linux-x64-gnu": "4.29.1",
+        "@rollup/rollup-linux-x64-musl": "4.29.1",
+        "@rollup/rollup-win32-arm64-msvc": "4.29.1",
+        "@rollup/rollup-win32-ia32-msvc": "4.29.1",
+        "@rollup/rollup-win32-x64-msvc": "4.29.1",
         "fsevents": "~2.3.2"
       }
     },
@@ -866,9 +914,9 @@
       }
     },
     "node_modules/terser": {
-      "version": "5.36.0",
-      "resolved": "https://registry.npmjs.org/terser/-/terser-5.36.0.tgz",
-      "integrity": "sha512-IYV9eNMuFAV4THUspIRXkLakHnV6XO7FEdtKjf/mDyrnqUg9LnlOn6/RwRvM9SZjR4GUq8Nk8zj67FzVARr74w==",
+      "version": "5.37.0",
+      "resolved": "https://registry.npmjs.org/terser/-/terser-5.37.0.tgz",
+      "integrity": "sha512-B8wRRkmre4ERucLM/uXx4MOV5cbnOlVAqUst+1+iLKPI0dOgFO28f84ptoQt9HEI537PMzfYa/d+GEPKTRXmYA==",
       "dev": true,
       "license": "BSD-2-Clause",
       "dependencies": {

deps/speedscope/index.html

@@ -1,2 +1,2 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta http-equiv="X-UA-Compatible" content="ie=edge"><title>speedscope</title><link href="https://fonts.googleapis.com/css?family=Source+Code+Pro" rel="stylesheet"><script></script><link rel="stylesheet" href="reset.8c46b7a1.css"><link rel="icon" type="image/png" sizes="32x32" href="favicon-32x32.bc503437.png"><link rel="icon" type="image/png" sizes="16x16" href="favicon-16x16.f74b3187.png"></head><body> <script src="speedscope.80eb88d2.js"></script>
+<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta http-equiv="X-UA-Compatible" content="ie=edge"><title>speedscope</title><link href="source-code-pro.52b1676f.css" rel="stylesheet"><script></script><link rel="stylesheet" href="reset.8c46b7a1.css"><link rel="icon" type="image/png" sizes="32x32" href="favicon-32x32.bc503437.png"><link rel="icon" type="image/png" sizes="16x16" href="favicon-16x16.f74b3187.png"></head><body> <script src="speedscope.6f107512.js"></script>
 </body></html>

deps/speedscope/release.txt

@@ -1,3 +1,3 @@
-speedscope@1.20.0
-Fri Jan 12 09:57:49 PST 2024
-68fd88ceaf93d89aa27f3f0a20a27c9cfdc015c5
+speedscope@1.21.0
+Sat Nov 16 22:13:27 PST 2024
+d36c3a54424063a8df7bc67a7b824a223d73861b

deps/speedscope/source-code-pro.52b1676f.css

@@ -0,0 +1,2 @@
+@font-face{font-family:Source Code Pro;font-weight:400;font-style:normal;font-stretch:normal;src:url(SourceCodePro-Regular.ttf.f546cbe0.woff2) format("woff2")}
+/*# sourceMappingURL=source-code-pro.52b1676f.css.map */

deps/speedscope/source-code-pro.LICENSE.md

@@ -0,0 +1,93 @@
+Copyright 2010-2019 Adobe (http://www.adobe.com/), with Reserved Font Name 'Source'. All Rights Reserved. Source is a trademark of Adobe in the United States and/or other countries.
+
+This Font Software is licensed under the SIL Open Font License, Version 1.1.
+
+This license is copied below, and is also available with a FAQ at: http://scripts.sil.org/OFL
+
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide
+development of collaborative font projects, to support the font creation
+efforts of academic and linguistic communities, and to provide a free and
+open framework in which fonts may be shared and improved in partnership
+with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and
+redistributed freely as long as they are not sold by themselves. The
+fonts, including any derivative works, can be bundled, embedded,
+redistributed and/or sold with any software provided that any reserved
+names are not used by derivative works. The fonts and derivatives,
+however, cannot be released under any other type of license. The
+requirement for fonts to remain under this license does not apply
+to any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright
+Holder(s) under this license and clearly marked as such. This may
+include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the
+copyright statement(s).
+
+"Original Version" refers to the collection of Font Software components as
+distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to, deleting,
+or substituting -- in part or in whole -- any of the components of the
+Original Version, by changing formats or by porting the Font Software to a
+new environment.
+
+"Author" refers to any designer, engineer, programmer, technical
+writer or other person who contributed to the Font Software.
+
+PERMISSION & CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Font Software, to use, study, copy, merge, embed, modify,
+redistribute, and sell modified and unmodified copies of the Font
+Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components,
+in Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled,
+redistributed and/or sold with any software, provided that each copy
+contains the above copyright notice and this license. These can be
+included either as stand-alone text files, human-readable headers or
+in the appropriate machine-readable metadata fields within text or
+binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font
+Name(s) unless explicit written permission is granted by the corresponding
+Copyright Holder. This restriction only applies to the primary font name as
+presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
+Software shall not be used to promote, endorse or advertise any
+Modified Version, except to acknowledge the contribution(s) of the
+Copyright Holder(s) and the Author(s) or with their explicit written
+permission.
+
+5) The Font Software, modified or unmodified, in part or in whole,
+must be distributed entirely under this license, and must not be
+distributed under any other license. The requirement for fonts to
+remain under this license does not apply to any document created
+using the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are
+not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
+OTHER DEALINGS IN THE FONT SOFTWARE.

deps/sqlite/shell.c

@@ -460,7 +460,7 @@ char *sqlite3_fgets(char *buf, int sz, FILE *in){
     ** that into UTF-8.  Otherwise, non-ASCII characters all get translated
     ** into '?'.
     */
-    wchar_t *b1 = malloc( sz*sizeof(wchar_t) );
+    wchar_t *b1 = sqlite3_malloc( sz*sizeof(wchar_t) );
     if( b1==0 ) return 0;
     _setmode(_fileno(in), IsConsole(in) ? _O_WTEXT : _O_U8TEXT);
     if( fgetws(b1, sz/4, in)==0 ){
@@ -526,7 +526,7 @@ int sqlite3_fputs(const char *z, FILE *out){
     ** use O_U8TEXT for everything in text mode.
     */
     int sz = (int)strlen(z);
-    wchar_t *b1 = malloc( (sz+1)*sizeof(wchar_t) );
+    wchar_t *b1 = sqlite3_malloc( (sz+1)*sizeof(wchar_t) );
     if( b1==0 ) return 0;
     sz = MultiByteToWideChar(CP_UTF8, 0, z, sz, b1, sz);
     b1[sz] = 0;
@@ -5072,10 +5072,10 @@ int sqlite3_percentile_init(
 ){
   int rc = SQLITE_OK;
   unsigned int i;
-#if defined(SQLITE3_H) || defined(SQLITE_STATIC_PERCENTILE)
-  (void)pApi;      /* Unused parameter */
-#else
+#ifdef SQLITE3EXT_H
   SQLITE_EXTENSION_INIT2(pApi);
+#else
+  (void)pApi;      /* Unused parameter */
 #endif
   (void)pzErrMsg;  /* Unused parameter */
   for(i=0; i<sizeof(aPercentFunc)/sizeof(aPercentFunc[0]); i++){
@@ -6833,7 +6833,7 @@ static int seriesBestIndex(
       continue;
     }
     if( pConstraint->iColumn<SERIES_COLUMN_START ){
-      if( pConstraint->iColumn==SERIES_COLUMN_VALUE ){
+      if( pConstraint->iColumn==SERIES_COLUMN_VALUE && pConstraint->usable ){
         switch( op ){
           case SQLITE_INDEX_CONSTRAINT_EQ:
           case SQLITE_INDEX_CONSTRAINT_IS: {
@@ -6841,7 +6841,9 @@ static int seriesBestIndex(
             idxNum &= ~0x3300;
             aIdx[5] = i;
             aIdx[6] = -1;
+#ifndef ZERO_ARGUMENT_GENERATE_SERIES
             bStartSeen = 1;
+#endif
             break;
           }
           case SQLITE_INDEX_CONSTRAINT_GE: {
@@ -6849,7 +6851,9 @@ static int seriesBestIndex(
             idxNum |=  0x0100;
             idxNum &= ~0x0200;
             aIdx[5] = i;
+#ifndef ZERO_ARGUMENT_GENERATE_SERIES
             bStartSeen = 1;
+#endif
             break;
           }
           case SQLITE_INDEX_CONSTRAINT_GT: {
@@ -6857,7 +6861,9 @@ static int seriesBestIndex(
             idxNum |=  0x0200;
             idxNum &= ~0x0100;
             aIdx[5] = i;
+#ifndef ZERO_ARGUMENT_GENERATE_SERIES
             bStartSeen = 1;
+#endif
             break;
           }
           case SQLITE_INDEX_CONSTRAINT_LE: {
@@ -14169,7 +14175,7 @@ static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
     }else{
       IdxTable *pTab;
       rc = idxGetTableInfo(p->db, zName, &pTab, pzErrmsg);
-      if( rc==SQLITE_OK ){
+      if( rc==SQLITE_OK && ALWAYS(pTab!=0) ){
         int i;
         char *zInner = 0;
         char *zOuter = 0;
@@ -31840,7 +31846,6 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss,
   char cWait = (char)qss; /* intentional narrowing loss */
   if( cWait==0 ){
   PlainScan:
-    assert( cWait==0 );
     while( (cin = *zLine++)!=0 ){
       if( IsSpace(cin) )
         continue;
@@ -31892,7 +31897,6 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss,
           if( *zLine != '/' )
             continue;
           ++zLine;
-          cWait = 0;
           CONTINUE_PROMPT_AWAITC(pst, 0);
           qss = QSS_SETV(qss, 0);
           goto PlainScan;
@@ -31904,7 +31908,6 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss,
           }
           deliberate_fall_through;
         case ']':
-          cWait = 0;
           CONTINUE_PROMPT_AWAITC(pst, 0);
           qss = QSS_SETV(qss, 0);
           goto PlainScan;

deps/sqlite/sqlite3.c

@@ -1,6 +1,6 @@
 /******************************************************************************
 ** This file is an amalgamation of many separate C source files from SQLite
-** version 3.47.0.  By combining all the individual C code files into this
+** version 3.47.2.  By combining all the individual C code files into this
 ** single large file, the entire code can be compiled as a single translation
 ** unit.  This allows many compilers to do optimizations that would not be
 ** possible if the files were compiled separately.  Performance improvements
@@ -18,7 +18,7 @@
 ** separate file. This file contains only code for the core SQLite library.
 **
 ** The content in this amalgamation comes from Fossil check-in
-** 03a9703e27c44437c39363d0baf82db4ebc9.
+** 2aabe05e2e8cae4847a802ee2daddc1d7413.
 */
 #define SQLITE_CORE 1
 #define SQLITE_AMALGAMATION 1
@@ -462,9 +462,9 @@ extern "C" {
 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
 ** [sqlite_version()] and [sqlite_source_id()].
 */
-#define SQLITE_VERSION        "3.47.0"
-#define SQLITE_VERSION_NUMBER 3047000
-#define SQLITE_SOURCE_ID      "2024-10-21 16:30:22 03a9703e27c44437c39363d0baf82db4ebc94538a0f28411c85dda156f82636e"
+#define SQLITE_VERSION        "3.47.2"
+#define SQLITE_VERSION_NUMBER 3047002
+#define SQLITE_SOURCE_ID      "2024-12-07 20:39:59 2aabe05e2e8cae4847a802ee2daddc1d7413d8fc560254d93ee3e72c14685b6c"
 
 /*
 ** CAPI3REF: Run-Time Library Version Numbers
@@ -968,6 +968,13 @@ SQLITE_API int sqlite3_exec(
 ** filesystem supports doing multiple write operations atomically when those
 ** write operations are bracketed by [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] and
 ** [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE].
+**
+** The SQLITE_IOCAP_SUBPAGE_READ property means that it is ok to read
+** from the database file in amounts that are not a multiple of the
+** page size and that do not begin at a page boundary.  Without this
+** property, SQLite is careful to only do full-page reads and write
+** on aligned pages, with the one exception that it will do a sub-page
+** read of the first page to access the database header.
 */
 #define SQLITE_IOCAP_ATOMIC                 0x00000001
 #define SQLITE_IOCAP_ATOMIC512              0x00000002
@@ -984,6 +991,7 @@ SQLITE_API int sqlite3_exec(
 #define SQLITE_IOCAP_POWERSAFE_OVERWRITE    0x00001000
 #define SQLITE_IOCAP_IMMUTABLE              0x00002000
 #define SQLITE_IOCAP_BATCH_ATOMIC           0x00004000
+#define SQLITE_IOCAP_SUBPAGE_READ           0x00008000
 
 /*
 ** CAPI3REF: File Locking Levels
@@ -1130,6 +1138,7 @@ struct sqlite3_file {
 ** <li> [SQLITE_IOCAP_POWERSAFE_OVERWRITE]
 ** <li> [SQLITE_IOCAP_IMMUTABLE]
 ** <li> [SQLITE_IOCAP_BATCH_ATOMIC]
+** <li> [SQLITE_IOCAP_SUBPAGE_READ]
 ** </ul>
 **
 ** The SQLITE_IOCAP_ATOMIC property means that all writes of
@@ -32298,6 +32307,7 @@ SQLITE_PRIVATE void sqlite3RecordErrorOffsetOfExpr(sqlite3 *db, const Expr *pExp
     pExpr = pExpr->pLeft;
   }
   if( pExpr==0 ) return;
+  if( ExprHasProperty(pExpr, EP_FromDDL) ) return;
   db->errByteOffset = pExpr->w.iOfst;
 }
 
@@ -35687,8 +35697,8 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en
   int eValid = 1;  /* True exponent is either not used or is well-formed */
   int nDigit = 0;  /* Number of digits processed */
   int eType = 1;   /* 1: pure integer,  2+: fractional  -1 or less: bad UTF16 */
+  u64 s2;          /* round-tripped significand */
   double rr[2];
-  u64 s2;
 
   assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE );
   *pResult = 0.0;   /* Default return value, in case of an error */
@@ -35791,7 +35801,7 @@ do_atof_calc:
   e = (e*esign) + d;
 
   /* Try to adjust the exponent to make it smaller */
-  while( e>0 && s<(LARGEST_UINT64/10) ){
+  while( e>0 && s<((LARGEST_UINT64-0x7ff)/10) ){
     s *= 10;
     e--;
   }
@@ -35801,11 +35811,16 @@ do_atof_calc:
   }
 
   rr[0] = (double)s;
-  s2 = (u64)rr[0];
-#if defined(_MSC_VER) && _MSC_VER<1700
-  if( s2==0x8000000000000000LL ){ s2 = 2*(u64)(0.5*rr[0]); }
-#endif
-  rr[1] = s>=s2 ? (double)(s - s2) : -(double)(s2 - s);
+  assert( sizeof(s2)==sizeof(rr[0]) );
+  memcpy(&s2, &rr[0], sizeof(s2));
+  if( s2<=0x43efffffffffffffLL ){
+    s2 = (u64)rr[0];
+    rr[1] = s>=s2 ? (double)(s - s2) : -(double)(s2 - s);
+  }else{
+    rr[1] = 0.0;
+  }
+  assert( rr[1]<=1.0e-10*rr[0] );  /* Equal only when rr[0]==0.0 */
+
   if( e>0 ){
     while( e>=100  ){
       e -= 100;
@@ -42591,6 +42606,7 @@ static void setDeviceCharacteristics(unixFile *pFd){
     if( pFd->ctrlFlags & UNIXFILE_PSOW ){
       pFd->deviceCharacteristics |= SQLITE_IOCAP_POWERSAFE_OVERWRITE;
     }
+    pFd->deviceCharacteristics |= SQLITE_IOCAP_SUBPAGE_READ;
 
     pFd->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE;
   }
@@ -50391,7 +50407,7 @@ static int winSectorSize(sqlite3_file *id){
 */
 static int winDeviceCharacteristics(sqlite3_file *id){
   winFile *p = (winFile*)id;
-  return SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN |
+  return SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN | SQLITE_IOCAP_SUBPAGE_READ |
          ((p->ctrlFlags & WINFILE_PSOW)?SQLITE_IOCAP_POWERSAFE_OVERWRITE:0);
 }
 
@@ -51779,7 +51795,7 @@ static int winOpen(
 
   int rc = SQLITE_OK;            /* Function Return Code */
 #if !defined(NDEBUG) || SQLITE_OS_WINCE
-  int eType = flags&0xFFFFFF00;  /* Type of file to open */
+  int eType = flags&0x0FFF00;  /* Type of file to open */
 #endif
 
   int isExclusive  = (flags & SQLITE_OPEN_EXCLUSIVE);
@@ -57999,18 +58015,26 @@ static const unsigned char aJournalMagic[] = {
 ** Return true if page pgno can be read directly from the database file
 ** by the b-tree layer. This is the case if:
 **
-**   * the database file is open,
-**   * there are no dirty pages in the cache, and
-**   * the desired page is not currently in the wal file.
+**   (1)  the database file is open
+**   (2)  the VFS for the database is able to do unaligned sub-page reads
+**   (3)  there are no dirty pages in the cache, and
+**   (4)  the desired page is not currently in the wal file.
 */
 SQLITE_PRIVATE int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){
-  if( pPager->fd->pMethods==0 ) return 0;
-  if( sqlite3PCacheIsDirty(pPager->pPCache) ) return 0;
+  assert( pPager!=0 );
+  assert( pPager->fd!=0 );
+  if( pPager->fd->pMethods==0 ) return 0;  /* Case (1) */
+  assert( pPager->fd->pMethods->xDeviceCharacteristics!=0 );
+  if( (pPager->fd->pMethods->xDeviceCharacteristics(pPager->fd)
+        & SQLITE_IOCAP_SUBPAGE_READ)==0 ){
+    return 0; /* Case (2) */
+  }
+  if( sqlite3PCacheIsDirty(pPager->pPCache) ) return 0; /* Failed (3) */
 #ifndef SQLITE_OMIT_WAL
   if( pPager->pWal ){
     u32 iRead = 0;
     (void)sqlite3WalFindFrame(pPager->pWal, pgno, &iRead);
-    return iRead==0;
+    return iRead==0; /* Condition (4) */
   }
 #endif
   return 1;
@@ -147586,32 +147610,32 @@ static Expr *substExpr(
         if( pSubst->isOuterJoin ){
           ExprSetProperty(pNew, EP_CanBeNull);
         }
+        if( pNew->op==TK_TRUEFALSE ){
+          pNew->u.iValue = sqlite3ExprTruthValue(pNew);
+          pNew->op = TK_INTEGER;
+          ExprSetProperty(pNew, EP_IntValue);
+        }
+
+        /* Ensure that the expression now has an implicit collation sequence,
+        ** just as it did when it was a column of a view or sub-query. */
+        {
+          CollSeq *pNat = sqlite3ExprCollSeq(pSubst->pParse, pNew);
+          CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse,
+                pSubst->pCList->a[iColumn].pExpr
+          );
+          if( pNat!=pColl || (pNew->op!=TK_COLUMN && pNew->op!=TK_COLLATE) ){
+            pNew = sqlite3ExprAddCollateString(pSubst->pParse, pNew,
+                (pColl ? pColl->zName : "BINARY")
+            );
+          }
+        }
+        ExprClearProperty(pNew, EP_Collate);
         if( ExprHasProperty(pExpr,EP_OuterON|EP_InnerON) ){
           sqlite3SetJoinExpr(pNew, pExpr->w.iJoin,
                              pExpr->flags & (EP_OuterON|EP_InnerON));
         }
         sqlite3ExprDelete(db, pExpr);
         pExpr = pNew;
-        if( pExpr->op==TK_TRUEFALSE ){
-          pExpr->u.iValue = sqlite3ExprTruthValue(pExpr);
-          pExpr->op = TK_INTEGER;
-          ExprSetProperty(pExpr, EP_IntValue);
-        }
-
-        /* Ensure that the expression now has an implicit collation sequence,
-        ** just as it did when it was a column of a view or sub-query. */
-        {
-          CollSeq *pNat = sqlite3ExprCollSeq(pSubst->pParse, pExpr);
-          CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse,
-                pSubst->pCList->a[iColumn].pExpr
-          );
-          if( pNat!=pColl || (pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE) ){
-            pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr,
-                (pColl ? pColl->zName : "BINARY")
-            );
-          }
-        }
-        ExprClearProperty(pExpr, EP_Collate);
       }
     }
   }else{
@@ -158939,6 +158963,7 @@ static Expr *removeUnindexableInClauseTerms(
         pNew->pLeft->x.pList = pLhs;
       }
       pSelect->pEList = pRhs;
+      pSelect->selId = ++pParse->nSelect; /* Req'd for SubrtnSig validity */
       if( pLhs && pLhs->nExpr==1 ){
         /* Take care here not to generate a TK_VECTOR containing only a
         ** single value. Since the parser never creates such a vector, some
@@ -189798,10 +189823,15 @@ static int fts3PoslistPhraseMerge(
   if( *p1==POS_COLUMN ){
     p1++;
     p1 += fts3GetVarint32(p1, &iCol1);
+    /* iCol1==0 indicates corruption. Column 0 does not have a POS_COLUMN
+    ** entry, so this is actually end-of-doclist. */
+    if( iCol1==0 ) return 0;
   }
   if( *p2==POS_COLUMN ){
     p2++;
     p2 += fts3GetVarint32(p2, &iCol2);
+    /* As above, iCol2==0 indicates corruption. */
+    if( iCol2==0 ) return 0;
   }
 
   while( 1 ){
@@ -192972,7 +193002,7 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){
       nTmp += p->pRight->pPhrase->doclist.nList;
     }
     nTmp += p->pPhrase->doclist.nList;
-    aTmp = sqlite3_malloc64(nTmp*2);
+    aTmp = sqlite3_malloc64(nTmp*2 + FTS3_VARINT_MAX);
     if( !aTmp ){
       *pRc = SQLITE_NOMEM;
       res = 0;
@@ -194525,10 +194555,11 @@ static int getNextString(
         Fts3PhraseToken *pToken;
 
         p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken));
-        if( !p ) goto no_mem;
-
         zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte);
-        if( !zTemp ) goto no_mem;
+        if( !zTemp || !p ){
+          rc = SQLITE_NOMEM;
+          goto getnextstring_out;
+        }
 
         assert( nToken==ii );
         pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii];
@@ -194543,9 +194574,6 @@ static int getNextString(
         nToken = ii+1;
       }
     }
-
-    pModule->xClose(pCursor);
-    pCursor = 0;
   }
 
   if( rc==SQLITE_DONE ){
@@ -194553,7 +194581,10 @@ static int getNextString(
     char *zBuf = 0;
 
     p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp);
-    if( !p ) goto no_mem;
+    if( !p ){
+      rc = SQLITE_NOMEM;
+      goto getnextstring_out;
+    }
     memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p);
     p->eType = FTSQUERY_PHRASE;
     p->pPhrase = (Fts3Phrase *)&p[1];
@@ -194561,11 +194592,9 @@ static int getNextString(
     p->pPhrase->nToken = nToken;
 
     zBuf = (char *)&p->pPhrase->aToken[nToken];
+    assert( nTemp==0 || zTemp );
     if( zTemp ){
       memcpy(zBuf, zTemp, nTemp);
-      sqlite3_free(zTemp);
-    }else{
-      assert( nTemp==0 );
     }
 
     for(jj=0; jj<p->pPhrase->nToken; jj++){
@@ -194575,17 +194604,17 @@ static int getNextString(
     rc = SQLITE_OK;
   }
 
-  *ppExpr = p;
-  return rc;
-no_mem:
-
+ getnextstring_out:
   if( pCursor ){
     pModule->xClose(pCursor);
   }
   sqlite3_free(zTemp);
-  sqlite3_free(p);
-  *ppExpr = 0;
-  return SQLITE_NOMEM;
+  if( rc!=SQLITE_OK ){
+    sqlite3_free(p);
+    p = 0;
+  }
+  *ppExpr = p;
+  return rc;
 }
 
 /*
@@ -232806,7 +232835,27 @@ SQLITE_API int sqlite3session_config(int op, void *pArg){
 /************** End of sqlite3session.c **************************************/
 /************** Begin file fts5.c ********************************************/
 
-
+/*
+** This, the "fts5.c" source file, is a composite file that is itself
+** assembled from the following files:
+**
+**    fts5.h
+**    fts5Int.h
+**    fts5parse.h          <--- Generated from fts5parse.y by Lemon
+**    fts5parse.c          <--- Generated from fts5parse.y by Lemon
+**    fts5_aux.c
+**    fts5_buffer.c
+**    fts5_config.c
+**    fts5_expr.c
+**    fts5_hash.c
+**    fts5_index.c
+**    fts5_main.c
+**    fts5_storage.c
+**    fts5_tokenize.c
+**    fts5_unicode2.c
+**    fts5_varint.c
+**    fts5_vocab.c
+*/
 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5)
 
 #if !defined(NDEBUG) && !defined(SQLITE_DEBUG)
@@ -232816,6 +232865,12 @@ SQLITE_API int sqlite3session_config(int op, void *pArg){
 # undef NDEBUG
 #endif
 
+#ifdef HAVE_STDINT_H
+/* #include <stdint.h> */
+#endif
+#ifdef HAVE_INTTYPES_H
+/* #include <inttypes.h> */
+#endif
 /*
 ** 2014 May 31
 **
@@ -254888,7 +254943,7 @@ static void fts5SourceIdFunc(
 ){
   assert( nArg==0 );
   UNUSED_PARAM2(nArg, apUnused);
-  sqlite3_result_text(pCtx, "fts5: 2024-10-21 16:30:22 03a9703e27c44437c39363d0baf82db4ebc94538a0f28411c85dda156f82636e", -1, SQLITE_TRANSIENT);
+  sqlite3_result_text(pCtx, "fts5: 2024-12-07 20:39:59 2aabe05e2e8cae4847a802ee2daddc1d7413d8fc560254d93ee3e72c14685b6c", -1, SQLITE_TRANSIENT);
 }
 
 /*
@@ -260079,7 +260134,7 @@ static int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){
 }
 
 
-
+/* Here ends the fts5.c composite file. */
 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) */
 
 /************** End of fts5.c ************************************************/

deps/sqlite/sqlite3.h

@@ -146,9 +146,9 @@ extern "C" {
 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
 ** [sqlite_version()] and [sqlite_source_id()].
 */
-#define SQLITE_VERSION        "3.47.0"
-#define SQLITE_VERSION_NUMBER 3047000
-#define SQLITE_SOURCE_ID      "2024-10-21 16:30:22 03a9703e27c44437c39363d0baf82db4ebc94538a0f28411c85dda156f82636e"
+#define SQLITE_VERSION        "3.47.2"
+#define SQLITE_VERSION_NUMBER 3047002
+#define SQLITE_SOURCE_ID      "2024-12-07 20:39:59 2aabe05e2e8cae4847a802ee2daddc1d7413d8fc560254d93ee3e72c14685b6c"
 
 /*
 ** CAPI3REF: Run-Time Library Version Numbers
@@ -652,6 +652,13 @@ SQLITE_API int sqlite3_exec(
 ** filesystem supports doing multiple write operations atomically when those
 ** write operations are bracketed by [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] and
 ** [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE].
+**
+** The SQLITE_IOCAP_SUBPAGE_READ property means that it is ok to read
+** from the database file in amounts that are not a multiple of the
+** page size and that do not begin at a page boundary.  Without this
+** property, SQLite is careful to only do full-page reads and write
+** on aligned pages, with the one exception that it will do a sub-page
+** read of the first page to access the database header.
 */
 #define SQLITE_IOCAP_ATOMIC                 0x00000001
 #define SQLITE_IOCAP_ATOMIC512              0x00000002
@@ -668,6 +675,7 @@ SQLITE_API int sqlite3_exec(
 #define SQLITE_IOCAP_POWERSAFE_OVERWRITE    0x00001000
 #define SQLITE_IOCAP_IMMUTABLE              0x00002000
 #define SQLITE_IOCAP_BATCH_ATOMIC           0x00004000
+#define SQLITE_IOCAP_SUBPAGE_READ           0x00008000
 
 /*
 ** CAPI3REF: File Locking Levels
@@ -814,6 +822,7 @@ struct sqlite3_file {
 ** <li> [SQLITE_IOCAP_POWERSAFE_OVERWRITE]
 ** <li> [SQLITE_IOCAP_IMMUTABLE]
 ** <li> [SQLITE_IOCAP_BATCH_ATOMIC]
+** <li> [SQLITE_IOCAP_SUBPAGE_READ]
 ** </ul>
 **
 ** The SQLITE_IOCAP_ATOMIC property means that all writes of

docs/cheatsheet.md

@@ -1,63 +0,0 @@
-# Tilde Friends Cheat Sheet
-
-Making apps for the impatient tilde friend.
-
-## Prerequisites
-
-- either run your own instance or use [tildefriends.net](https://www.tildefriends.net/)
-- register and login
-- [optional] use the `ssb` app to create yourself an SSB identity
-
-## Development Process
-
-1.  hit the `edit` link from any app or new app URL
-2.  make sure the path in the text box is under your username: `/~username/app/`
-3.  write server-side code in `app.js`
-4.  click the `save` button or press the save hotkey (Alt+S or _[browser-specific modifiers]_+S)
-5.  see the app reload on the right side
-
-## Output
-
-- **`app.setDocument(html)`** - send HTML to the browser
-- **`print(...)`** - send values to the browser's developer console
-
-## Persistence
-
-- **`app.localStorageGet(key)`** -> **`value`**
-- **`app.localStorageSet(key, value)`**
-- **`database()`**, **`shared_database(key)`**, **`my_shared_database(package, key)`**
-  - **`db.get(key)`** -> **`value`**
-  - **`db.set(key, value)`**
-  - **`db.exchange(key, expected, value)`** -> **`exchanged`**
-  - **`db.remove(key)`**
-  - **`db.getAll()`** -> **`[key1, ...]`**
-  - **`db.getLike(pattern)`** -> **`{key1: value1, ...}`**
-
-## SSB
-
-- **`ssb.createIdentity()`** -> **`id`**
-- **`ssb.getIdentities()`** -> **`[id1, ...]`**
-- **`ssb.appendMessageWithIdentity(id, content)`** -> **`message_id`**
-- **`ssb.blobStore(blob)`** -> **`blob_id`**
-- **`ssb.blobGet(id)`** -> **`blob`**
-- **`ssb.sqlAsync(query, args, row_callback)`**
-
-## TF-RPC
-
-Stock helper code for calling functions across the web server and browser boundary.
-
-- on the server: `import * as tfrpc from '/tfrpc.js';`
-- in the browser: `import * as tfrpc from '/static/tfrpc.js';`
-- either direction:
-  - register a function: `tfrpc.register(function my_function() {});`
-  - call a remote function: `let promise = tfrpc.rpc.my_function();`
-
-## Share
-
-- give out web links: [https://www.tildefriends.net/~cory/screwble/](https://www.tildefriends.net/~cory/screwble/)
-- use the `Attach App` button when composing a post in [the SSB app](https://www.tildefriends.net/~core/ssb/)
-
-## More Docs
-
-- [api reference](https://www.tildefriends.net/~cory/api/)
-- [source code](https://dev.tildefriends.net/cory/tildefriends/releases)

docs/guide.md

@@ -1,166 +0,0 @@
-# Tilde Friends Developer's Guide
-
-A Tilde Friends application starts with code that runs on a Tilde Friends server, possibly far away from where you wrote it, in a little JavaScript environment, in its own restricted process, with the only access to the outside world being the ability to send messages to the server. This document gives some recipes showing how that can be used to build a functional user-facing application in light of the unique constraints present.
-
-## Example 1: Hello, world!
-
-Of course we must start with a classic.
-
-### app.js
-
-```
-app.setDocument('<h1 style="color: #fff">Hello, world!</h1>');
-```
-
-### Output
-
-<iframe srcdoc="&lt;h1 style=&quot;color: #fff&quot;&gt;Hello, world!&lt;/h1&gt;"></iframe>
-
-### Explanation
-
-At a glance, this might seem mundane, but for it to work:
-
-- the server starts a real process for your app and loads your code into it
-- your code runs
-- `app.setDocument()` sends a message back to the server
-- the server interprets the message and redirects it to the browser
-- `core/client.js` in the browser receives the message and puts your HTML into an iframe
-- your HTML is presented by the browser in an iframe sandbox
-
-But you don't have to think about all that. Call a function, and you see the result.
-
-## Example 2: Hit Counter
-
-Let's take advantage of code running on the server and create a little hit counter using a key value store shared between all visitors.
-
-### app.js
-
-```
-async function main() {
-	let db = await shared_database('visitors');
-	let count = parseInt((await db.get('visitors')) ?? '0') + 1;
-	await db.set('visitors', count.toString());
-	await app.setDocument(`
-		<h1 style="color: #fff">Welcome, visitor #${count}!</h1>
-	`);
-}
-
-main();
-```
-
-### Output
-
-<iframe srcdoc="&lt;h1 style=&quot;color: #fff&quot;&gt;Welcome, visitor #1!&lt;/h1&gt;"></iframe>
-
-### Explanation
-
-Just as pure browser apps have access to `localStorage`, Tilde Friends apps have access to key-value storage on the server.
-
-The interface is a bit clunky and will likely change someday, but this example gets a database object, from which you can get and set string values by key. There are various on `shared_database` that let you store data that is private to the user or shared by different criteria.
-
-Also, even though any browser-side code is sandboxed, it is allowed to access browser local storage by going through Tilde Friends API, because sometimes that is useful.
-
-## Example 3: Files
-
-Suppose you don't want to create your entire app in a single server-side file as we've done with the previous examples. There are some tools to allow you to begin to organize.
-
-### app.js
-
-```
-async function main() {
-	let html = utf8Decode(await getFile('index.html'));
-	app.setDocument(html);
-}
-
-main();
-```
-
-### index.html
-
-```
-<html>
-	<head>
-		<script type="module" src="script.js"></script>
-	</head>
-	<body style="color: #fff">
-		<h1>File Test</h1>
-	</body>
-</html>
-```
-
-### script.js
-
-```
-window.addEventListener('load', function() {
-	document.body.appendChild(document.createTextNode('Hello, world');
-});
-```
-
-### Output
-
-<iframe srcdoc="&lt;body style=&quot;color: #fff&quot;&gt;<h1>File Test</h1>Hello, world!&lt;/body&gt;"></iframe>
-
-### Explanation
-
-On the server, `utf8Decode(await getFile(fileName))` lets you load a file from your app. In the browser, your app files are made available by HTTP, so you can `<script src="my_script.js"></script>` and such to access them.
-
-## Example 4: Remote Procedure Call
-
-While making calls between the client and the server, it is possible to pass functions across that boundary. `tfrpc.js` is a tiny script which builds on that feature to try to hide some of the complexities.
-
-### app.js
-
-```
-import * as tf from '/tfrpc.js';
-
-function sum() {
-	let s = 0
-	for (let x of arguments) {
-		s += x;
-	}
-	return s;
-}
-tf.register(sum);
-
-async function main() {
-	app.setDocument(utf8Decode(await getFile('index.html')));
-}
-main();
-```
-
-### index.html
-
-```
-<html>
-	<body>
-		<h1 id='result'>Calculating...</h1>
-	</body>
-	<script type="module" src="script.js"></script>
-</html>
-```
-
-### script.js
-
-```
-import * as tf from '/static/tfrpc.js';
-
-window.addEventListener('load', async function() {
-	document.getElementById('result').innerText = await tf.rpc.sum(1, 2, 3);
-});
-```
-
-### Output
-
-<iframe srcdoc="&lt;body style=&quot;color: #fff&quot;&gt;<h1>6</h1>&lt;/body&gt;"></iframe>
-
-### Explanation
-
-Here the browser makes an asynchronous call to the server to do some basic math and update its DOM with the result.
-
-With your favorite Vue/Lit/React/... library on the client-side and your favorite Tilde Friends API calls registered with tfrpc, it becomes pretty easy to start extracting interesting information from, say, SQL queries over Secure Scuttlebutt data, and generating complicated, dynamic user interface. These are the building blocks I used to make the current Tilde Friends SSB client interface.
-
-## Conclusion
-
-Tilde Friends is currently a pile of all the parts that I thought I needed to build interesting web applications, tied together by code that tries to walk the fine line between being secure enough to let us safely run code on the same device and being usable enough that you can open a tab in your browser and start building just by typing code.
-
-I don't claim it thoroughly accomplishes either yet, but I believe it is at a stage where it is showing how promising this approach can be, and I am excited for you to take it for a spin and share.

docs/release.md

@@ -1,19 +0,0 @@
-# Release Checklist
-
-- make sure ci is passing
-- run the tests
-- format + prettier
-- update metadata/en-US/changelogs
-- git tag
-- push
-- make dist
-- make a release on gitea
-- upload the artifacts
-- nix
-  - comment out the hash in default.nix
-  - update the version
-  - run `nix build`
-  - update the hash
-- bump the versions in GNUmakefile for the next release
-- make
-- commit

docs/vision.md

@@ -1,64 +0,0 @@
-# Tilde Friends Vision
-
-Tilde Friends is a tool for making and sharing.
-
-It is both a peer-to-peer social network client, participating in Secure
-Scuttlebutt, and an environment for creating and running web applications.
-
-## Why
-
-This is a thing that I wanted to exist and wanted to work on. No other reason.
-There is not a business model. I believe it is interesting and unique.
-
-## Goals
-
-1. Make it **easy and fun** to run all sorts of web applications.
-
-2. Provide **security** that is easy to understand and protects your data.
-
-3. Make **creating and sharing** web applications accessible to anyone with a
-   browser.
-
-## Ways to Use Tilde Friends
-
-1. **Social Network User**: This is a social network first. You are just here,
-   because your friends are. Or you like how we limit your message length or
-   short videos or whatever the trend is. If you are ambitious, you click links
-   and see interactive experiences (apps) that you wouldn't see elsewhere.
-
-2. **Web Visitor**: You get links from a friend to meeting invites, polls, games,
-   lists, wiki pages, ..., and you interact with them as though they were
-   cloud-hosted by a megacorporation. They just work, and you don't think twice.
-
-3. **Group leader**: You host or use a small public instance, installing apps for
-   a group of friends to use as web visitors.
-
-4. **Developer**: You like to write code and make or improve apps for fun or to
-   solve problems. When you encounter a Tilde Friends app on a strange server,
-   you know you can trivially modify it or download it to your own instance.
-
-## Future Goals / Endgame
-
-1. Mobile apps. This can run on your old phone. Maybe you won't be hosting
-   the web interface publicly, but you can sync, install and edit apps, and
-   otherwise get the full experience from a tiny touch screen.
-
-2. The universal application runtime. The web browser is the universal
-   platform, but even for the simplest application that you might want to host
-   for your friends, cloud hosting, containers, and complicated dependencies might
-   all enter the mix. Tilde Friends, though it is yet another thing to host,
-   includes everything you need out of the box to run a vast variety of interesting
-   apps.
-
-   Tilde Friends will be built out, gradually providing safe access to host
-   resources and client resources the same way web browsers extended access to
-   resources like GPU, persistent storage, cameras, ... over the years.
-
-   Not much effort has been put forward yet to having a robust, long-lasting API,
-   but since the client side longevity is already handled by web browsers, it
-   seems possible that the server-side API can be managed in a similar way.
-
-3. An awesome development environment. Right now it runs JavaScript from the
-   first embeddable text editor I could poorly configure enough to edit code,
-   but it could incorporate a debugger, source control integration a la ssb-git,
-   merge tools, and transpiling from all sorts of different languages.

metadata/en-US/changelogs/30.txt

@@ -0,0 +1,14 @@
+* Improve, test, and fix some first time flow issues.
+* Show connect failure reasons.
+* Actually use CommonMark JS in "safe" mode(!).
+* Move app blob handling from JS to C.
+* Fixed "publish" command-line interface and used it to add a script that posts development activity updates.
+* Minor style improvements.
+* Make it possible to see the little graphs on mobile.
+* Shutdown fixes.
+* Updated dependencies:
+  * CodeMirror
+  * c-ares 1.34.3
+  * libbacktrace
+  * speedscope 1.21.0
+  * sqlite 3.47.1

metadata/en-US/changelogs/31.txt

@@ -0,0 +1,15 @@
+* Add channels to the ssb user interface.
+* Confirm setting global settings.
+* Fix the wiki app.
+* Multiple crash fixes, especially on Android.
+* Following depth fixes.
+* Try harder to request profile images.
+* Updated the emoji list used for reactions.
+* Admin users can encrypt/decrypt using the server identity.
+* Request more messages as the follow graph changes.
+* Add a store_blob cli option.
+* Updated dependencies:
+  * CodeMirror
+  * Prettier
+  * sqlite 3.47.2
+  * c-ares 1.34.4

package-lock.json

@@ -11,9 +11,9 @@
 			}
 		},
 		"node_modules/prettier": {
-			"version": "3.3.3",
-			"resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.3.tgz",
-			"integrity": "sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==",
+			"version": "3.4.2",
+			"resolved": "https://registry.npmjs.org/prettier/-/prettier-3.4.2.tgz",
+			"integrity": "sha512-e9MewbtFo+Fevyuxn/4rrcDAaq0IYxPGLvObpQjiZBMAzB9IGmzlnG9RZy3FFas+eBMu2vA0CszMeduow5dIuQ==",
 			"license": "MIT",
 			"bin": {
 				"prettier": "bin/prettier.cjs"

src/android/AndroidManifest.xml

@@ -1,8 +1,8 @@
 <?xml version="1.0" encoding="utf-8"?>
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
 	package="com.unprompted.tildefriends"
-	android:versionCode="29"
-	android:versionName="0.0.24">
+	android:versionCode="31"
+	android:versionName="0.0.26">
 	<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE"/>
 	<uses-permission android:name="android.permission.INTERNET"/>
 	<application

src/android/com/unprompted/tildefriends/TildeFriendsActivity.java

@@ -385,8 +385,11 @@ public class TildeFriendsActivity extends Activity {
 			public void onServiceConnected(ComponentName name, IBinder binder) {
 				Log.w("tildefriends", "onServiceConnected");
 				Parcel data = Parcel.obtain();
-				ParcelFileDescriptor pfd = ParcelFileDescriptor.adoptFd(pipe_fd);
-				data.writeParcelable(pfd, 0);
+				try (ParcelFileDescriptor pfd = ParcelFileDescriptor.fromFd(pipe_fd)) {
+					data.writeParcelable(pfd, 0);
+				} catch (java.io.IOException e) {
+					Log.w("tildefriends", "IOException: " + e);
+				}
 				try {
 					binder.transact(TildeFriendsSandboxService.START_CALL,  data, null, IBinder.FLAG_ONEWAY);
 				} catch (RemoteException e) {

src/http.c

@@ -67,7 +67,6 @@ typedef struct _tf_http_connection_t
 typedef struct _tf_http_handler_t
 {
 	const char* pattern;
-	bool is_wildcard;
 	tf_http_callback_t* callback;
 	tf_http_cleanup_t* cleanup;
 	void* user_data;
@@ -129,9 +128,15 @@ static void _http_allocate_buffer(uv_handle_t* handle, size_t suggested_size, uv
 	*buf = uv_buf_init(connection->incoming, sizeof(connection->incoming));
 }
 
-static bool _http_pattern_matches(const char* pattern, const char* path, bool is_wildcard)
+bool tf_http_pattern_matches(const char* pattern, const char* path)
 {
-	if (!pattern || !*pattern || (!is_wildcard && strcmp(path, pattern) == 0))
+	if (!*pattern && !*path)
+	{
+		return true;
+	}
+	const char* k_word = "{word}";
+	bool is_wildcard = strchr(pattern, '*') || strstr(pattern, k_word);
+	if (!is_wildcard && strcmp(path, pattern) == 0)
 	{
 		return true;
 	}
@@ -140,17 +145,36 @@ static bool _http_pattern_matches(const char* pattern, const char* path, bool is
 	{
 		int i = 0;
 		int j = 0;
-		while (pattern[i] && path[j] && pattern[i] != '*' && pattern[i] == path[j])
+		while (pattern[i] && path[j] && pattern[i] == path[j])
 		{
 			i++;
 			j++;
 		}
 
-		if (pattern[i] == '*')
+		size_t k_word_len = strlen(k_word);
+		if (strncmp(pattern + i, k_word, k_word_len) == 0 && ((path[j] >= 'a' && path[j] <= 'z') || (path[j] >= 'A' && path[j] <= 'Z')))
 		{
 			while (true)
 			{
-				if (_http_pattern_matches(pattern + i + 1, path + j, strchr(pattern + i + 1, '*') != NULL))
+				if ((path[j] >= 'a' && path[j] <= 'z') || (path[j] >= 'A' && path[j] <= 'Z') || (path[j] >= '0' && path[j] <= '9'))
+				{
+					if (tf_http_pattern_matches(pattern + i + k_word_len, path + j + 1))
+					{
+						return true;
+					}
+				}
+				else
+				{
+					break;
+				}
+				j++;
+			}
+		}
+		else if (pattern[i] == '*')
+		{
+			while (true)
+			{
+				if (tf_http_pattern_matches(pattern + i + 1, path + j))
 				{
 					return true;
 				}
@@ -170,7 +194,7 @@ static bool _http_find_handler(tf_http_t* http, const char* path, tf_http_callba
 {
 	for (int i = 0; i < http->handlers_count; i++)
 	{
-		if (_http_pattern_matches(http->handlers[i].pattern, path, http->handlers[i].is_wildcard))
+		if (tf_http_pattern_matches(http->handlers[i].pattern, path))
 		{
 			*out_callback = http->handlers[i].callback;
 			*out_trace_name = http->handlers[i].pattern;
@@ -741,7 +765,6 @@ void tf_http_add_handler(tf_http_t* http, const char* pattern, tf_http_callback_
 	http->handlers = tf_resize_vec(http->handlers, sizeof(tf_http_handler_t) * (http->handlers_count + 1));
 	http->handlers[http->handlers_count++] = (tf_http_handler_t) {
 		.pattern = tf_strdup(pattern),
-		.is_wildcard = pattern && strchr(pattern, '*') != NULL,
 		.callback = callback,
 		.cleanup = cleanup,
 		.user_data = user_data,

src/http.h

@@ -228,4 +228,12 @@ void tf_http_request_websocket_upgrade(tf_http_request_t* request);
 */
 const char* tf_http_status_text(int status);
 
+/**
+** Match URL patterns.  "*" matches anything, and "{word}" matches [a-zA-Z][a-zA-Z0-9]*".
+** @param pattern The pattern to match.
+** @param path The path to test.
+** @return true if the path matches the pattern.
+*/
+bool tf_http_pattern_matches(const char* pattern, const char* path);
+
 /** @} */

src/httpd.js.c

@@ -10,6 +10,7 @@
 #include "tlscontext.js.h"
 #include "trace.h"
 #include "util.js.h"
+#include "version.h"
 
 #include "ow-crypt.h"
 
@@ -29,10 +30,9 @@
 #include <alloca.h>
 #endif
 
-#define tf_countof(a) ((int)(sizeof((a)) / sizeof(*(a))))
-
 #define CYAN "\e[1;36m"
 #define MAGENTA "\e[1;35m"
+#define YELLOW "\e[1;33m"
 #define RESET "\e[0m"
 
 const int64_t k_refresh_interval = 1ULL * 7 * 24 * 60 * 60 * 1000;
@@ -222,6 +222,17 @@ static void _httpd_message_callback(tf_http_request_t* request, int op_code, con
 	JS_FreeValue(context, on_message);
 }
 
+static JSValue _httpd_make_response_object(JSContext* context, tf_http_request_t* request)
+{
+	JSValue response_object = JS_NewObjectClass(context, _httpd_request_class_id);
+	JS_SetOpaque(response_object, request);
+	JS_SetPropertyStr(context, response_object, "writeHead", JS_NewCFunction(context, _httpd_response_write_head, "writeHead", 2));
+	JS_SetPropertyStr(context, response_object, "end", JS_NewCFunction(context, _httpd_response_end, "end", 1));
+	JS_SetPropertyStr(context, response_object, "send", JS_NewCFunction(context, _httpd_response_send, "send", 2));
+	JS_SetPropertyStr(context, response_object, "upgrade", JS_NewCFunction(context, _httpd_websocket_upgrade, "upgrade", 2));
+	return response_object;
+}
+
 static void _httpd_callback_internal(tf_http_request_t* request, bool is_websocket)
 {
 	http_handler_data_t* data = request->user_data;
@@ -248,14 +259,9 @@ static void _httpd_callback_internal(tf_http_request_t* request, bool is_websock
 	JS_SetPropertyStr(context, client, "tls", request->is_tls ? JS_TRUE : JS_FALSE);
 	JS_SetPropertyStr(context, request_object, "client", client);
 
-	JSValue response_object = JS_NewObjectClass(context, _httpd_request_class_id);
+	JSValue response_object = _httpd_make_response_object(context, request);
 	/* The ref is owned by the JS object and will be released by the finalizer. */
 	tf_http_request_ref(request);
-	JS_SetOpaque(response_object, request);
-	JS_SetPropertyStr(context, response_object, "writeHead", JS_NewCFunction(context, _httpd_response_write_head, "writeHead", 2));
-	JS_SetPropertyStr(context, response_object, "end", JS_NewCFunction(context, _httpd_response_end, "end", 1));
-	JS_SetPropertyStr(context, response_object, "send", JS_NewCFunction(context, _httpd_response_send, "send", 2));
-	JS_SetPropertyStr(context, response_object, "upgrade", JS_NewCFunction(context, _httpd_websocket_upgrade, "upgrade", 2));
 	JSValue args[] = {
 		request_object,
 		response_object,
@@ -423,7 +429,7 @@ static JSValue _httpd_endpoint_start(JSContext* context, JSValueConst this_val,
 	*listener = (httpd_listener_t) { .context = context, .tls = JS_DupValue(context, argv[1]) };
 	tf_tls_context_t* tls = tf_tls_context_get(listener->tls);
 	int assigned_port = tf_http_listen(http, port, tls, _httpd_listener_cleanup, listener);
-	tf_printf(CYAN "~😎 Tilde Friends" RESET " is now up at " MAGENTA "http%s://127.0.0.1:%d/" RESET ".\n", tls ? "s" : "", assigned_port);
+	tf_printf(CYAN "~😎 Tilde Friends" RESET " " YELLOW VERSION_NUMBER RESET " is now up at " MAGENTA "http%s://127.0.0.1:%d/" RESET ".\n", tls ? "s" : "", assigned_port);
 	return JS_NewInt32(context, assigned_port);
 }
 
@@ -558,7 +564,7 @@ static bool _magic_bytes_match(const magic_bytes_t* magic, const uint8_t* actual
 	return true;
 }
 
-static const char* _httpd_mime_type_from_magic_bytes_internal(const uint8_t* bytes, size_t size)
+static const char* _httpd_mime_type_from_magic_bytes(const uint8_t* bytes, size_t size)
 {
 	const char* type = "application/binary";
 	if (bytes)
@@ -635,13 +641,6 @@ static const char* _httpd_mime_type_from_magic_bytes_internal(const uint8_t* byt
 	return type;
 }
 
-static JSValue _httpd_mime_type_from_magic_bytes(JSContext* context, JSValueConst this_val, int argc, JSValueConst* argv)
-{
-	size_t size = 0;
-	uint8_t* bytes = tf_util_try_get_array_buffer(context, &size, argv[0]);
-	return JS_NewString(context, _httpd_mime_type_from_magic_bytes_internal(bytes, size));
-}
-
 static const char* _ext_to_content_type(const char* ext, bool use_fallback)
 {
 	if (ext)
@@ -674,14 +673,6 @@ static const char* _ext_to_content_type(const char* ext, bool use_fallback)
 	return use_fallback ? "application/binary" : NULL;
 }
 
-static JSValue _httpd_mime_type_from_extension(JSContext* context, JSValueConst this_val, int argc, JSValueConst* argv)
-{
-	const char* name = JS_ToCString(context, argv[0]);
-	const char* type = _ext_to_content_type(strrchr(name, '.'), false);
-	JS_FreeCString(context, name);
-	return type ? JS_NewString(context, type) : JS_UNDEFINED;
-}
-
 static void _httpd_finalizer(JSRuntime* runtime, JSValue value)
 {
 	tf_http_t* http = JS_GetOpaque(value, _httpd_class_id);
@@ -966,6 +957,22 @@ static void _httpd_endpoint_static(tf_http_request_t* request)
 	tf_file_stat(task, path, _httpd_endpoint_static_stat, request);
 }
 
+static void _httpd_endpoint_add_slash(tf_http_request_t* request)
+{
+	const char* host = tf_http_request_get_header(request, "x-forwarded-host");
+	if (!host)
+	{
+		host = tf_http_request_get_header(request, "host");
+	}
+	char url[1024];
+	snprintf(url, sizeof(url), "%s%s%s/", request->is_tls ? "https://" : "http://", host, request->path);
+	const char* headers[] = {
+		"Location",
+		url,
+	};
+	tf_http_respond(request, 303, headers, tf_countof(headers) / 2, "", 0);
+}
+
 typedef struct _user_app_t
 {
 	const char* user;
@@ -1016,12 +1023,204 @@ static user_app_t* _parse_user_app_from_path(const char* path, const char* expec
 	return result;
 }
 
+typedef struct _app_blob_t
+{
+	tf_http_request_t* request;
+	bool found;
+	bool not_modified;
+	bool use_handler;
+	void* data;
+	size_t size;
+	char app_blob_id[k_blob_id_len];
+	const char* file;
+	user_app_t* user_app;
+	char etag[256];
+} app_blob_t;
+
+static void _httpd_endpoint_app_blob_work(tf_ssb_t* ssb, void* user_data)
+{
+	app_blob_t* data = user_data;
+	tf_http_request_t* request = data->request;
+	if (request->path[0] == '/' && request->path[1] == '~')
+	{
+		const char* last_slash = strchr(request->path + 1, '/');
+		if (last_slash)
+		{
+			last_slash = strchr(last_slash + 1, '/');
+		}
+		data->user_app = last_slash ? _parse_user_app_from_path(request->path, last_slash) : NULL;
+		if (data->user_app)
+		{
+			size_t path_length = strlen("path:") + strlen(data->user_app->app) + 1;
+			char* app_path = tf_malloc(path_length);
+			snprintf(app_path, path_length, "path:%s", data->user_app->app);
+			const char* value = tf_ssb_db_get_property(ssb, data->user_app->user, app_path);
+			snprintf(data->app_blob_id, sizeof(data->app_blob_id), "%s", value);
+			tf_free(app_path);
+			tf_free((void*)value);
+			data->file = last_slash + 1;
+		}
+	}
+	else if (request->path[0] == '/' && request->path[1] == '&')
+	{
+		const char* end = strstr(request->path, ".sha256/");
+		if (end)
+		{
+			snprintf(data->app_blob_id, sizeof(data->app_blob_id), "%.*s", (int)(end + strlen(".sha256") - request->path - 1), request->path + 1);
+			data->file = end + strlen(".sha256/");
+		}
+	}
+
+	char* app_blob = NULL;
+	size_t app_blob_size = 0;
+	if (*data->app_blob_id && tf_ssb_db_blob_get(ssb, data->app_blob_id, (uint8_t**)&app_blob, &app_blob_size))
+	{
+		JSMallocFunctions funcs = { 0 };
+		tf_get_js_malloc_functions(&funcs);
+		JSRuntime* runtime = JS_NewRuntime2(&funcs, NULL);
+		JSContext* context = JS_NewContext(runtime);
+
+		JSValue app_object = JS_ParseJSON(context, app_blob, app_blob_size, NULL);
+		JSValue files = JS_GetPropertyStr(context, app_object, "files");
+		JSValue blob_id = JS_GetPropertyStr(context, files, data->file);
+		if (JS_IsUndefined(blob_id))
+		{
+			blob_id = JS_GetPropertyStr(context, files, "handler.js");
+			if (!JS_IsUndefined(blob_id))
+			{
+				data->use_handler = true;
+			}
+		}
+		else
+		{
+			const char* blob_id_str = JS_ToCString(context, blob_id);
+			if (blob_id_str)
+			{
+				snprintf(data->etag, sizeof(data->etag), "\"%s\"", blob_id_str);
+				const char* match = tf_http_request_get_header(data->request, "if-none-match");
+				if (match && strcmp(match, data->etag) == 0)
+				{
+					data->not_modified = true;
+				}
+				else
+				{
+					data->found = tf_ssb_db_blob_get(ssb, blob_id_str, (uint8_t**)&data->data, &data->size);
+				}
+			}
+			JS_FreeCString(context, blob_id_str);
+		}
+		JS_FreeValue(context, blob_id);
+		JS_FreeValue(context, files);
+		JS_FreeValue(context, app_object);
+
+		JS_FreeContext(context);
+		JS_FreeRuntime(runtime);
+		tf_free(app_blob);
+	}
+}
+
+static void _httpd_call_app_handler(tf_ssb_t* ssb, tf_http_request_t* request, const char* app_blob_id, const char* path, const char* package_owner, const char* app)
+{
+	JSContext* context = tf_ssb_get_context(ssb);
+	JSValue global = JS_GetGlobalObject(context);
+	JSValue exports = JS_GetPropertyStr(context, global, "exports");
+	JSValue call_app_handler = JS_GetPropertyStr(context, exports, "callAppHandler");
+
+	JSValue response = _httpd_make_response_object(context, request);
+	tf_http_request_ref(request);
+	JSValue handler_blob_id = JS_NewString(context, app_blob_id);
+	JSValue path_value = JS_NewString(context, path);
+	JSValue package_owner_value = JS_NewString(context, package_owner);
+	JSValue app_value = JS_NewString(context, app);
+	JSValue query_value = request->query ? JS_NewString(context, request->query) : JS_UNDEFINED;
+
+	JSValue headers = JS_NewObject(context);
+	for (int i = 0; i < request->headers_count; i++)
+	{
+		char name[256] = "";
+		snprintf(name, sizeof(name), "%.*s", (int)request->headers[i].name_len, request->headers[i].name);
+		JS_SetPropertyStr(context, headers, name, JS_NewStringLen(context, request->headers[i].value, request->headers[i].value_len));
+	}
+
+	JSValue args[] = {
+		response,
+		handler_blob_id,
+		path_value,
+		query_value,
+		headers,
+		package_owner_value,
+		app_value,
+	};
+
+	JSValue result = JS_Call(context, call_app_handler, JS_NULL, tf_countof(args), args);
+	tf_util_report_error(context, result);
+	JS_FreeValue(context, result);
+
+	JS_FreeValue(context, headers);
+	JS_FreeValue(context, query_value);
+	JS_FreeValue(context, app_value);
+	JS_FreeValue(context, package_owner_value);
+	JS_FreeValue(context, handler_blob_id);
+	JS_FreeValue(context, path_value);
+	JS_FreeValue(context, response);
+	JS_FreeValue(context, call_app_handler);
+	JS_FreeValue(context, exports);
+	JS_FreeValue(context, global);
+}
+
+static void _httpd_endpoint_app_blob_after_work(tf_ssb_t* ssb, int status, void* user_data)
+{
+	app_blob_t* data = user_data;
+	if (data->not_modified)
+	{
+		tf_http_respond(data->request, 304, NULL, 0, NULL, 0);
+	}
+	else if (data->use_handler)
+	{
+		_httpd_call_app_handler(ssb, data->request, data->app_blob_id, data->file, data->user_app->user, data->user_app->app);
+	}
+	else if (data->found)
+	{
+		const char* mime_type = _ext_to_content_type(strrchr(data->request->path, '.'), false);
+		if (!mime_type)
+		{
+			mime_type = _httpd_mime_type_from_magic_bytes(data->data, data->size);
+		}
+		const char* headers[] = {
+			"Access-Control-Allow-Origin",
+			"*",
+			"Content-Security-Policy",
+			"sandbox allow-downloads allow-top-navigation-by-user-activation",
+			"Content-Type",
+			mime_type ? mime_type : "application/binary",
+			"etag",
+			data->etag,
+		};
+		tf_http_respond(data->request, 200, headers, tf_countof(headers) / 2, data->data, data->size);
+	}
+	tf_free(data->user_app);
+	tf_free(data->data);
+	tf_http_request_unref(data->request);
+	tf_free(data);
+}
+
+static void _httpd_endpoint_app_blob(tf_http_request_t* request)
+{
+	tf_http_request_ref(request);
+	tf_task_t* task = request->user_data;
+	tf_ssb_t* ssb = tf_task_get_ssb(task);
+	app_blob_t* data = tf_malloc(sizeof(app_blob_t));
+	*data = (app_blob_t) { .request = request };
+	tf_ssb_run_work(ssb, _httpd_endpoint_app_blob_work, _httpd_endpoint_app_blob_after_work, data);
+}
+
 typedef struct _view_t
 {
 	tf_http_request_t* request;
 	const char** form_data;
 	void* data;
 	size_t size;
+	char etag[256];
 	bool not_modified;
 } view_t;
 
@@ -1045,7 +1244,7 @@ static void _httpd_endpoint_view_work(tf_ssb_t* ssb, void* user_data)
 {
 	view_t* view = user_data;
 	tf_http_request_t* request = view->request;
-	char blob_id[256] = "";
+	char blob_id[k_blob_id_len] = "";
 
 	user_app_t* user_app = _parse_user_app_from_path(request->path, "/view");
 	if (user_app)
@@ -1066,10 +1265,11 @@ static void _httpd_endpoint_view_work(tf_ssb_t* ssb, void* user_data)
 
 	if (*blob_id)
 	{
+		snprintf(view->etag, sizeof(view->etag), "\"%s\"", blob_id);
 		const char* if_none_match = tf_http_request_get_header(request, "if-none-match");
 		char match[258];
 		snprintf(match, sizeof(match), "\"%s\"", blob_id);
-		if (if_none_match && strcmp(if_none_match, match))
+		if (if_none_match && strcmp(if_none_match, match) == 0)
 		{
 			view->not_modified = true;
 		}
@@ -1097,7 +1297,9 @@ static void _httpd_endpoint_view_after_work(tf_ssb_t* ssb, int status, void* use
 		"Content-Security-Policy",
 		"sandbox allow-downloads allow-top-navigation-by-user-activation",
 		"Content-Type",
-		view->data ? _httpd_mime_type_from_magic_bytes_internal(view->data, view->size) : "text/plain",
+		view->data ? _httpd_mime_type_from_magic_bytes(view->data, view->size) : "text/plain",
+		"etag",
+		view->etag,
 		filename ? "Content-Disposition" : NULL,
 		filename ? content_disposition : NULL,
 	};
@@ -1135,7 +1337,7 @@ typedef struct _save_t
 {
 	tf_http_request_t* request;
 	int response;
-	char blob_id[256];
+	char blob_id[k_blob_id_len];
 } save_t;
 
 static void _httpd_endpoint_save_work(tf_ssb_t* ssb, void* user_data)
@@ -1158,7 +1360,7 @@ static void _httpd_endpoint_save_work(tf_ssb_t* ssb, void* user_data)
 		user_app_t* user_app = _parse_user_app_from_path(request->path, "/save");
 		if (user_app)
 		{
-			if (strcmp(user_string, user_app->user) == 0 || (strcmp(user_app->user, "core") == 0 && tf_ssb_db_user_has_permission(ssb, user_string, "administration")))
+			if (strcmp(user_string, user_app->user) == 0 || (strcmp(user_app->user, "core") == 0 && tf_ssb_db_user_has_permission(ssb, NULL, user_string, "administration")))
 			{
 				size_t path_length = strlen("path:") + strlen(user_app->app) + 1;
 				char* app_path = tf_malloc(path_length);
@@ -1212,12 +1414,12 @@ static void _httpd_endpoint_save_work(tf_ssb_t* ssb, void* user_data)
 						size_t new_app_length = 0;
 						const char* new_app_str = JS_ToCStringLen(context, &new_app_length, new_app_json);
 
-						char blob_id[250] = { 0 };
+						char blob_id[k_blob_id_len] = { 0 };
 						if (tf_ssb_db_blob_store(ssb, (const uint8_t*)new_app_str, new_app_length, blob_id, sizeof(blob_id), NULL) &&
 							tf_ssb_db_set_property(ssb, user_app->user, app_path, blob_id))
 						{
 							tf_ssb_db_add_value_to_array_property(ssb, user_app->user, "apps", user_app->app);
-							snprintf(save->blob_id, sizeof(save->blob_id), "/%s", blob_id);
+							snprintf(save->blob_id, sizeof(save->blob_id), "%s", blob_id);
 							save->response = 200;
 						}
 
@@ -1242,10 +1444,10 @@ static void _httpd_endpoint_save_work(tf_ssb_t* ssb, void* user_data)
 		}
 		else if (strcmp(request->path, "/save") == 0)
 		{
-			char blob_id[250] = { 0 };
+			char blob_id[k_blob_id_len] = { 0 };
 			if (tf_ssb_db_blob_store(ssb, request->body, request->content_length, blob_id, sizeof(blob_id), NULL))
 			{
-				snprintf(save->blob_id, sizeof(save->blob_id), "/%s", blob_id);
+				snprintf(save->blob_id, sizeof(save->blob_id), "%s", blob_id);
 				save->response = 200;
 			}
 		}
@@ -1269,7 +1471,9 @@ static void _httpd_endpoint_save_after_work(tf_ssb_t* ssb, int status, void* use
 	tf_http_request_t* request = save->request;
 	if (*save->blob_id)
 	{
-		tf_http_respond(request, 200, NULL, 0, save->blob_id, strlen(save->blob_id));
+		char body[256] = "";
+		int length = snprintf(body, sizeof(body), "/%s", save->blob_id);
+		tf_http_respond(request, 200, NULL, 0, body, length);
 	}
 	tf_http_request_unref(request);
 	tf_free(save);
@@ -1312,7 +1516,7 @@ static void _httpd_endpoint_delete_work(tf_ssb_t* ssb, void* user_data)
 		user_app_t* user_app = _parse_user_app_from_path(request->path, "/delete");
 		if (user_app)
 		{
-			if (strcmp(user_string, user_app->user) == 0 || (strcmp(user_app->user, "core") == 0 && tf_ssb_db_user_has_permission(ssb, user_string, "administration")))
+			if (strcmp(user_string, user_app->user) == 0 || (strcmp(user_app->user, "core") == 0 && tf_ssb_db_user_has_permission(ssb, NULL, user_string, "administration")))
 			{
 				size_t path_length = strlen("path:") + strlen(user_app->app) + 1;
 				char* app_path = tf_malloc(path_length);
@@ -2099,11 +2303,16 @@ void tf_httpd_register(JSContext* context)
 	tf_http_add_handler(http, "/speedscope/*", _httpd_endpoint_static, NULL, task);
 	tf_http_add_handler(http, "/static/*", _httpd_endpoint_static, NULL, task);
 	tf_http_add_handler(http, "/.well-known/*", _httpd_endpoint_static, NULL, task);
-	tf_http_add_handler(http, "/~*/*/", _httpd_endpoint_static, NULL, task);
+	tf_http_add_handler(http, "/&*.sha256", _httpd_endpoint_add_slash, NULL, task);
 	tf_http_add_handler(http, "/&*.sha256/", _httpd_endpoint_static, NULL, task);
-	tf_http_add_handler(http, "/*/view", _httpd_endpoint_view, NULL, task);
-	tf_http_add_handler(http, "/~*/*/save", _httpd_endpoint_save, NULL, task);
-	tf_http_add_handler(http, "/~*/*/delete", _httpd_endpoint_delete, NULL, task);
+	tf_http_add_handler(http, "/&*.sha256/view", _httpd_endpoint_view, NULL, task);
+	tf_http_add_handler(http, "/&*.sha256/*", _httpd_endpoint_app_blob, NULL, task);
+	tf_http_add_handler(http, "/~{word}/{word}", _httpd_endpoint_add_slash, NULL, task);
+	tf_http_add_handler(http, "/~{word}/{word}/", _httpd_endpoint_static, NULL, task);
+	tf_http_add_handler(http, "/~{word}/{word}/save", _httpd_endpoint_save, NULL, task);
+	tf_http_add_handler(http, "/~{word}/{word}/delete", _httpd_endpoint_delete, NULL, task);
+	tf_http_add_handler(http, "/~{word}/{word}/view", _httpd_endpoint_view, NULL, task);
+	tf_http_add_handler(http, "/~{word}/{word}/*", _httpd_endpoint_app_blob, NULL, task);
 	tf_http_add_handler(http, "/save", _httpd_endpoint_save, NULL, task);
 
 	tf_http_add_handler(http, "/robots.txt", _httpd_endpoint_robots_txt, NULL, NULL);
@@ -2116,13 +2325,10 @@ void tf_httpd_register(JSContext* context)
 	tf_http_add_handler(http, "/login/logout", _httpd_endpoint_logout, NULL, task);
 	tf_http_add_handler(http, "/login", _httpd_endpoint_login, NULL, task);
 
-	JS_SetPropertyStr(context, httpd, "handlers", JS_NewObject(context));
 	JS_SetPropertyStr(context, httpd, "all", JS_NewCFunction(context, _httpd_endpoint_all, "all", 2));
 	JS_SetPropertyStr(context, httpd, "start", JS_NewCFunction(context, _httpd_endpoint_start, "start", 2));
 	JS_SetPropertyStr(context, httpd, "set_http_redirect", JS_NewCFunction(context, _httpd_set_http_redirect, "set_http_redirect", 1));
 	JS_SetPropertyStr(context, httpd, "auth_query", JS_NewCFunction(context, _httpd_auth_query, "auth_query", 1));
-	JS_SetPropertyStr(context, httpd, "mime_type_from_magic_bytes", JS_NewCFunction(context, _httpd_mime_type_from_magic_bytes, "mime_type_from_magic_bytes", 1));
-	JS_SetPropertyStr(context, httpd, "mime_type_from_extension", JS_NewCFunction(context, _httpd_mime_type_from_extension, "mime_type_from_extension", 1));
 	JS_SetPropertyStr(context, global, "httpd", httpd);
 	JS_FreeValue(context, global);
 }

src/main.c

@@ -39,10 +39,6 @@
 #include "jni.h"
 #endif
 
-#if !defined(_countof)
-#define _countof(a) ((int)(sizeof((a)) / sizeof(*(a))))
-#endif
-
 struct backtrace_state* g_backtrace_state;
 
 const char* k_db_path_default = "db.sqlite";
@@ -53,6 +49,7 @@ static int _tf_command_import(const char* file, int argc, char* argv[]);
 static int _tf_command_publish(const char* file, int argc, char* argv[]);
 static int _tf_command_run(const char* file, int argc, char* argv[]);
 static int _tf_command_sandbox(const char* file, int argc, char* argv[]);
+static int _tf_command_store_blob(const char* file, int argc, char* argv[]);
 static int _tf_command_test(const char* file, int argc, char* argv[]);
 static int _tf_command_verify(const char* file, int argc, char* argv[]);
 static int _tf_command_usage(const char* file);
@@ -70,6 +67,7 @@ const command_t k_commands[] = {
 	{ "import", _tf_command_import, "Import apps to SSB." },
 	{ "export", _tf_command_export, "Export apps from SSB." },
 	{ "publish", _tf_command_publish, "Append a message to a feed." },
+	{ "store_blob", _tf_command_store_blob, "Write a file to the blob store." },
 	{ "verify", _tf_command_verify, "Verify a feed." },
 	{ "test", _tf_command_test, "Test SSB." },
 };
@@ -264,7 +262,7 @@ static int _tf_command_export(const char* file, int argc, char* argv[])
 			"ssb",
 			"todo",
 		};
-		for (int i = 0; i < (int)_countof(k_export); i++)
+		for (int i = 0; i < tf_countof(k_export); i++)
 		{
 			char buffer[256];
 			snprintf(buffer, sizeof(buffer), "/~%s/%s", user, k_export[i]);
@@ -346,8 +344,8 @@ static int _tf_command_publish(const char* file, int argc, char* argv[])
 	{
 		tf_printf("\n%s publish [options]\n\n", file);
 		tf_printf("options:\n");
-		tf_printf("  -y, --user user          User owning identity with which to publish.\n");
-		tf_printf("  -i, --identity identity  Identity with which to publish message.\n");
+		tf_printf("  -u, --user user          User owning identity with which to publish.\n");
+		tf_printf("  -i, --id identity        Identity with which to publish message.\n");
 		tf_printf("  -d, --db-path db_path    SQLite database path (default: %s).\n", k_db_path_default);
 		tf_printf("  -c, --content json       JSON content of message to publish.\n");
 		tf_printf("  -h, --help               Show this usage information.\n");
@@ -391,6 +389,96 @@ static int _tf_command_publish(const char* file, int argc, char* argv[])
 	return result;
 }
 
+static int _tf_command_store_blob(const char* file, int argc, char* argv[])
+{
+	const char* db_path = k_db_path_default;
+	const char* file_path = NULL;
+	bool show_usage = false;
+
+	while (!show_usage)
+	{
+		static const struct option k_options[] = {
+			{ "db-path", required_argument, NULL, 'd' },
+			{ "file", required_argument, NULL, 'f' },
+			{ "help", no_argument, NULL, 'h' },
+			{ 0 },
+		};
+		int c = getopt_long(argc, argv, "d:f:h", k_options, NULL);
+		if (c == -1)
+		{
+			break;
+		}
+
+		switch (c)
+		{
+		case '?':
+		case 'h':
+		default:
+			show_usage = true;
+			break;
+		case 'd':
+			db_path = optarg;
+			break;
+		case 'f':
+			file_path = optarg;
+			break;
+		}
+	}
+
+	if (show_usage || !file_path)
+	{
+		tf_printf("\n%s store_blob [options]\n\n", file);
+		tf_printf("options:\n");
+		tf_printf("  -d, --db-path db_path    SQLite database path (default: %s).\n", k_db_path_default);
+		tf_printf("  -f, --file file_path     Path to file to add to the blob store.\n");
+		tf_printf("  -h, --help               Show this usage information.\n");
+		return EXIT_FAILURE;
+	}
+
+	char* data = NULL;
+	size_t size = 0;
+	FILE* blob_file = fopen(file_path, "rb");
+	if (!blob_file)
+	{
+		tf_printf("Failed to open %s: %s.\n", file_path, strerror(errno));
+		return EXIT_FAILURE;
+	}
+
+	char buffer[16 * 1024];
+	while (true)
+	{
+		size_t bytes = fread(buffer, 1, sizeof(buffer), blob_file);
+		if (bytes > 0)
+		{
+			data = tf_resize_vec(data, size + bytes);
+			memcpy(data + size, buffer, bytes);
+			size += bytes;
+		}
+		else
+		{
+			break;
+		}
+	}
+	if (ferror(blob_file))
+	{
+		tf_printf("Failed to read %s: %s.\n", file_path, strerror(errno));
+		fclose(blob_file);
+		tf_free(data);
+		return EXIT_FAILURE;
+	}
+	fclose(blob_file);
+
+	char id[256];
+	tf_ssb_t* ssb = tf_ssb_create(NULL, NULL, db_path, NULL);
+	if (tf_ssb_db_blob_store(ssb, (const uint8_t*)data, size, id, sizeof(id), NULL))
+	{
+		tf_printf("%s\n", id);
+	}
+	tf_ssb_destroy(ssb);
+	tf_free(data);
+	return EXIT_SUCCESS;
+}
+
 static int _tf_command_verify(const char* file, int argc, char* argv[])
 {
 	const char* identity = NULL;
@@ -778,7 +866,7 @@ static int _tf_command_usage(const char* file)
 {
 	tf_printf("Usage: %s command [command-options]\n", file);
 	tf_printf("commands:\n");
-	for (int i = 0; i < (int)_countof(k_commands); i++)
+	for (int i = 0; i < tf_countof(k_commands); i++)
 	{
 		tf_printf("  %s - %s\n", k_commands[i].name, k_commands[i].description);
 	}
@@ -908,7 +996,7 @@ static jint _tf_server_main(JNIEnv* env, jobject this_object, jstring files_dir,
 	};
 
 	tf_task_set_android_service_callbacks(_tf_service_start, _tf_service_stop);
-	result = _tf_command_run(apk, _countof(args), (char**)args);
+	result = _tf_command_run(apk, tf_countof(args), (char**)args);
 	tf_task_set_android_service_callbacks(NULL, NULL);
 
 	(*env)->ReleaseStringUTFChars(env, files_dir, files);
@@ -939,7 +1027,7 @@ static jint _tf_sandbox_main(JNIEnv* env, jobject this_object, int pipe_fd)
 		fd,
 	};
 
-	int result = _tf_command_sandbox(NULL, _countof(args), (char**)args);
+	int result = _tf_command_sandbox(NULL, tf_countof(args), (char**)args);
 
 	tf_mem_shutdown();
 	tf_printf("tf_sandbox_main finished with %d.", result);
@@ -971,7 +1059,7 @@ JNIEXPORT jint JNI_OnLoad(JavaVM* vm, void* reserved)
 		{ "tf_server_main", "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Landroid/net/ConnectivityManager;)I", _tf_server_main },
 		{ "tf_sandbox_main", "(I)I", _tf_sandbox_main },
 	};
-	int result = (*env)->RegisterNatives(env, c, methods, (int)_countof(methods));
+	int result = (*env)->RegisterNatives(env, c, methods, tf_countof(methods));
 	if (result != JNI_OK)
 	{
 		return result;
@@ -1018,7 +1106,7 @@ int main(int argc, char* argv[])
 	int result = 0;
 	if (argc >= 2)
 	{
-		for (int i = 0; i < (int)_countof(k_commands); i++)
+		for (int i = 0; i < tf_countof(k_commands); i++)
 		{
 			const command_t* command = &k_commands[i];
 			if (strcmp(argv[1], command->name) == 0)

src/packetstream.c

@@ -12,6 +12,8 @@ typedef struct _tf_packetstream_t
 {
 	tf_packetstream_onreceive_t* onreceive;
 	void* onreceive_user_data;
+	tf_packetstream_on_close_t* on_close;
+	void* on_close_user_data;
 	uv_pipe_t stream;
 	char* buffer;
 	size_t buffer_size;
@@ -30,6 +32,8 @@ void tf_packetstream_destroy(tf_packetstream_t* stream)
 {
 	stream->onreceive = NULL;
 	stream->onreceive_user_data = NULL;
+	stream->on_close = NULL;
+	stream->on_close_user_data = NULL;
 	stream->destroyed = true;
 	if (stream->buffer)
 	{
@@ -110,6 +114,14 @@ static void _packetstream_on_read(uv_stream_t* handle, ssize_t count, const uv_b
 	}
 	else
 	{
+		tf_packetstream_on_close_t* on_close = stream->on_close;
+		void* user_data = stream->on_close_user_data;
+		stream->on_close = NULL;
+		stream->on_close_user_data = NULL;
+		if (on_close)
+		{
+			on_close(user_data);
+		}
 		tf_packetstream_close(stream);
 	}
 	tf_free(buffer->base);
@@ -150,7 +162,7 @@ void tf_packetstream_send(tf_packetstream_t* stream, int packet_type, const char
 		int result = uv_write(request, (uv_stream_t*)&stream->stream, &write_buffer, 1, _packetstream_on_write);
 		if (result)
 		{
-			tf_printf("uv_write: %s\n", uv_strerror(result));
+			tf_printf("tf_packetstream_send: uv_write: %s\n", uv_strerror(result));
 			tf_free(request);
 		}
 	}
@@ -162,6 +174,12 @@ void tf_packetstream_set_on_receive(tf_packetstream_t* stream, tf_packetstream_o
 	stream->onreceive_user_data = user_data;
 }
 
+void tf_packetstream_set_on_close(tf_packetstream_t* stream, tf_packetstream_on_close_t* callback, void* user_data)
+{
+	stream->on_close = callback;
+	stream->on_close_user_data = user_data;
+}
+
 static void _tf_packetstream_handle_closed(uv_handle_t* handle)
 {
 	tf_packetstream_t* packetstream = handle->data;

src/packetstream.h

@@ -23,6 +23,12 @@ typedef struct _tf_packetstream_t tf_packetstream_t;
 */
 typedef void(tf_packetstream_onreceive_t)(int packet_type, const char* begin, size_t length, void* user_data);
 
+/**
+** A function called when a packetstream reads EOF.
+** @param user_data User data.
+*/
+typedef void(tf_packetstream_on_close_t)(void* user_data);
+
 /**
 ** Create a packet stream.
 ** @return The packet stream.
@@ -58,6 +64,14 @@ void tf_packetstream_send(tf_packetstream_t* stream, int packet_type, const char
 */
 void tf_packetstream_set_on_receive(tf_packetstream_t* stream, tf_packetstream_onreceive_t* callback, void* user_data);
 
+/**
+** Register a callback for when a stream reads EOF.
+** @param stream The packet stream.
+** @param callback The callback.
+** @param user_data User data to pass to the callback.
+*/
+void tf_packetstream_set_on_close(tf_packetstream_t* stream, tf_packetstream_on_close_t* callback, void* user_data);
+
 /**
 ** Close a packet stream.
 ** @param stream The packet stream.

src/ssb.c

@@ -31,10 +31,6 @@
 #include <string.h>
 #include <time.h>
 
-#if !defined(_countof)
-#define _countof(a) ((int)(sizeof((a)) / sizeof(*(a))))
-#endif
-
 #define GREEN "\e[1;32m"
 #define MAGENTA "\e[1;35m"
 #define CYAN "\e[1;36m"
@@ -224,6 +220,7 @@ typedef struct _tf_ssb_t
 	bool verbose;
 	bool store_debug_messages;
 	bool shutting_down;
+	bool shutting_down_deferred;
 
 	int messages_stored;
 	int blobs_stored;
@@ -297,6 +294,7 @@ typedef struct _tf_ssb_connection_t
 	uv_tcp_t tcp;
 	uv_connect_t connect;
 	uv_async_t async;
+	uv_async_t scheduled_async;
 	uv_timer_t handshake_timer;
 	bool closing;
 
@@ -367,6 +365,9 @@ typedef struct _tf_ssb_connection_t
 	uint64_t last_notified_active;
 
 	int flags;
+
+	tf_ssb_connect_callback_t* connect_callback;
+	void* connect_callback_user_data;
 } tf_ssb_connection_t;
 
 static JSClassID _connection_class_id;
@@ -385,6 +386,34 @@ static bool _tf_ssb_parse_broadcast(const char* in_broadcast, tf_ssb_broadcast_t
 static void _tf_ssb_start_update_settings(tf_ssb_t* ssb);
 static void _tf_ssb_update_settings(tf_ssb_t* ssb);
 static void _tf_ssb_write(tf_ssb_connection_t* connection, void* data, size_t size);
+static void _tf_ssb_connection_dispatch_scheduled(tf_ssb_connection_t* connection);
+
+static const char* _tf_ssb_connection_state_to_string(tf_ssb_state_t state)
+{
+	switch (state)
+	{
+	case k_tf_ssb_state_invalid:
+		return "invalid";
+	case k_tf_ssb_state_connected:
+		return "connected";
+	case k_tf_ssb_state_sent_hello:
+		return "sent hello";
+	case k_tf_ssb_state_sent_identity:
+		return "sent identity";
+	case k_tf_ssb_state_verified:
+		return "verified";
+	case k_tf_ssb_state_server_wait_hello:
+		return "server wait hello";
+	case k_tf_ssb_state_server_wait_client_identity:
+		return "server wait client identity";
+	case k_tf_ssb_state_server_verified:
+		return "server verified";
+	case k_tf_ssb_state_closing:
+		return "closing";
+	default:
+		return "unknown";
+	}
+}
 
 static void _tf_ssb_add_debug_close(tf_ssb_t* ssb, tf_ssb_connection_t* connection, const char* reason)
 {
@@ -508,7 +537,9 @@ static void _tf_ssb_write(tf_ssb_connection_t* connection, void* data, size_t si
 		if (result)
 		{
 			tf_ssb_connection_adjust_write_count(connection, -1);
-			_tf_ssb_connection_close(connection, "write failed");
+			char buffer[256];
+			snprintf(buffer, sizeof(buffer), "write failed : %s", uv_strerror(result));
+			_tf_ssb_connection_close(connection, buffer);
 			tf_free(write);
 		}
 	}
@@ -634,9 +665,15 @@ static void _tf_ssb_connection_box_stream_send(tf_ssb_connection_t* connection,
 	}
 }
 
+static void _tf_ssb_connection_scheduled_async(uv_async_t* async)
+{
+	tf_ssb_connection_t* connection = async->data;
+	_tf_ssb_connection_dispatch_scheduled(connection);
+}
+
 static void _tf_ssb_connection_dispatch_scheduled(tf_ssb_connection_t* connection)
 {
-	while ((connection->active_write_count == 0 || connection->closing) && connection->scheduled_count && connection->scheduled)
+	while (((connection->active_write_count == 0 && connection->read_back_pressure == 0) || connection->closing) && connection->scheduled_count && connection->scheduled)
 	{
 		tf_ssb_connection_scheduled_t scheduled = connection->scheduled[0];
 		memmove(connection->scheduled, connection->scheduled + 1, sizeof(tf_ssb_connection_scheduled_t) * (connection->scheduled_count - 1));
@@ -654,7 +691,7 @@ void tf_ssb_connection_schedule_idle(tf_ssb_connection_t* connection, tf_ssb_sch
 		.callback = callback,
 		.user_data = user_data,
 	};
-	_tf_ssb_connection_dispatch_scheduled(connection);
+	uv_async_send(&connection->scheduled_async);
 }
 
 static int _request_compare(const void* a, const void* b)
@@ -1321,6 +1358,12 @@ static void _tf_ssb_connection_verify_identity(tf_ssb_connection_t* connection,
 	JS_SetPropertyStr(context, connection->object, "is_client", JS_TRUE);
 
 	connection->state = k_tf_ssb_state_verified;
+	if (connection->connect_callback)
+	{
+		connection->connect_callback(connection, NULL, connection->connect_callback_user_data);
+		connection->connect_callback = NULL;
+		connection->connect_callback_user_data = NULL;
+	}
 	if (connection->handshake_timer.data)
 	{
 		uv_timer_stop(&connection->handshake_timer);
@@ -1639,7 +1682,7 @@ static void _tf_ssb_connection_rpc_recv(tf_ssb_connection_t* connection, uint8_t
 				if (callback)
 				{
 					char buffer[64];
-					snprintf(buffer, sizeof(buffer), "request %d", request_number);
+					snprintf(buffer, sizeof(buffer), "request %s:%d", request_name, request_number);
 					tf_trace_begin(connection->ssb->trace, buffer);
 					PRE_CALLBACK(connection->ssb, callback);
 					callback(connection, flags, request_number, val, message, size, user_data);
@@ -1702,7 +1745,7 @@ static void _tf_ssb_connection_rpc_recv(tf_ssb_connection_t* connection, uint8_t
 			if (callback)
 			{
 				char buffer[64];
-				snprintf(buffer, sizeof(buffer), "request %d", request_number);
+				snprintf(buffer, sizeof(buffer), "request %s:%d", request_name, request_number);
 				tf_trace_begin(connection->ssb->trace, buffer);
 				PRE_CALLBACK(connection->ssb, callback);
 				callback(connection, flags, request_number, JS_UNDEFINED, message, size, user_data);
@@ -1878,6 +1921,12 @@ static void _tf_ssb_connection_destroy(tf_ssb_connection_t* connection, const ch
 {
 	tf_ssb_t* ssb = connection->ssb;
 	connection->closing = true;
+	if (connection->connect_callback)
+	{
+		connection->connect_callback(NULL, reason, connection->connect_callback_user_data);
+		connection->connect_callback = NULL;
+		connection->connect_callback_user_data = NULL;
+	}
 	if (!connection->destroy_reason)
 	{
 		connection->destroy_reason = reason;
@@ -1947,6 +1996,10 @@ static void _tf_ssb_connection_destroy(tf_ssb_connection_t* connection, const ch
 	{
 		uv_close((uv_handle_t*)&connection->async, _tf_ssb_connection_on_close);
 	}
+	if (connection->scheduled_async.data && !uv_is_closing((uv_handle_t*)&connection->scheduled_async))
+	{
+		uv_close((uv_handle_t*)&connection->scheduled_async, _tf_ssb_connection_on_close);
+	}
 	if (connection->tcp.data && !uv_is_closing((uv_handle_t*)&connection->tcp))
 	{
 		uv_close((uv_handle_t*)&connection->tcp, _tf_ssb_connection_on_close);
@@ -1956,8 +2009,8 @@ static void _tf_ssb_connection_destroy(tf_ssb_connection_t* connection, const ch
 		uv_close((uv_handle_t*)&connection->handshake_timer, _tf_ssb_connection_on_close);
 	}
 
-	if (JS_IsUndefined(connection->object) && !connection->async.data && !connection->tcp.data && !connection->connect.data && !connection->handshake_timer.data &&
-		connection->ref_count == 0)
+	if (JS_IsUndefined(connection->object) && !connection->async.data && !connection->scheduled_async.data && !connection->tcp.data && !connection->connect.data &&
+		!connection->handshake_timer.data && connection->ref_count == 0)
 	{
 		tf_free(connection->message_requests);
 		connection->message_requests = NULL;
@@ -1969,7 +2022,7 @@ static void _tf_ssb_connection_destroy(tf_ssb_connection_t* connection, const ch
 			connection->debug_messages[i] = NULL;
 		}
 
-		if (--connection->ssb->connection_ref_count == 0 && connection->ssb->shutting_down)
+		if (--connection->ssb->connection_ref_count == 0 && connection->ssb->shutting_down_deferred)
 		{
 			tf_ssb_destroy(connection->ssb);
 		}
@@ -2112,8 +2165,9 @@ static bool _tf_ssb_connection_read_start(tf_ssb_connection_t* connection)
 	int result = uv_read_start((uv_stream_t*)&connection->tcp, _tf_ssb_connection_on_tcp_alloc, _tf_ssb_connection_on_tcp_recv);
 	if (result && result != UV_EALREADY)
 	{
-		tf_printf("uv_read_start => %s\n", uv_strerror(result));
-		_tf_ssb_connection_close(connection, "uv_read_start failed");
+		char reason[1024];
+		snprintf(reason, sizeof(reason), "uv_read_start failed: %s", uv_strerror(result));
+		_tf_ssb_connection_close(connection, reason);
 		return false;
 	}
 	return true;
@@ -2124,8 +2178,9 @@ static bool _tf_ssb_connection_read_stop(tf_ssb_connection_t* connection)
 	int result = uv_read_stop((uv_stream_t*)&connection->tcp);
 	if (result && result != UV_EALREADY)
 	{
-		tf_printf("uv_read_stop => %s\n", uv_strerror(result));
-		_tf_ssb_connection_close(connection, "uv_read_stop failed");
+		char reason[1024];
+		snprintf(reason, sizeof(reason), "uv_read_stop failed: %s", uv_strerror(result));
+		_tf_ssb_connection_close(connection, reason);
 		return false;
 	}
 	return true;
@@ -2145,7 +2200,9 @@ static void _tf_ssb_connection_on_connect(uv_connect_t* connect, int status)
 	}
 	else
 	{
-		_tf_ssb_connection_close(connection, "uv_tcp_connect failed");
+		char reason[1024];
+		snprintf(reason, sizeof(reason), "uv_tcp_connect failed: %s", uv_strerror(status));
+		_tf_ssb_connection_close(connection, reason);
 	}
 }
 
@@ -2190,7 +2247,7 @@ static void _tf_ssb_trace_timer(uv_timer_t* timer)
 		ssb->broadcasts_changed_count,
 	};
 
-	tf_trace_counter(ssb->trace, "ssb", _countof(values), names, values);
+	tf_trace_counter(ssb->trace, "ssb", tf_countof(values), names, values);
 }
 
 void tf_ssb_get_stats(tf_ssb_t* ssb, tf_ssb_stats_t* out_stats)
@@ -2472,12 +2529,6 @@ void tf_ssb_destroy(tf_ssb_t* ssb)
 	tf_printf("tf_ssb_destroy\n");
 	ssb->shutting_down = true;
 
-	if (ssb->connections_tracker)
-	{
-		tf_ssb_connections_destroy(ssb->connections_tracker);
-		ssb->connections_tracker = NULL;
-	}
-
 	if (ssb->broadcast_listener.data && !uv_is_closing((uv_handle_t*)&ssb->broadcast_listener))
 	{
 		uv_close((uv_handle_t*)&ssb->broadcast_listener, _tf_ssb_on_handle_close);
@@ -2593,10 +2644,20 @@ void tf_ssb_destroy(tf_ssb_t* ssb)
 		tf_ssb_connection_close(connection);
 		connection = next;
 	}
+	uv_run(ssb->loop, UV_RUN_NOWAIT);
 	tf_printf("Closed.\n");
 
+	if (ssb->connections_tracker)
+	{
+		tf_ssb_connections_destroy(ssb->connections_tracker);
+		ssb->connections_tracker = NULL;
+	}
+
+	uv_run(ssb->loop, UV_RUN_NOWAIT);
+
 	if (ssb->loop == &ssb->own_loop)
 	{
+		tf_printf("uv_loop_close\n");
 		int r = uv_loop_close(ssb->loop);
 		if (r != 0)
 		{
@@ -2651,6 +2712,7 @@ void tf_ssb_destroy(tf_ssb_t* ssb)
 		ssb->room_name = NULL;
 	}
 
+	ssb->shutting_down_deferred = true;
 	if (ssb->connection_ref_count == 0)
 	{
 		uv_mutex_destroy(&ssb->db_readers_lock);
@@ -2698,22 +2760,33 @@ static void _tf_ssb_connection_handshake_timer_callback(uv_timer_t* timer)
 	}
 }
 
-tf_ssb_connection_t* tf_ssb_connection_create(tf_ssb_t* ssb, const char* host, const struct sockaddr_in* addr, const uint8_t* public_key)
+static tf_ssb_connection_t* _tf_ssb_connection_create(
+	tf_ssb_t* ssb, const char* host, const struct sockaddr_in* addr, const uint8_t* public_key, tf_ssb_connect_callback_t* callback, void* user_data)
 {
 	for (tf_ssb_connection_t* connection = ssb->connections; connection; connection = connection->next)
 	{
 		if (memcmp(connection->serverpub, public_key, k_id_bin_len) == 0 && connection->state != k_tf_ssb_state_invalid)
 		{
-			char id[k_id_base64_len];
-			tf_ssb_id_bin_to_str(id, sizeof(id), public_key);
-			tf_printf("Not connecting to %s:%d, because we are already connected to %s (state = %d).\n", host, ntohs(addr->sin_port), id, connection->state);
+			if (callback)
+			{
+				char id[k_id_base64_len];
+				tf_ssb_id_bin_to_str(id, sizeof(id), public_key);
+				char reason[1024];
+				snprintf(reason, sizeof(reason), "Already connected to %s (%s).", id, _tf_ssb_connection_state_to_string(connection->state));
+				callback(NULL, reason, user_data);
+			}
 			return NULL;
 		}
 		else if (memcmp(ssb->pub, public_key, k_id_bin_len) == 0)
 		{
-			char id[k_id_base64_len];
-			tf_ssb_id_bin_to_str(id, sizeof(id), public_key);
-			tf_printf("Not connecting to %s:%d, because they appear to be ourselves %s.\n", host, ntohs(addr->sin_port), id);
+			if (callback)
+			{
+				char id[k_id_base64_len];
+				tf_ssb_id_bin_to_str(id, sizeof(id), public_key);
+				char reason[1024];
+				snprintf(reason, sizeof(reason), "Not connecting to ourself: %s.", id);
+				callback(NULL, reason, user_data);
+			}
 			return NULL;
 		}
 	}
@@ -2731,6 +2804,10 @@ tf_ssb_connection_t* tf_ssb_connection_create(tf_ssb_t* ssb, const char* host, c
 	connection->port = ntohs(addr->sin_port);
 	connection->async.data = connection;
 	uv_async_init(ssb->loop, &connection->async, _tf_ssb_connection_process_message_async);
+	connection->scheduled_async.data = connection;
+	uv_async_init(ssb->loop, &connection->scheduled_async, _tf_ssb_connection_scheduled_async);
+	connection->connect_callback = callback;
+	connection->connect_callback_user_data = user_data;
 
 	connection->handshake_timer.data = connection;
 	uv_timer_init(ssb->loop, &connection->handshake_timer);
@@ -2751,9 +2828,10 @@ tf_ssb_connection_t* tf_ssb_connection_create(tf_ssb_t* ssb, const char* host, c
 	int result = uv_tcp_connect(&connection->connect, &connection->tcp, (const struct sockaddr*)addr, _tf_ssb_connection_on_connect);
 	if (result)
 	{
-		tf_printf("uv_tcp_connect(%s): %s\n", host, uv_strerror(result));
+		char reason[1024];
+		snprintf(reason, sizeof(reason), "uv_tcp_connect(%s) => %s", host, uv_strerror(result));
 		connection->connect.data = NULL;
-		_tf_ssb_connection_destroy(connection, "connect failed");
+		_tf_ssb_connection_destroy(connection, reason);
 	}
 	else
 	{
@@ -2784,6 +2862,10 @@ static void _tf_ssb_connection_tunnel_callback(
 tf_ssb_connection_t* tf_ssb_connection_tunnel_create(tf_ssb_t* ssb, const char* portal_id, int32_t request_number, const char* target_id, int connect_flags)
 {
 	tf_ssb_connection_t* connection = tf_ssb_connection_get(ssb, portal_id);
+	if (!connection)
+	{
+		return NULL;
+	}
 
 	JSContext* context = ssb->context;
 	tf_ssb_connection_t* tunnel = tf_malloc(sizeof(tf_ssb_connection_t));
@@ -2797,6 +2879,8 @@ tf_ssb_connection_t* tf_ssb_connection_tunnel_create(tf_ssb_t* ssb, const char*
 	tunnel->send_request_number = 1;
 	tunnel->async.data = tunnel;
 	uv_async_init(ssb->loop, &tunnel->async, _tf_ssb_connection_process_message_async);
+	tunnel->scheduled_async.data = tunnel;
+	uv_async_init(ssb->loop, &tunnel->scheduled_async, _tf_ssb_connection_scheduled_async);
 
 	tunnel->handshake_timer.data = tunnel;
 	uv_timer_init(ssb->loop, &tunnel->handshake_timer);
@@ -2836,6 +2920,8 @@ typedef struct _connect_t
 	int port;
 	int flags;
 	uint8_t key[k_id_bin_len];
+	tf_ssb_connect_callback_t* callback;
+	void* user_data;
 } connect_t;
 
 static void _tf_on_connect_getaddrinfo(uv_getaddrinfo_t* addrinfo, int result, struct addrinfo* info)
@@ -2847,26 +2933,36 @@ static void _tf_on_connect_getaddrinfo(uv_getaddrinfo_t* addrinfo, int result, s
 		{
 			struct sockaddr_in addr = *(struct sockaddr_in*)info->ai_addr;
 			addr.sin_port = htons(connect->port);
-			tf_ssb_connection_t* connection = tf_ssb_connection_create(connect->ssb, connect->host, &addr, connect->key);
+			tf_ssb_connection_t* connection = _tf_ssb_connection_create(connect->ssb, connect->host, &addr, connect->key, connect->callback, connect->user_data);
 			if (connection)
 			{
 				connection->flags = connect->flags;
 			}
 		}
-		else
+		else if (connect->callback)
 		{
-			tf_printf("getaddrinfo(%s) => %s\n", connect->host, uv_strerror(result));
+			char reason[1024];
+			snprintf(reason, sizeof(reason), "uv_getaddrinfo(%s) => %s", connect->host, uv_strerror(result));
+			connect->callback(NULL, reason, connect->user_data);
 		}
 	}
+	else if (connect->callback)
+	{
+		connect->callback(NULL, "Shutting down.", connect->user_data);
+	}
 	uv_freeaddrinfo(info);
 	tf_ssb_unref(connect->ssb);
 	tf_free(connect);
 }
 
-void tf_ssb_connect(tf_ssb_t* ssb, const char* host, int port, const uint8_t* key, int connect_flags)
+void tf_ssb_connect(tf_ssb_t* ssb, const char* host, int port, const uint8_t* key, int connect_flags, tf_ssb_connect_callback_t* callback, void* user_data)
 {
 	if (ssb->shutting_down)
 	{
+		if (callback)
+		{
+			callback(NULL, "Shutting down.", user_data);
+		}
 		return;
 	}
 	connect_t* connect = tf_malloc(sizeof(connect_t));
@@ -2875,6 +2971,8 @@ void tf_ssb_connect(tf_ssb_t* ssb, const char* host, int port, const uint8_t* ke
 		.port = port,
 		.flags = connect_flags,
 		.req.data = connect,
+		.callback = callback,
+		.user_data = user_data,
 	};
 	char id[k_id_base64_len] = { 0 };
 	tf_ssb_id_bin_to_str(id, sizeof(id), key);
@@ -2885,6 +2983,12 @@ void tf_ssb_connect(tf_ssb_t* ssb, const char* host, int port, const uint8_t* ke
 	int r = uv_getaddrinfo(ssb->loop, &connect->req, _tf_on_connect_getaddrinfo, host, NULL, &(struct addrinfo) { .ai_family = AF_INET });
 	if (r < 0)
 	{
+		if (callback)
+		{
+			char reason[1024];
+			snprintf(reason, sizeof(reason), "uv_getaddr_info(%s): %s", host, uv_strerror(r));
+			callback(NULL, reason, user_data);
+		}
 		tf_printf("uv_getaddrinfo(%s): %s\n", host, uv_strerror(r));
 		tf_free(connect);
 		tf_ssb_unref(ssb);
@@ -2914,19 +3018,27 @@ static void _tf_ssb_on_connection(uv_stream_t* stream, int status)
 	connection->send_request_number = 1;
 	connection->async.data = connection;
 	uv_async_init(ssb->loop, &connection->async, _tf_ssb_connection_process_message_async);
+	connection->scheduled_async.data = connection;
+	uv_async_init(ssb->loop, &connection->scheduled_async, _tf_ssb_connection_scheduled_async);
 
 	connection->object = JS_NewObjectClass(ssb->context, _connection_class_id);
 	JS_SetOpaque(connection->object, connection);
 
-	if (uv_tcp_init(ssb->loop, &connection->tcp) != 0)
+	int result = uv_tcp_init(ssb->loop, &connection->tcp);
+	if (result != 0)
 	{
-		_tf_ssb_connection_destroy(connection, "init failed");
+		char reason[1024];
+		snprintf(reason, sizeof(reason), "uv_tcp_init() => %s", uv_strerror(result));
+		_tf_ssb_connection_destroy(connection, reason);
 		return;
 	}
 
-	if (uv_accept(stream, (uv_stream_t*)&connection->tcp) != 0)
+	result = uv_accept(stream, (uv_stream_t*)&connection->tcp);
+	if (result != 0)
 	{
-		_tf_ssb_connection_destroy(connection, "accept failed");
+		char reason[1024];
+		snprintf(reason, sizeof(reason), "uv_accept() => %s", uv_strerror(result));
+		_tf_ssb_connection_destroy(connection, reason);
 		return;
 	}
 
@@ -3164,16 +3276,18 @@ static bool _tf_ssb_parse_broadcast(const char* in_broadcast, tf_ssb_broadcast_t
 	return false;
 }
 
-void tf_ssb_connect_str(tf_ssb_t* ssb, const char* address, int connect_flags)
+void tf_ssb_connect_str(tf_ssb_t* ssb, const char* address, int connect_flags, tf_ssb_connect_callback_t* callback, void* user_data)
 {
 	tf_ssb_broadcast_t broadcast = { 0 };
 	if (_tf_ssb_parse_broadcast(address, &broadcast))
 	{
-		tf_ssb_connect(ssb, broadcast.host, ntohs(broadcast.addr.sin_port), broadcast.pub, connect_flags);
+		tf_ssb_connect(ssb, broadcast.host, ntohs(broadcast.addr.sin_port), broadcast.pub, connect_flags, callback, user_data);
 	}
-	else
+	else if (callback)
 	{
-		tf_printf("Unable to parse: %s\n", address);
+		char reason[1024] = "";
+		snprintf(reason, sizeof(reason), "Unable to parse: '%s'.", address);
+		callback(NULL, reason, user_data);
 	}
 }
 
@@ -4284,6 +4398,7 @@ void tf_ssb_connection_adjust_read_backpressure(tf_ssb_connection_t* connection,
 	const int k_threshold = 256;
 	int old_pressure = connection->read_back_pressure;
 	connection->read_back_pressure += delta;
+	uv_async_send(&connection->scheduled_async);
 	if (!connection->closing)
 	{
 		if (old_pressure < k_threshold && connection->read_back_pressure >= k_threshold)
@@ -4305,7 +4420,7 @@ void tf_ssb_connection_adjust_read_backpressure(tf_ssb_connection_t* connection,
 void tf_ssb_connection_adjust_write_count(tf_ssb_connection_t* connection, int delta)
 {
 	connection->active_write_count += delta;
-	_tf_ssb_connection_dispatch_scheduled(connection);
+	uv_async_send(&connection->scheduled_async);
 }
 
 void tf_ssb_sync_start(tf_ssb_t* ssb)
@@ -4335,8 +4450,7 @@ bool tf_ssb_tunnel_create(tf_ssb_t* ssb, const char* portal_id, const char* targ
 		tf_ssb_connection_rpc_send_json(connection, k_ssb_rpc_flag_stream | k_ssb_rpc_flag_new_request, request_number, "tunnel.connect", message, NULL, NULL, NULL);
 		JS_FreeValue(context, message);
 
-		tf_ssb_connection_tunnel_create(ssb, portal_id, request_number, target_id, connect_flags);
-		return true;
+		return tf_ssb_connection_tunnel_create(ssb, portal_id, request_number, target_id, connect_flags) != NULL;
 	}
 	return false;
 }

src/ssb.connections.c

@@ -3,16 +3,13 @@
 #include "log.h"
 #include "mem.h"
 #include "ssb.h"
+#include "util.js.h"
 
 #include "sqlite3.h"
 #include "uv.h"
 
 #include <string.h>
 
-#if !defined(_countof)
-#define _countof(a) ((int)(sizeof((a)) / sizeof(*(a))))
-#endif
-
 typedef struct _tf_ssb_connections_t
 {
 	tf_ssb_t* ssb;
@@ -106,7 +103,7 @@ static void _tf_ssb_connections_get_next_after_work(tf_ssb_t* ssb, int status, v
 		uint8_t key_bin[k_id_bin_len];
 		if (tf_ssb_id_str_to_bin(key_bin, next->key))
 		{
-			tf_ssb_connect(ssb, next->host, next->port, key_bin, 0);
+			tf_ssb_connect(ssb, next->host, next->port, key_bin, 0, NULL, NULL);
 		}
 	}
 	tf_free(next);
@@ -115,9 +112,13 @@ static void _tf_ssb_connections_get_next_after_work(tf_ssb_t* ssb, int status, v
 static void _tf_ssb_connections_timer(uv_timer_t* timer)
 {
 	tf_ssb_connections_t* connections = timer->data;
+	if (tf_ssb_is_shutting_down(connections->ssb))
+	{
+		return;
+	}
 	tf_ssb_connection_t* active[4];
-	int count = tf_ssb_get_connections(connections->ssb, active, _countof(active));
-	if (count < (int)_countof(active))
+	int count = tf_ssb_get_connections(connections->ssb, active, tf_countof(active));
+	if (count < tf_countof(active))
 	{
 		tf_ssb_connections_get_next_t* next = tf_malloc(sizeof(tf_ssb_connections_get_next_t));
 		*next = (tf_ssb_connections_get_next_t) {
@@ -286,7 +287,7 @@ static void _tf_ssb_connections_sync_broadcast_visit(
 	}
 	else
 	{
-		tf_ssb_connect(ssb, host, ntohs(addr->sin_port), pub, k_tf_ssb_connect_flag_one_shot);
+		tf_ssb_connect(ssb, host, ntohs(addr->sin_port), pub, k_tf_ssb_connect_flag_one_shot, NULL, NULL);
 	}
 }
 
@@ -332,7 +333,7 @@ static void _tf_ssb_connections_get_all_after_work(tf_ssb_t* ssb, int status, vo
 	tf_ssb_connections_get_all_work_t* work = user_data;
 	for (int i = 0; i < work->connections_count; i++)
 	{
-		tf_ssb_connect_str(ssb, work->connections[i], k_tf_ssb_connect_flag_one_shot);
+		tf_ssb_connect_str(ssb, work->connections[i], k_tf_ssb_connect_flag_one_shot, NULL, NULL);
 		tf_free(work->connections[i]);
 	}
 	tf_free(work->connections);

src/ssb.db.c

@@ -134,6 +134,8 @@ void tf_ssb_db_init(tf_ssb_t* ssb)
 	_tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_author_timestamp_index ON messages (author, timestamp)");
 	_tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_timestamp_index ON messages (timestamp)");
 	_tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_type_timestamp_index ON messages (content ->> 'type', timestamp)");
+	_tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_size_by_author_index ON messages (author, length(content))");
+	_tf_ssb_db_exec(db, "CREATE INDEX IF NOT EXISTS messages_type_author_channel_index ON messages (content ->> 'type', author, content ->> 'channel');");
 	_tf_ssb_db_exec(db,
 		"CREATE TABLE IF NOT EXISTS blobs ("
 		"  id TEXT PRIMARY KEY,"
@@ -232,13 +234,21 @@ void tf_ssb_db_init(tf_ssb_t* ssb)
 	_tf_ssb_db_exec(db, "DROP VIEW IF EXISTS blob_wants_view");
 	_tf_ssb_db_exec(db,
 		"CREATE VIEW IF NOT EXISTS blob_wants_view (id, timestamp) AS "
-		"  SELECT messages_refs.ref AS id, messages.timestamp AS timestamp "
-		"  FROM messages_refs "
-		"  JOIN messages ON messages.id = messages_refs.message "
-		"  LEFT OUTER JOIN blobs ON messages_refs.ref = blobs.id "
+		"  WITH wanted AS ( "
+		"    SELECT messages_refs.ref AS id, messages.timestamp AS timestamp "
+		"    FROM messages_refs "
+		"    JOIN messages ON messages.id = messages_refs.message "
+		"    UNION "
+		"    SELECT messages_refs.ref AS id, unixepoch() * 1000 AS timestamp "
+		"    FROM messages_refs "
+		"    JOIN messages ON messages.id = messages_refs.message "
+		"    WHERE messages.content ->> 'type' = 'about' "
+		"  ) "
+		"  SELECT wanted.id, wanted.timestamp FROM wanted "
+		"  LEFT OUTER JOIN blobs ON wanted.id = blobs.id "
 		"  WHERE blobs.id IS NULL "
-		"    AND LENGTH(messages_refs.ref) = 52 "
-		"    AND messages_refs.ref LIKE '&%.sha256'");
+		"    AND LENGTH(wanted.id) = 52 "
+		"    AND wanted.id LIKE '&%.sha256'");
 
 	bool need_add_flags = true;
 	bool need_convert_timestamp_to_real = false;
@@ -767,14 +777,6 @@ bool tf_ssb_db_blob_store(tf_ssb_t* ssb, const uint8_t* blob, size_t size, char*
 	}
 	tf_ssb_release_db_writer(ssb, db);
 
-	if (rows)
-	{
-		if (!out_new)
-		{
-			tf_printf("blob stored %s %zd => %d\n", id, size, result);
-		}
-	}
-
 	if (result && out_id)
 	{
 		snprintf(out_id, out_id_size, "%s", id);
@@ -1733,7 +1735,6 @@ bool tf_ssb_db_register_account(uv_loop_t* loop, sqlite3* db, JSContext* context
 		{
 			if (sqlite3_bind_text(statement, 1, value, value_length, NULL) == SQLITE_OK)
 			{
-				tf_printf("added user to properties\n");
 				result = sqlite3_step(statement) == SQLITE_DONE;
 			}
 			sqlite3_finalize(statement);
@@ -2006,12 +2007,12 @@ bool tf_ssb_db_verify(tf_ssb_t* ssb, const char* id)
 	return verified;
 }
 
-bool tf_ssb_db_user_has_permission(tf_ssb_t* ssb, const char* id, const char* permission)
+bool tf_ssb_db_user_has_permission(tf_ssb_t* ssb, sqlite3* db, const char* id, const char* permission)
 {
 	bool has_permission = false;
-	sqlite3* db = tf_ssb_acquire_db_reader(ssb);
+	sqlite3* reader = db ? db : tf_ssb_acquire_db_reader(ssb);
 	sqlite3_stmt* statement = NULL;
-	if (sqlite3_prepare(db,
+	if (sqlite3_prepare(reader,
 			"SELECT COUNT(*) FROM properties, json_each(properties.value -> 'permissions' -> ?) AS permission WHERE properties.id = 'core' AND properties.key = 'settings' AND "
 			"permission.value = ?",
 			-1, &statement, NULL) == SQLITE_OK)
@@ -2023,6 +2024,9 @@ bool tf_ssb_db_user_has_permission(tf_ssb_t* ssb, const char* id, const char* pe
 		}
 		sqlite3_finalize(statement);
 	}
-	tf_ssb_release_db_reader(ssb, db);
+	if (reader != db)
+	{
+		tf_ssb_release_db_reader(ssb, reader);
+	}
 	return has_permission;
 }

src/ssb.db.h

@@ -448,11 +448,12 @@ bool tf_ssb_db_verify(tf_ssb_t* ssb, const char* id);
 /**
 ** Check if a user has a specific permission.
 ** @param ssb The SSB instance.
+** @param db Optional database instance.  If NULL, one will be acquired from ssb.
 ** @param id The user ID.
 ** @param permission The name of the permission.
 ** @return true If the user has the requested permission.
 */
-bool tf_ssb_db_user_has_permission(tf_ssb_t* ssb, const char* id, const char* permission);
+bool tf_ssb_db_user_has_permission(tf_ssb_t* ssb, sqlite3* db, const char* id, const char* permission);
 
 /**
 ** An SQLite authorizer callback.  See https://www.sqlite.org/c3ref/set_authorizer.html for use.

src/ssb.export.c

@@ -90,7 +90,7 @@ void tf_ssb_export(tf_ssb_t* ssb, const char* key)
 		return;
 	}
 
-	char app_blob_id[64] = { 0 };
+	char app_blob_id[k_blob_id_len] = { 0 };
 	sqlite3* db = tf_ssb_acquire_db_reader(ssb);
 	sqlite3_busy_timeout(db, 10000);
 	sqlite3_stmt* statement;

src/ssb.h

@@ -348,6 +348,14 @@ const char** tf_ssb_get_connection_ids(tf_ssb_t* ssb);
 */
 int tf_ssb_get_connections(tf_ssb_t* ssb, tf_ssb_connection_t** out_connections, int out_connections_count);
 
+/**
+** Callback for completing establishing a connection.
+** @param connection The established connection if successful or null.
+** @param reason The reason for failure if the connection failed.
+** @param user_data User data.
+*/
+typedef void(tf_ssb_connect_callback_t)(tf_ssb_connection_t* connection, const char* reason, void* user_data);
+
 /**
 ** Establish an SHS connection with a host.
 ** @param ssb The SSB instance.
@@ -355,16 +363,20 @@ int tf_ssb_get_connections(tf_ssb_t* ssb, tf_ssb_connection_t** out_connections,
 ** @param port The host's SHS port.
 ** @param key The host's SSB identity.
 ** @param connect_flags Flags affecting the connection.
+** @param callback Completion callback.
+** @param user_data User data to be passed to the callback.
 */
-void tf_ssb_connect(tf_ssb_t* ssb, const char* host, int port, const uint8_t* key, int connect_flags);
+void tf_ssb_connect(tf_ssb_t* ssb, const char* host, int port, const uint8_t* key, int connect_flags, tf_ssb_connect_callback_t* callback, void* user_data);
 
 /**
 ** Establish an SHS connection with a host by string address.
 ** @param ssb The SSB instance.
 ** @param address The address.
 ** @param connect_flags Flags affecting the connection.
+** @param callback Completion callback.
+** @param user_data User data to be passed to the callback.
 */
-void tf_ssb_connect_str(tf_ssb_t* ssb, const char* address, int connect_flags);
+void tf_ssb_connect_str(tf_ssb_t* ssb, const char* address, int connect_flags, tf_ssb_connect_callback_t* callback, void* user_data);
 
 /**
 ** Begin listening for SHS connections on the given port.

src/ssb.js.c

@@ -20,10 +20,6 @@
 #include <assert.h>
 #include <inttypes.h>
 
-#if !defined(_countof)
-#define _countof(a) ((int)(sizeof((a)) / sizeof(*(a))))
-#endif
-
 static const int k_sql_async_timeout_ms = 60 * 1000;
 
 static JSClassID _tf_ssb_classId;
@@ -271,7 +267,7 @@ static JSValue _set_server_following_internal(tf_ssb_t* ssb, JSValueConst this_v
 		server_id,
 		message,
 	};
-	JSValue result = _tf_ssb_appendMessageWithIdentity(context, this_val, _countof(args), args);
+	JSValue result = _tf_ssb_appendMessageWithIdentity(context, this_val, tf_countof(args), args);
 	JS_FreeValue(context, server_id);
 	JS_FreeValue(context, server_user);
 	JS_FreeValue(context, message);
@@ -372,9 +368,9 @@ typedef struct _swap_with_server_identity_t
 static void _tf_ssb_swap_with_server_identity_work(tf_ssb_t* ssb, void* user_data)
 {
 	swap_with_server_identity_t* work = user_data;
-	if (tf_ssb_db_user_has_permission(ssb, work->user, "administration"))
+	sqlite3* db = tf_ssb_acquire_db_writer(ssb);
+	if (tf_ssb_db_user_has_permission(ssb, db, work->user, "administration"))
 	{
-		sqlite3* db = tf_ssb_acquire_db_writer(ssb);
 		char* error = NULL;
 		if (sqlite3_exec(db, "BEGIN TRANSACTION", NULL, NULL, &error) == SQLITE_OK)
 		{
@@ -408,12 +404,12 @@ static void _tf_ssb_swap_with_server_identity_work(tf_ssb_t* ssb, void* user_dat
 		{
 			work->error = error ? tf_strdup(error) : tf_strdup(sqlite3_errmsg(db));
 		}
-		tf_ssb_release_db_writer(ssb, db);
 	}
 	else
 	{
 		work->error = tf_strdup("not administrator");
 	}
+	tf_ssb_release_db_writer(ssb, db);
 }
 
 static void _tf_ssb_swap_with_server_identity_after_work(tf_ssb_t* ssb, int status, void* user_data)
@@ -484,7 +480,7 @@ static void _tf_ssb_getIdentities_visit(const char* identity, void* user_data)
 static void _tf_ssb_get_identities_work(tf_ssb_t* ssb, void* user_data)
 {
 	identities_visit_t* work = user_data;
-	if (tf_ssb_db_user_has_permission(ssb, work->user, "administration"))
+	if (tf_ssb_db_user_has_permission(ssb, NULL, work->user, "administration"))
 	{
 		char id[k_id_base64_len] = "";
 		if (tf_ssb_whoami(ssb, id, sizeof(id)))
@@ -556,7 +552,7 @@ static void _tf_ssb_get_private_key_work(tf_ssb_t* ssb, void* user_data)
 {
 	get_private_key_t* work = user_data;
 	work->got_private_key = tf_ssb_db_identity_get_private_key(ssb, work->user, work->id, work->private_key, sizeof(work->private_key));
-	if (!work->got_private_key && tf_ssb_db_user_has_permission(ssb, work->user, "administration"))
+	if (!work->got_private_key && tf_ssb_db_user_has_permission(ssb, NULL, work->user, "administration"))
 	{
 		work->got_private_key = tf_ssb_db_identity_get_private_key(ssb, ":admin", work->id, work->private_key, sizeof(work->private_key));
 	}
@@ -646,7 +642,7 @@ static void _tf_ssb_getActiveIdentity_visit(const char* identity, void* user_dat
 	active_identity_work_t* request = user_data;
 	if (!*request->identity)
 	{
-		snprintf(request->identity, sizeof(request->identity), "%s", identity);
+		snprintf(request->identity, sizeof(request->identity), "@%s", identity);
 	}
 }
 
@@ -661,6 +657,11 @@ static void _tf_ssb_getActiveIdentity_work(tf_ssb_t* ssb, void* user_data)
 	{
 		tf_ssb_db_identity_visit(ssb, request->name, _tf_ssb_getActiveIdentity_visit, request);
 	}
+
+	if (!*request->identity && tf_ssb_db_user_has_permission(ssb, NULL, request->name, "administration"))
+	{
+		tf_ssb_whoami(ssb, request->identity, sizeof(request->identity));
+	}
 }
 
 static void _tf_ssb_getActiveIdentity_after_work(tf_ssb_t* ssb, int status, void* user_data)
@@ -741,7 +742,7 @@ static void _tf_ssb_getIdentityInfo_work(tf_ssb_t* ssb, void* user_data)
 {
 	identity_info_work_t* request = user_data;
 	char id[k_id_base64_len] = "";
-	if (tf_ssb_db_user_has_permission(ssb, request->name, "administration"))
+	if (tf_ssb_db_user_has_permission(ssb, NULL, request->name, "administration"))
 	{
 		if (tf_ssb_whoami(ssb, id, sizeof(id)))
 		{
@@ -903,7 +904,7 @@ static void _tf_ssb_append_message_with_identity_get_key_work(tf_ssb_t* ssb, voi
 {
 	append_message_t* work = user_data;
 	work->got_private_key = tf_ssb_db_identity_get_private_key(ssb, work->user, work->id, work->private_key, sizeof(work->private_key));
-	if (!work->got_private_key && tf_ssb_db_user_has_permission(ssb, work->user, "administration"))
+	if (!work->got_private_key && tf_ssb_db_user_has_permission(ssb, NULL, work->user, "administration"))
 	{
 		work->got_private_key = tf_ssb_db_identity_get_private_key(ssb, ":admin", work->id, work->private_key, sizeof(work->private_key));
 	}
@@ -1088,7 +1089,7 @@ static JSValue _tf_ssb_connections(JSContext* context, JSValueConst this_val, in
 	if (ssb)
 	{
 		tf_ssb_connection_t* connections[32];
-		int count = tf_ssb_get_connections(ssb, connections, _countof(connections));
+		int count = tf_ssb_get_connections(ssb, connections, tf_countof(connections));
 
 		result = JS_NewArray(context);
 		for (int i = 0; i < count; i++)
@@ -1633,17 +1634,41 @@ static JSValue _tf_ssb_getBroadcasts(JSContext* context, JSValueConst this_val,
 	return result;
 }
 
+typedef struct _connect_t
+{
+	JSContext* context;
+	JSValue promise[2];
+} connect_t;
+
+static void _tf_ssb_connect_callback(tf_ssb_connection_t* connection, const char* reason, void* user_data)
+{
+	connect_t* connect = user_data;
+	JSContext* context = connect->context;
+	JSValue arg = connection ? JS_UNDEFINED : JS_NewString(context, reason);
+	JSValue result = JS_Call(context, connection ? connect->promise[0] : connect->promise[1], JS_UNDEFINED, connection ? 0 : 1, &arg);
+	tf_util_report_error(context, result);
+	JS_FreeValue(context, result);
+	JS_FreeValue(context, connect->promise[0]);
+	JS_FreeValue(context, connect->promise[1]);
+	JS_FreeValue(context, arg);
+	tf_free(connect);
+}
+
 static JSValue _tf_ssb_connect(JSContext* context, JSValueConst this_val, int argc, JSValueConst* argv)
 {
+	JSValue result = JS_UNDEFINED;
 	JSValue args = argv[0];
 	tf_ssb_t* ssb = JS_GetOpaque(this_val, _tf_ssb_classId);
 	if (ssb)
 	{
+		connect_t* connect = tf_malloc(sizeof(connect_t));
+		*connect = (connect_t) { .context = context };
+		result = JS_NewPromiseCapability(context, connect->promise);
 		if (JS_IsString(args))
 		{
 			const char* address_str = JS_ToCString(context, args);
 			tf_printf("Connecting to %s\n", address_str);
-			tf_ssb_connect_str(ssb, address_str, 0);
+			tf_ssb_connect_str(ssb, address_str, 0, _tf_ssb_connect_callback, connect);
 			JS_FreeCString(context, address_str);
 		}
 		else
@@ -1660,11 +1685,11 @@ static JSValue _tf_ssb_connect(JSContext* context, JSValueConst this_val, int ar
 				tf_printf("Connecting to %s:%d\n", address_str, port_int);
 				uint8_t pubkey_bin[k_id_bin_len];
 				tf_ssb_id_str_to_bin(pubkey_bin, pubkey_str);
-				tf_ssb_connect(ssb, address_str, port_int, pubkey_bin, 0);
+				tf_ssb_connect(ssb, address_str, port_int, pubkey_bin, 0, _tf_ssb_connect_callback, connect);
 			}
 			else
 			{
-				tf_printf("Not connecting to null.\n");
+				_tf_ssb_connect_callback(NULL, "Not connecting to null.", connect);
 			}
 			JS_FreeCString(context, pubkey_str);
 			JS_FreeCString(context, address_str);
@@ -1673,7 +1698,7 @@ static JSValue _tf_ssb_connect(JSContext* context, JSValueConst this_val, int ar
 			JS_FreeValue(context, pubkey);
 		}
 	}
-	return JS_UNDEFINED;
+	return result;
 }
 
 typedef struct _forget_stored_connection_t
@@ -1949,7 +1974,7 @@ enum
 	k_max_private_message_recipients = 8
 };
 
-static bool _tf_ssb_get_private_key_curve25519(sqlite3* db, const char* user, const char* identity, uint8_t out_private_key[static crypto_sign_SECRETKEYBYTES])
+static bool _tf_ssb_get_private_key_curve25519_internal(sqlite3* db, const char* user, const char* identity, uint8_t out_private_key[static crypto_sign_SECRETKEYBYTES])
 {
 	if (!user || !identity)
 	{
@@ -1978,6 +2003,21 @@ static bool _tf_ssb_get_private_key_curve25519(sqlite3* db, const char* user, co
 	return success;
 }
 
+static bool _tf_ssb_get_private_key_curve25519(tf_ssb_t* ssb, sqlite3* db, const char* user, const char* identity, uint8_t out_private_key[static crypto_sign_SECRETKEYBYTES])
+{
+	if (_tf_ssb_get_private_key_curve25519_internal(db, user, identity, out_private_key))
+	{
+		return true;
+	}
+
+	if (tf_ssb_db_user_has_permission(ssb, db, user, "administration"))
+	{
+		return _tf_ssb_get_private_key_curve25519_internal(db, ":admin", identity, out_private_key);
+	}
+
+	return false;
+}
+
 typedef struct _private_message_encrypt_t
 {
 	const char* signer_user;
@@ -2000,7 +2040,7 @@ static void _tf_ssb_private_message_encrypt_work(tf_ssb_t* ssb, void* user_data)
 
 	uint8_t private_key[crypto_sign_SECRETKEYBYTES] = { 0 };
 	sqlite3* db = tf_ssb_acquire_db_reader(ssb);
-	bool found = _tf_ssb_get_private_key_curve25519(db, work->signer_user, work->signer_identity, private_key);
+	bool found = _tf_ssb_get_private_key_curve25519(ssb, db, work->signer_user, work->signer_identity, private_key);
 	tf_ssb_release_db_reader(ssb, db);
 
 	if (found)
@@ -2189,7 +2229,7 @@ static void _tf_ssb_private_message_decrypt_work(tf_ssb_t* ssb, void* user_data)
 
 	uint8_t private_key[crypto_sign_SECRETKEYBYTES] = { 0 };
 	sqlite3* db = tf_ssb_acquire_db_reader(ssb);
-	bool found = _tf_ssb_get_private_key_curve25519(db, work->user, work->identity, private_key);
+	bool found = _tf_ssb_get_private_key_curve25519(ssb, db, work->user, work->identity, private_key);
 	tf_ssb_release_db_reader(ssb, db);
 
 	if (found)

src/ssb.rpc.c

@@ -14,14 +14,11 @@
 #include <string.h>
 #include <time.h>
 
-#if !defined(_countof)
-#define _countof(a) ((int)(sizeof((a)) / sizeof(*(a))))
-#endif
-
 static void _tf_ssb_connection_send_history_stream(
 	tf_ssb_connection_t* connection, int32_t request_number, const char* author, int64_t sequence, bool keys, bool live, bool end_request);
 static void _tf_ssb_rpc_send_peers_exchange(tf_ssb_connection_t* connection);
 static void _tf_ssb_rpc_start_delete_blobs(tf_ssb_t* ssb, int delay_ms);
+static void _tf_ssb_rpc_start_delete_feeds(tf_ssb_t* ssb, int delay_ms);
 
 static int64_t _get_global_setting_int64(tf_ssb_t* ssb, const char* name, int64_t default_value)
 {
@@ -47,6 +44,30 @@ static int64_t _get_global_setting_int64(tf_ssb_t* ssb, const char* name, int64_
 	return result;
 }
 
+static bool _get_global_setting_bool(tf_ssb_t* ssb, const char* name, bool default_value)
+{
+	bool result = default_value;
+	sqlite3* db = tf_ssb_acquire_db_reader(ssb);
+	sqlite3_stmt* statement;
+	if (sqlite3_prepare(db, "SELECT json_extract(value, '$.' || ?) FROM properties WHERE id = 'core' AND key = 'settings'", -1, &statement, NULL) == SQLITE_OK)
+	{
+		if (sqlite3_bind_text(statement, 1, name, -1, NULL) == SQLITE_OK)
+		{
+			if (sqlite3_step(statement) == SQLITE_ROW)
+			{
+				result = sqlite3_column_int(statement, 0) != 0;
+			}
+		}
+		sqlite3_finalize(statement);
+	}
+	else
+	{
+		tf_printf("prepare failed: %s\n", sqlite3_errmsg(db));
+	}
+	tf_ssb_release_db_reader(ssb, db);
+	return result;
+}
+
 static void _tf_ssb_rpc_gossip_ping_callback(
 	tf_ssb_connection_t* connection, uint8_t flags, int32_t request_number, JSValue args, const uint8_t* message, size_t size, void* user_data)
 {
@@ -240,7 +261,7 @@ static void _tf_ssb_request_blob_wants_work(tf_ssb_connection_t* connection, voi
 	if (sqlite3_prepare(db, "SELECT id FROM blob_wants_view WHERE id > ? AND timestamp > ? ORDER BY id LIMIT ?", -1, &statement, NULL) == SQLITE_OK)
 	{
 		if (sqlite3_bind_text(statement, 1, blob_wants->last_id, -1, NULL) == SQLITE_OK && sqlite3_bind_int64(statement, 2, timestamp) == SQLITE_OK &&
-			sqlite3_bind_int(statement, 3, _countof(work->out_id)) == SQLITE_OK)
+			sqlite3_bind_int(statement, 3, tf_countof(work->out_id)) == SQLITE_OK)
 		{
 			while (sqlite3_step(statement) == SQLITE_ROW)
 			{
@@ -456,7 +477,7 @@ static void _tf_ssb_rpc_room_attendants(tf_ssb_connection_t* connection, uint8_t
 	JSValue ids = JS_NewArray(context);
 	int id_count = 0;
 	tf_ssb_connection_t* connections[1024];
-	int count = tf_ssb_get_connections(ssb, connections, _countof(connections));
+	int count = tf_ssb_get_connections(ssb, connections, tf_countof(connections));
 
 	for (int i = 0; i < count; i++)
 	{
@@ -795,8 +816,7 @@ static void _tf_ssb_connection_send_history_stream_work(tf_ssb_connection_t* con
 	const int k_max = 32;
 	if (sqlite3_prepare(db,
 			"SELECT previous, author, id, sequence, timestamp, hash, json(content), signature, flags FROM messages WHERE author = ?1 AND sequence > ?2 AND "
-			"sequence "
-			"< ?3 ORDER BY sequence",
+			"sequence < ?3 ORDER BY sequence",
 			-1, &statement, NULL) == SQLITE_OK)
 	{
 		if (sqlite3_bind_text(statement, 1, request->author, -1, NULL) == SQLITE_OK && sqlite3_bind_int64(statement, 2, request->sequence) == SQLITE_OK &&
@@ -807,7 +827,8 @@ static void _tf_ssb_connection_send_history_stream_work(tf_ssb_connection_t* con
 			JSRuntime* runtime = JS_NewRuntime2(&funcs, NULL);
 			JSContext* context = JS_NewContext(runtime);
 
-			while (sqlite3_step(statement) == SQLITE_ROW)
+			int r = SQLITE_OK;
+			while ((r = sqlite3_step(statement)) == SQLITE_ROW)
 			{
 				JSValue message = JS_UNDEFINED;
 				request->out_max_sequence_seen = sqlite3_column_int64(statement, 3);
@@ -982,8 +1003,10 @@ static void _tf_ssb_rpc_ebt_replicate_send_clock_work(tf_ssb_connection_t* conne
 	tf_ssb_t* ssb = tf_ssb_connection_get_ssb(connection);
 	JSValue full_clock = JS_NewObject(context);
 
+	int64_t depth = _get_global_setting_int64(ssb, "replication_hops", 2);
+
 	/* Ask for every identity we know is being followed from local accounts. */
-	const char** visible = tf_ssb_db_get_all_visible_identities(ssb, 2);
+	const char** visible = tf_ssb_db_get_all_visible_identities(ssb, depth);
 	for (int i = 0; visible[i]; i++)
 	{
 		int64_t sequence = 0;
@@ -1150,6 +1173,20 @@ static void _tf_ssb_rpc_ebt_replicate_store_callback(const char* id, bool verifi
 	tf_ssb_connection_adjust_read_backpressure(connection, -1);
 }
 
+typedef struct _resend_clock_t
+{
+	tf_ssb_connection_t* connection;
+	int32_t request_number;
+} resend_clock_t;
+
+static void _tf_ssb_rpc_ebt_replicate_resend_clock(tf_ssb_connection_t* connection, void* user_data)
+{
+	resend_clock_t* resend = user_data;
+	_tf_ssb_rpc_ebt_replicate_send_clock(resend->connection, resend->request_number, JS_UNDEFINED);
+	tf_ssb_connection_set_sent_clock(resend->connection, true);
+	tf_free(user_data);
+}
+
 static void _tf_ssb_rpc_ebt_replicate(tf_ssb_connection_t* connection, uint8_t flags, int32_t request_number, JSValue args, const uint8_t* message, size_t size, void* user_data)
 {
 	tf_ssb_t* ssb = tf_ssb_connection_get_ssb(connection);
@@ -1175,6 +1212,17 @@ static void _tf_ssb_rpc_ebt_replicate(tf_ssb_connection_t* connection, uint8_t f
 		/* Looks like a message. */
 		tf_ssb_connection_adjust_read_backpressure(connection, 1);
 		tf_ssb_verify_strip_and_store_message(ssb, args, _tf_ssb_rpc_ebt_replicate_store_callback, connection);
+
+		if (tf_ssb_connection_get_sent_clock(connection))
+		{
+			tf_ssb_connection_set_sent_clock(connection, false);
+			resend_clock_t* resend = tf_malloc(sizeof(resend_clock_t));
+			*resend = (resend_clock_t) {
+				.connection = connection,
+				.request_number = request_number,
+			};
+			tf_ssb_connection_schedule_idle(connection, _tf_ssb_rpc_ebt_replicate_resend_clock, resend);
+		}
 	}
 	else
 	{
@@ -1292,7 +1340,7 @@ static void _tf_ssb_rpc_connections_changed_callback(tf_ssb_t* ssb, tf_ssb_chang
 			JS_SetPropertyStr(context, left, "type", JS_NewString(context, "left"));
 			JS_SetPropertyStr(context, left, "id", JS_NewString(context, id));
 			tf_ssb_connection_t* connections[1024];
-			int count = tf_ssb_get_connections(ssb, connections, _countof(connections));
+			int count = tf_ssb_get_connections(ssb, connections, tf_countof(connections));
 			for (int i = 0; i < count; i++)
 			{
 				if (tf_ssb_connection_is_attendant(connections[i]))
@@ -1343,8 +1391,15 @@ static void _tf_ssb_rpc_checkpoint(tf_ssb_t* ssb)
 	tf_ssb_release_db_writer(ssb, db);
 }
 
+typedef struct _delete_t
+{
+	int deleted;
+	int64_t duration_ms;
+} delete_t;
+
 static void _tf_ssb_rpc_delete_blobs_work(tf_ssb_t* ssb, void* user_data)
 {
+	delete_t* delete = user_data;
 	int64_t age = _get_global_setting_int64(ssb, "blob_expire_age_seconds", -1);
 	if (age <= 0)
 	{
@@ -1386,30 +1441,122 @@ static void _tf_ssb_rpc_delete_blobs_work(tf_ssb_t* ssb, void* user_data)
 		tf_printf("prepare failed: %s\n", sqlite3_errmsg(db));
 	}
 	tf_ssb_release_db_writer(ssb, db);
-	int64_t duration_ms = (uv_hrtime() - start_ns) / 1000000LL;
-	tf_printf("Deleted %d blobs in %d ms.\n", deleted, (int)duration_ms);
+	delete->duration_ms = (uv_hrtime() - start_ns) / 1000000LL;
+	tf_printf("Deleted %d blobs in %d ms.\n", deleted, (int)delete->duration_ms);
 	_tf_ssb_rpc_checkpoint(ssb);
-	_tf_ssb_rpc_start_delete_blobs(ssb, deleted ? (int)duration_ms : (15 * 60 * 1000));
 }
 
 static void _tf_ssb_rpc_delete_blobs_after_work(tf_ssb_t* ssb, int status, void* user_data)
 {
+	delete_t* delete = user_data;
+	_tf_ssb_rpc_start_delete_blobs(ssb, delete->deleted ? (int)delete->duration_ms : (15 * 60 * 1000));
+	tf_free(delete);
 }
 
-static void _tf_ssb_rpc_start_delete_callback(tf_ssb_t* ssb, void* user_data)
+static void _tf_ssb_rpc_start_delete_blobs_callback(tf_ssb_t* ssb, void* user_data)
 {
-	tf_ssb_run_work(ssb, _tf_ssb_rpc_delete_blobs_work, _tf_ssb_rpc_delete_blobs_after_work, NULL);
+	delete_t* delete = tf_malloc(sizeof(delete_t));
+	*delete = (delete_t) { 0 };
+	tf_ssb_run_work(ssb, _tf_ssb_rpc_delete_blobs_work, _tf_ssb_rpc_delete_blobs_after_work, delete);
 }
 
 static void _tf_ssb_rpc_start_delete_blobs(tf_ssb_t* ssb, int delay_ms)
 {
 	tf_printf("will delete more blobs in %d ms\n", delay_ms);
-	tf_ssb_schedule_work(ssb, delay_ms, _tf_ssb_rpc_start_delete_callback, NULL);
+	tf_ssb_schedule_work(ssb, delay_ms, _tf_ssb_rpc_start_delete_blobs_callback, NULL);
+}
+
+static void _tf_ssb_rpc_delete_feeds_work(tf_ssb_t* ssb, void* user_data)
+{
+	delete_t* delete = user_data;
+	if (!_get_global_setting_bool(ssb, "delete_stale_feeds", false))
+	{
+		return;
+	}
+	int64_t start_ns = uv_hrtime();
+	int replication_hops = (int)_get_global_setting_int64(ssb, "replication_hops", 2);
+	const char** identities = tf_ssb_db_get_all_visible_identities(ssb, replication_hops);
+
+	JSMallocFunctions funcs = { 0 };
+	tf_get_js_malloc_functions(&funcs);
+	JSRuntime* runtime = JS_NewRuntime2(&funcs, NULL);
+	JSContext* context = JS_NewContext(runtime);
+	JSValue array = JS_NewArray(context);
+	for (int i = 0; identities[i]; i++)
+	{
+		JS_SetPropertyUint32(context, array, i, JS_NewString(context, identities[i]));
+	}
+	tf_free(identities);
+
+	JSValue json = JS_JSONStringify(context, array, JS_NULL, JS_NULL);
+	const char* arg = JS_ToCString(context, json);
+	JS_FreeValue(context, json);
+	JS_FreeValue(context, array);
+
+	sqlite3* db = tf_ssb_acquire_db_writer(ssb);
+	sqlite3_stmt* statement;
+	if (sqlite3_prepare(db,
+			"DELETE FROM messages WHERE author IN ("
+			"  SELECT author FROM messages WHERE author NOT IN (SELECT value FROM json_each(?)) GROUP BY author LIMIT 1"
+			") RETURNING author",
+			-1, &statement, NULL) == SQLITE_OK)
+	{
+		int status = SQLITE_OK;
+		bool printed = false;
+		if (sqlite3_bind_text(statement, 1, arg, -1, NULL) == SQLITE_OK)
+		{
+			while ((status = sqlite3_step(statement)) == SQLITE_ROW)
+			{
+				if (!printed)
+				{
+					tf_printf("deleting %s\n", sqlite3_column_text(statement, 0));
+					printed = true;
+					delete->deleted++;
+				}
+			}
+			if (status != SQLITE_DONE)
+			{
+				tf_printf("deleting feeds: %s\n", sqlite3_errmsg(db));
+			}
+		}
+		sqlite3_finalize(statement);
+	}
+	tf_ssb_release_db_writer(ssb, db);
+
+	JS_FreeCString(context, arg);
+
+	JS_FreeContext(context);
+	JS_FreeRuntime(runtime);
+
+	delete->duration_ms = (uv_hrtime() - start_ns) / 1000000LL;
+	tf_printf("Deleted %d feeds in %d ms.\n", delete->deleted, (int)delete->duration_ms);
+	_tf_ssb_rpc_checkpoint(ssb);
+}
+
+static void _tf_ssb_rpc_delete_feeds_after_work(tf_ssb_t* ssb, int status, void* user_data)
+{
+	delete_t* delete = user_data;
+	_tf_ssb_rpc_start_delete_feeds(ssb, delete->deleted ? (int)delete->duration_ms : (15 * 60 * 1000));
+	tf_free(delete);
+}
+
+static void _tf_ssb_rpc_start_delete_feeds_callback(tf_ssb_t* ssb, void* user_data)
+{
+	delete_t* delete = tf_malloc(sizeof(delete_t));
+	*delete = (delete_t) { 0 };
+	tf_ssb_run_work(ssb, _tf_ssb_rpc_delete_feeds_work, _tf_ssb_rpc_delete_feeds_after_work, delete);
+}
+
+static void _tf_ssb_rpc_start_delete_feeds(tf_ssb_t* ssb, int delay_ms)
+{
+	tf_printf("will delete more feeds in %d ms\n", delay_ms);
+	tf_ssb_schedule_work(ssb, delay_ms, _tf_ssb_rpc_start_delete_feeds_callback, NULL);
 }
 
 void tf_ssb_rpc_start_periodic(tf_ssb_t* ssb)
 {
 	_tf_ssb_rpc_start_delete_blobs(ssb, 30 * 1000);
+	_tf_ssb_rpc_start_delete_feeds(ssb, 25 * 1000);
 }
 
 typedef struct _peers_exchange_t

src/ssb.tests.c

@@ -16,6 +16,8 @@
 #include <time.h>
 #include <unistd.h>
 
+#include "sodium/crypto_sign.h"
+
 #if !defined(_WIN32)
 #include <sys/wait.h>
 #endif
@@ -268,7 +270,7 @@ void tf_ssb_test_ssb(const tf_test_options_t* options)
 
 	uint8_t id0bin[k_id_bin_len];
 	tf_ssb_id_str_to_bin(id0bin, id0);
-	tf_ssb_connect(ssb1, "127.0.0.1", 12347, id0bin, 0);
+	tf_ssb_connect(ssb1, "127.0.0.1", 12347, id0bin, 0, NULL, NULL);
 
 	tf_printf("Waiting for connection.\n");
 	while (test.connection_count0 != 1 || test.connection_count1 != 1)
@@ -480,8 +482,8 @@ void tf_ssb_test_rooms(const tf_test_options_t* options)
 
 	uint8_t id0bin[k_id_bin_len];
 	tf_ssb_id_str_to_bin(id0bin, id0);
-	tf_ssb_connect(ssb1, "127.0.0.1", 12347, id0bin, 0);
-	tf_ssb_connect(ssb2, "127.0.0.1", 12347, id0bin, 0);
+	tf_ssb_connect(ssb1, "127.0.0.1", 12347, id0bin, 0, NULL, NULL);
+	tf_ssb_connect(ssb2, "127.0.0.1", 12347, id0bin, 0, NULL, NULL);
 
 	tf_printf("Waiting for connection.\n");
 	while (test.connection_count0 != 2 || test.connection_count1 != 1 || test.connection_count2 != 1)
@@ -712,7 +714,7 @@ void tf_ssb_test_bench(const tf_test_options_t* options)
 	tf_ssb_register(tf_ssb_get_context(ssb1), ssb1);
 
 	tf_ssb_server_open(ssb0, 12347);
-	tf_ssb_connect(ssb1, "127.0.0.1", 12347, id0bin, 0);
+	tf_ssb_connect(ssb1, "127.0.0.1", 12347, id0bin, 0, NULL, NULL);
 
 	tf_printf("Waiting for messages.\n");
 	clock_gettime(CLOCK_REALTIME, &start_time);
@@ -881,8 +883,8 @@ void tf_ssb_test_go_ssb_room(const tf_test_options_t* options)
 
 	tf_ssb_add_broadcasts_changed_callback(ssb0, _ssb_test_room_broadcasts_changed, NULL, NULL);
 
-	tf_ssb_connect_str(ssb0, "net:linode.unprompted.com:8008~shs:Q0pc/7kXQJGIlqJxuwayL2huayzddgkVDoGkYVWQS1Y=:SSB+Room+PSK3TLYC2T86EHQCUHBUHASCASE18JBV24=", 0);
-	tf_ssb_connect_str(ssb1, "net:linode.unprompted.com:8008~shs:Q0pc/7kXQJGIlqJxuwayL2huayzddgkVDoGkYVWQS1Y=:SSB+Room+PSK3TLYC2T86EHQCUHBUHASCASE18JBV24=", 0);
+	tf_ssb_connect_str(ssb0, "net:linode.unprompted.com:8008~shs:Q0pc/7kXQJGIlqJxuwayL2huayzddgkVDoGkYVWQS1Y=:SSB+Room+PSK3TLYC2T86EHQCUHBUHASCASE18JBV24=", 0, NULL, NULL);
+	tf_ssb_connect_str(ssb1, "net:linode.unprompted.com:8008~shs:Q0pc/7kXQJGIlqJxuwayL2huayzddgkVDoGkYVWQS1Y=:SSB+Room+PSK3TLYC2T86EHQCUHBUHASCASE18JBV24=", 0, NULL, NULL);
 
 	uv_run(&loop, UV_RUN_DEFAULT);
 
@@ -900,8 +902,7 @@ static void _write_file(const char* path, const char* contents)
 	FILE* file = fopen(path, "w");
 	if (!file)
 	{
-		printf("Unable to write %s: %s.\n", path, strerror(errno));
-		fflush(stdout);
+		tf_printf("Unable to write %s: %s.\n", path, strerror(errno));
 		abort();
 	}
 	fputs(contents, file);
@@ -931,7 +932,7 @@ void tf_ssb_test_encrypt(const tf_test_options_t* options)
 	int result = system(command);
 	(void)result;
 	assert(WIFEXITED(result));
-	printf("returned %d\n", WEXITSTATUS(result));
+	tf_printf("returned %d\n", WEXITSTATUS(result));
 	assert(WEXITSTATUS(result) == 0);
 }
 
@@ -982,8 +983,8 @@ void tf_ssb_test_peer_exchange(const tf_test_options_t* options)
 	tf_ssb_whoami(ssb0, id0, sizeof(id0));
 	uint8_t id0bin[k_id_bin_len];
 	tf_ssb_id_str_to_bin(id0bin, id0);
-	tf_ssb_connect(ssb1, "127.0.0.1", 12347, id0bin, 0);
-	tf_ssb_connect(ssb2, "127.0.0.1", 12347, id0bin, 0);
+	tf_ssb_connect(ssb1, "127.0.0.1", 12347, id0bin, 0, NULL, NULL);
+	tf_ssb_connect(ssb2, "127.0.0.1", 12347, id0bin, 0, NULL, NULL);
 
 	while (_count_broadcasts(ssb0) != 2 || _count_broadcasts(ssb1) != 1 || _count_broadcasts(ssb2) != 1)
 	{
@@ -1015,6 +1016,8 @@ void tf_ssb_test_publish(const tf_test_options_t* options)
 	char id[k_id_base64_len] = { 0 };
 	tf_ssb_whoami(ssb, id, sizeof(id));
 
+	tf_ssb_destroy(ssb);
+
 	char executable[1024];
 	size_t size = sizeof(executable);
 	uv_exepath(executable, &size);
@@ -1024,19 +1027,198 @@ void tf_ssb_test_publish(const tf_test_options_t* options)
 	int result = system(command);
 	(void)result;
 	assert(WIFEXITED(result));
-	printf("returned %d\n", WEXITSTATUS(result));
+	tf_printf("returned %d\n", WEXITSTATUS(result));
 	assert(WEXITSTATUS(result) == 0);
 
 	snprintf(command, sizeof(command), "%s publish -d out/test_db0.sqlite -u :admin -i %s -c '{\"type\": \"post\", \"text\": \"Two.\"}'", executable, id);
 	result = system(command);
 	assert(WIFEXITED(result));
-	printf("returned %d\n", WEXITSTATUS(result));
+	tf_printf("returned %d\n", WEXITSTATUS(result));
 	assert(WEXITSTATUS(result) == 0);
 
-	tf_ssb_destroy(ssb);
-
 	uv_run(&loop, UV_RUN_DEFAULT);
 	uv_loop_close(&loop);
 }
 
+static void _test_print_identity(const char* identity, void* user_data)
+{
+	tf_ssb_t* ssb = user_data;
+	int64_t sequence = -1;
+	char id[k_id_base64_len] = { 0 };
+	snprintf(id, sizeof(id), "@%s", identity);
+	tf_ssb_db_get_latest_message_by_author(ssb, id, &sequence, NULL, 0);
+	tf_printf("IDENTITY %s: %d\n", id, (int)sequence);
+}
+
+void tf_ssb_test_replicate(const tf_test_options_t* options)
+{
+	tf_printf("Testing replication.\n");
+
+	uv_loop_t loop = { 0 };
+	uv_loop_init(&loop);
+
+	unlink("out/test_db0.sqlite");
+	tf_ssb_t* ssb0 = tf_ssb_create(&loop, NULL, "file:out/test_db0.sqlite", NULL);
+	tf_ssb_register(tf_ssb_get_context(ssb0), ssb0);
+	unlink("out/test_db1.sqlite");
+	tf_ssb_t* ssb1 = tf_ssb_create(&loop, NULL, "file:out/test_db1.sqlite", NULL);
+	tf_ssb_register(tf_ssb_get_context(ssb1), ssb1);
+
+	uv_idle_t idle0 = { .data = ssb0 };
+	uv_idle_init(&loop, &idle0);
+	uv_idle_start(&idle0, _ssb_test_idle);
+
+	uv_idle_t idle1 = { .data = ssb1 };
+	uv_idle_init(&loop, &idle1);
+	uv_idle_start(&idle1, _ssb_test_idle);
+
+	test_t test = {
+		.ssb0 = ssb0,
+		.ssb1 = ssb1,
+	};
+
+	tf_ssb_add_connections_changed_callback(ssb0, _ssb_test_connections_changed, NULL, &test);
+	tf_ssb_add_connections_changed_callback(ssb1, _ssb_test_connections_changed, NULL, &test);
+
+	tf_ssb_generate_keys(ssb0);
+	tf_ssb_generate_keys(ssb1);
+
+	uint8_t priv0[crypto_sign_SECRETKEYBYTES] = { 0 };
+	uint8_t priv1[crypto_sign_SECRETKEYBYTES] = { 0 };
+	tf_ssb_get_private_key(ssb0, priv0, sizeof(priv0));
+	tf_ssb_get_private_key(ssb1, priv1, sizeof(priv1));
+
+	char id0[k_id_base64_len] = { 0 };
+	char id1[k_id_base64_len] = { 0 };
+	bool b = tf_ssb_whoami(ssb0, id0, sizeof(id0));
+	(void)b;
+	assert(b);
+	b = tf_ssb_whoami(ssb1, id1, sizeof(id1));
+	assert(b);
+	tf_printf("ID %s and %s\n", id0, id1);
+
+	char priv0_str[512] = { 0 };
+	char priv1_str[512] = { 0 };
+	tf_base64_encode(priv0, sizeof(priv0), priv0_str, sizeof(priv0_str));
+	tf_base64_encode(priv1, sizeof(priv0), priv1_str, sizeof(priv1_str));
+	tf_ssb_db_identity_add(ssb0, "test", id0 + 1, priv0_str);
+	tf_ssb_db_identity_add(ssb1, "test", id1 + 1, priv1_str);
+
+	static const int k_key_count = 5;
+	char public[k_key_count][k_id_base64_len - 1];
+	char private[k_key_count][512];
+	for (int i = 0; i < k_key_count; i++)
+	{
+		tf_ssb_generate_keys_buffer(public[i], sizeof(public[i]), private[i], sizeof(private[i]));
+		bool added = tf_ssb_db_identity_add(ssb0, "test", public[i], private[i]);
+		tf_printf("%s user %d = %s private=%s\n", added ? "added" : "failed", i, public[i], private[i]);
+	}
+
+	JSContext* context0 = tf_ssb_get_context(ssb0);
+	for (int i = 0; i < k_key_count - 1; i++)
+	{
+		JSValue obj = JS_NewObject(context0);
+		JS_SetPropertyStr(context0, obj, "type", JS_NewString(context0, "contact"));
+		char self[k_id_base64_len];
+		snprintf(self, sizeof(self), "@%s", public[i]);
+		char contact[k_id_base64_len];
+		snprintf(contact, sizeof(contact), "@%s", public[i + 1]);
+		JS_SetPropertyStr(context0, obj, "contact", JS_NewString(context0, contact));
+		JS_SetPropertyStr(context0, obj, "following", JS_TRUE);
+		bool stored = false;
+		uint8_t private_bin[512] = { 0 };
+		tf_base64_decode(private[i], strlen(private[i]) - strlen(".ed25519"), private_bin, sizeof(private_bin));
+		tf_printf("ssb0 %s following %s\n", self, contact);
+		JSValue signed_message = tf_ssb_sign_message(ssb0, self, private_bin, obj, NULL, 0);
+		tf_ssb_verify_strip_and_store_message(ssb0, signed_message, _message_stored, &stored);
+		JS_FreeValue(context0, signed_message);
+		_wait_stored(ssb0, &stored);
+		JS_FreeValue(context0, obj);
+
+		obj = JS_NewObject(context0);
+		JS_SetPropertyStr(context0, obj, "type", JS_NewString(context0, "post"));
+		JS_SetPropertyStr(context0, obj, "text", JS_NewString(context0, "Hello, world!"));
+		stored = false;
+		signed_message = tf_ssb_sign_message(ssb0, self, private_bin, obj, NULL, 0);
+		tf_ssb_verify_strip_and_store_message(ssb0, signed_message, _message_stored, &stored);
+		JS_FreeValue(context0, signed_message);
+		_wait_stored(ssb0, &stored);
+		JS_FreeValue(context0, obj);
+	}
+
+	JSContext* context1 = tf_ssb_get_context(ssb1);
+	{
+		JSValue obj = JS_NewObject(context1);
+		JS_SetPropertyStr(context1, obj, "type", JS_NewString(context1, "contact"));
+		char self[k_id_base64_len];
+		snprintf(self, sizeof(self), "%s", id1);
+		char contact[k_id_base64_len];
+		snprintf(contact, sizeof(contact), "@%s", public[0]);
+		JS_SetPropertyStr(context1, obj, "contact", JS_NewString(context1, contact));
+		JS_SetPropertyStr(context1, obj, "following", JS_TRUE);
+		bool stored = false;
+		tf_printf("ssb1 %s following %s\n", self, contact);
+		JSValue signed_message = tf_ssb_sign_message(ssb1, self, priv1, obj, NULL, 0);
+		tf_ssb_verify_strip_and_store_message(ssb1, signed_message, _message_stored, &stored);
+		JS_FreeValue(context1, signed_message);
+		_wait_stored(ssb1, &stored);
+		JS_FreeValue(context1, obj);
+	}
+
+	tf_printf("ssb0\n");
+	tf_ssb_db_identity_visit_all(ssb0, _test_print_identity, ssb0);
+	tf_printf("ssb1\n");
+	tf_ssb_db_identity_visit_all(ssb1, _test_print_identity, ssb1);
+
+	tf_ssb_server_open(ssb0, 12347);
+
+	uint8_t id0bin[k_id_bin_len];
+	tf_ssb_id_str_to_bin(id0bin, id0);
+	tf_ssb_connect(ssb1, "127.0.0.1", 12347, id0bin, 0, NULL, NULL);
+
+	tf_printf("Waiting for connection.\n");
+	while (test.connection_count0 != 1 || test.connection_count1 != 1)
+	{
+		tf_ssb_set_main_thread(ssb0, true);
+		tf_ssb_set_main_thread(ssb1, true);
+		uv_run(&loop, UV_RUN_ONCE);
+		tf_ssb_set_main_thread(ssb0, false);
+		tf_ssb_set_main_thread(ssb1, false);
+	}
+	tf_ssb_server_close(ssb0);
+
+	int count1 = 0;
+	tf_ssb_add_message_added_callback(ssb1, _message_added, NULL, &count1);
+	tf_printf("Waiting for message from other.\n");
+	while (count1 != 4)
+	{
+		tf_ssb_set_main_thread(ssb1, true);
+		uv_run(&loop, UV_RUN_ONCE);
+		tf_ssb_set_main_thread(ssb1, false);
+	}
+	tf_ssb_remove_message_added_callback(ssb1, _message_added, &count1);
+	tf_printf("done\n");
+
+	tf_ssb_send_close(ssb1);
+
+	uv_close((uv_handle_t*)&idle0, NULL);
+	uv_close((uv_handle_t*)&idle1, NULL);
+
+	tf_printf("final run\n");
+	tf_ssb_set_main_thread(ssb0, true);
+	tf_ssb_set_main_thread(ssb1, true);
+	uv_run(&loop, UV_RUN_DEFAULT);
+	tf_ssb_set_main_thread(ssb0, false);
+	tf_ssb_set_main_thread(ssb1, false);
+	tf_printf("done\n");
+
+	tf_printf("destroy 0\n");
+	tf_ssb_destroy(ssb0);
+	tf_printf("destroy 1\n");
+	tf_ssb_destroy(ssb1);
+
+	tf_printf("close\n");
+	uv_loop_close(&loop);
+}
+
 #endif

src/ssb.tests.h

@@ -65,4 +65,10 @@ void tf_ssb_test_peer_exchange(const tf_test_options_t* options);
 */
 void tf_ssb_test_publish(const tf_test_options_t* options);
 
+/**
+** Test replication.
+** @param options The test options.
+*/
+void tf_ssb_test_replicate(const tf_test_options_t* options);
+
 /** @} */

src/task.c

@@ -43,10 +43,6 @@
 #include <malloc.h>
 #endif
 
-#if !defined(_countof)
-#define _countof(a) ((int)(sizeof((a)) / sizeof(*(a))))
-#endif
-
 static JSClassID _import_class_id;
 static int _count;
 
@@ -919,7 +915,7 @@ char* tf_task_get_hitches(tf_task_t* task)
 	JSContext* context = task->_context;
 	tf_trace_begin(task->_trace, __func__);
 	JSValue object = JS_NewObject(context);
-	for (int i = 0; i < (int)_countof(task->hitches); i++)
+	for (int i = 0; i < tf_countof(task->hitches); i++)
 	{
 		if (*task->hitches[i].name)
 		{
@@ -1342,8 +1338,7 @@ void tf_task_resolve_promise(tf_task_t* task, promiseid_t promise, JSValue value
 	}
 	else
 	{
-		tf_printf("Didn't find promise %d to resolve.\n", promise);
-		abort();
+		tf_printf("WARNING: Didn't find promise %d to resolve.\n", promise);
 	}
 }
 
@@ -1372,8 +1367,7 @@ void tf_task_reject_promise(tf_task_t* task, promiseid_t promise, JSValue value)
 	}
 	else
 	{
-		tf_printf("Didn't find promise %d to reject.\n", promise);
-		abort();
+		tf_printf("WARNING: Didn't find promise %d to reject.\n", promise);
 	}
 }
 
@@ -1658,13 +1652,13 @@ static void _tf_task_trace_to_parent(tf_trace_t* trace, const char* buffer, size
 static void _tf_task_record_hitch(const char* name, uint64_t duration_ns, void* user_data)
 {
 	tf_task_t* task = user_data;
-	for (int i = 0; i < (int)_countof(task->hitches); i++)
+	for (int i = 0; i < tf_countof(task->hitches); i++)
 	{
 		if (duration_ns > task->hitches[i].duration_ns)
 		{
-			if (i + 1 < (int)_countof(task->hitches))
+			if (i + 1 < tf_countof(task->hitches))
 			{
-				memmove(task->hitches + i + 1, task->hitches + i, sizeof(hitch_t) * ((int)_countof(task->hitches) - i - 1));
+				memmove(task->hitches + i + 1, task->hitches + i, sizeof(hitch_t) * (tf_countof(task->hitches) - i - 1));
 			}
 			snprintf(task->hitches[i].name, sizeof(task->hitches[i].name), "%s", name);
 			task->hitches[i].duration_ns = duration_ns;
@@ -1869,7 +1863,6 @@ void tf_task_destroy(tf_task_t* task)
 	{
 		JSValue global = JS_GetGlobalObject(task->_context);
 		JS_SetPropertyStr(task->_context, global, "httpd", JS_UNDEFINED);
-		JS_SetPropertyStr(task->_context, global, "gProcesses", JS_NewObject(task->_context));
 		JS_FreeValue(task->_context, global);
 	}
 

src/taskstub.js.c

@@ -62,6 +62,7 @@ static JSValue _taskstub_set_on_print(JSContext* context, JSValueConst this_val,
 static JSValue _taskstub_loadFile(JSContext* context, JSValueConst this_val, int argc, JSValueConst* argv);
 static void _taskstub_on_process_exit(uv_process_t* process, int64_t status, int terminationSignal);
 static void _taskstub_finalizer(JSRuntime* runtime, JSValue value);
+static void _taskstub_cleanup(tf_taskstub_t* stub);
 
 static void _tf_taskstub_run_sandbox_thread(void* data)
 {
@@ -74,12 +75,60 @@ static void _tf_taskstub_run_sandbox_thread(void* data)
 	tf_task_destroy(task);
 }
 
+static void _taskstub_on_handle_close(uv_handle_t* handle)
+{
+	tf_taskstub_t* stub = handle->data;
+	tf_task_remove_child(stub->_owner, stub);
+	handle->data = NULL;
+	_taskstub_cleanup(stub);
+}
+
+static void _tf_taskstub_on_exit(tf_taskstub_t* stub, int64_t status, int termination_signal)
+{
+	JSContext* context = tf_task_get_context(stub->_owner);
+	if (!JS_IsUndefined(stub->_on_exit))
+	{
+		JSValue ref = JS_DupValue(context, stub->_on_exit);
+		JSValue argv[] = { JS_NewInt64(context, status), JS_NewInt32(context, termination_signal) };
+		JSValue result = JS_Call(context, stub->_on_exit, JS_NULL, 2, argv);
+		tf_util_report_error(context, result);
+		JS_FreeValue(context, result);
+		JS_FreeValue(context, argv[0]);
+		JS_FreeValue(context, argv[1]);
+		JS_FreeValue(context, ref);
+	}
+	if (stub->_stream)
+	{
+		tf_packetstream_destroy(stub->_stream);
+		stub->_stream = NULL;
+	}
+	tf_task_remove_child(stub->_owner, stub);
+	if (stub->_process.data)
+	{
+		uv_close((uv_handle_t*)&stub->_process, _taskstub_on_handle_close);
+	}
+	else
+	{
+		_taskstub_cleanup(stub);
+	}
+}
+
+static void _tf_taskstub_packetstream_close(void* user_data)
+{
+	tf_taskstub_t* stub = user_data;
+	if (!stub->_process.data)
+	{
+		_tf_taskstub_on_exit(stub, -1, -1);
+	}
+}
+
 static JSValue _taskstub_create(JSContext* context, JSValueConst this_val, int argc, JSValueConst* argv)
 {
 	tf_task_t* parent = tf_task_get(context);
 	tf_taskstub_t* stub = tf_malloc(sizeof(tf_taskstub_t));
 	memset(stub, 0, sizeof(*stub));
 	stub->_stream = tf_packetstream_create();
+	tf_packetstream_set_on_close(stub->_stream, _tf_taskstub_packetstream_close, stub);
 
 	JSValue taskObject = JS_NewObjectClass(context, _classId);
 	JS_SetOpaque(taskObject, stub);
@@ -314,35 +363,10 @@ static void _taskstub_finalizer(JSRuntime* runtime, JSValue value)
 	_taskstub_cleanup(stub);
 }
 
-static void _taskstub_on_handle_close(uv_handle_t* handle)
-{
-	tf_taskstub_t* stub = handle->data;
-	tf_task_remove_child(stub->_owner, stub);
-	handle->data = NULL;
-	_taskstub_cleanup(stub);
-}
-
-static void _taskstub_on_process_exit(uv_process_t* process, int64_t status, int terminationSignal)
+static void _taskstub_on_process_exit(uv_process_t* process, int64_t status, int termination_signal)
 {
 	tf_taskstub_t* stub = process->data;
-	JSContext* context = tf_task_get_context(stub->_owner);
-	if (!JS_IsUndefined(stub->_on_exit))
-	{
-		JSValue ref = JS_DupValue(context, stub->_on_exit);
-		JSValue argv[] = { JS_NewInt32(context, status), JS_NewInt32(context, terminationSignal) };
-		JSValue result = JS_Call(context, stub->_on_exit, JS_NULL, 2, argv);
-		tf_util_report_error(context, result);
-		JS_FreeValue(context, result);
-		JS_FreeValue(context, argv[0]);
-		JS_FreeValue(context, argv[1]);
-		JS_FreeValue(context, ref);
-	}
-	if (stub->_stream)
-	{
-		tf_packetstream_destroy(stub->_stream);
-		stub->_stream = NULL;
-	}
-	uv_close((uv_handle_t*)process, _taskstub_on_handle_close);
+	_tf_taskstub_on_exit(stub, status, termination_signal);
 }
 
 static JSValue _taskstub_getExports(JSContext* context, JSValueConst this_val, int argc, JSValueConst* argv)

src/tests.c

@@ -4,10 +4,13 @@
 #include "http.h"
 #include "log.h"
 #include "mem.h"
+#include "ssb.db.h"
+#include "ssb.h"
 #include "ssb.tests.h"
 #include "util.js.h"
 
 #include <assert.h>
+#include <inttypes.h>
 #include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -37,8 +40,7 @@ static void _write_file(const char* path, const char* contents)
 	FILE* file = fopen(path, "w");
 	if (!file)
 	{
-		printf("Unable to write %s: %s.\n", path, strerror(errno));
-		fflush(stdout);
+		tf_printf("Unable to write %s: %s.\n", path, strerror(errno));
 		abort();
 	}
 	fputs(contents, file);
@@ -782,10 +784,148 @@ static void _test_http(const tf_test_options_t* options)
 	uv_thread_join(&thread);
 }
 
+static int _http_get_status_code(const char* url)
+{
+	char command[1024];
+	snprintf(command, sizeof(command), "curl -s -o /dev/null -w '%%{http_code}' \"%s\"", url);
+	char buffer[256] = "";
+	FILE* file = popen(command, "r");
+	char* result = fgets(buffer, sizeof(buffer), file);
+	pclose(file);
+	return result ? atoi(result) : -1;
+}
+
+static void _http_check_status_code(const char* url, int expected_code)
+{
+	char command[1024];
+	snprintf(command, sizeof(command), "curl -s -o /dev/null -w '%%{http_code}' \"%s\"", url);
+	char buffer[256] = "";
+	FILE* file = popen(command, "r");
+	char* result = fgets(buffer, sizeof(buffer), file);
+	tf_printf("%s => %s\n", command, result);
+	assert(atoi(buffer) == expected_code);
+	assert(file);
+	int status = pclose(file);
+	(void)status;
+	assert(WEXITSTATUS(status) == 0);
+}
+
+static void _http_check_body_contains(const char* url, const char* expected)
+{
+	char command[1024];
+	snprintf(command, sizeof(command), "curl -s \"%s\"", url);
+	char buffer[1024] = "";
+	FILE* file = popen(command, "r");
+	bool found = false;
+	while (!found)
+	{
+		char* result = fgets(buffer, sizeof(buffer), file);
+		if (!result)
+		{
+			break;
+		}
+		found = strstr(buffer, expected) != NULL;
+		if (found)
+		{
+			tf_printf("%s => found: \"%s\"\n", url, expected);
+		}
+	}
+	if (!found)
+	{
+		tf_printf("Didn't find \"%s\" in %s.\n", expected, url);
+	}
+	assert(found);
+	assert(file);
+	int status = pclose(file);
+	(void)status;
+	assert(WEXITSTATUS(status) == 0);
+}
+
+static void _test_httpd(const tf_test_options_t* options)
+{
+	uv_loop_t loop = { 0 };
+	uv_loop_init(&loop);
+
+	unlink("out/test_db0.sqlite");
+	char command[256];
+	snprintf(command, sizeof(command), "%s run -b 0 --db-path=out/test_db0.sqlite" TEST_ARGS, options->exe_path);
+
+	uv_stdio_container_t stdio[] = {
+		[STDIN_FILENO] = { .flags = UV_IGNORE },
+		[STDOUT_FILENO] = { .flags = UV_INHERIT_FD },
+		[STDERR_FILENO] = { .flags = UV_INHERIT_FD },
+	};
+	uv_process_t process = { 0 };
+	uv_spawn(&loop, &process,
+		&(uv_process_options_t) {
+			.file = options->exe_path,
+			.args = (char*[]) { (char*)options->exe_path, "run", "-b0", "--db-path=out/test_db0.sqlite", "--http-port=8080", "--https-port=0", NULL },
+			.stdio_count = sizeof(stdio) / sizeof(*stdio),
+			.stdio = stdio,
+		});
+
+	for (int i = 0; i < 100; i++)
+	{
+		if (_http_get_status_code("http://localhost:8080/debug") == 200)
+		{
+			break;
+		}
+		uv_sleep(1000);
+	}
+
+#if !defined(__HAIKU__)
+	tf_ssb_t* ssb = tf_ssb_create(&loop, NULL, "file:out/test_db0.sqlite", NULL);
+	const char* app_id = tf_ssb_db_get_property(ssb, "core", "path:test");
+	tf_ssb_destroy(ssb);
+#endif
+
+	_http_check_status_code("http://localhost:8080/404", 404);
+	_http_check_status_code("http://localhost:8080/", 303);
+	_http_check_status_code("http://localhost:8080/~core/apps/", 200);
+	_http_check_status_code("http://localhost:8080/~core/apps", 303);
+	_http_check_status_code("http://localhost:8080/~core/apps/view", 200);
+	_http_check_body_contains("http://localhost:8080/~core/apps/", "<title>Tilde Friends</title>");
+	_http_check_body_contains("http://localhost:8080/~core/apps/view", "\"type\":\"tildefriends-app\"");
+	_http_check_body_contains("http://localhost:8080/~core/test/hello.txt", "Hello, world!");
+	_http_check_status_code("http://localhost:8080/~core/test/nonexistent.txt", 404);
+	_http_check_body_contains("http://localhost:8080/&MV9b23bQeMQ7isAGTkoBZGErH853yGk0W/yUx1iU7dM=.sha256/view", "Hello, world!");
+
+#if !defined(__HAIKU__)
+	char url[1024];
+	snprintf(url, sizeof(url), "http://localhost:8080/%s/", app_id);
+	_http_check_body_contains(url, "<title>Tilde Friends</title>");
+	snprintf(url, sizeof(url), "http://localhost:8080/%s/view", app_id);
+	_http_check_body_contains(url, "\"type\":\"tildefriends-app\"");
+	snprintf(url, sizeof(url), "http://localhost:8080/%s/hello.txt", app_id);
+	_http_check_body_contains(url, "Hello, world!");
+	tf_free((void*)app_id);
+#endif
+
+	uv_process_kill(&process, SIGTERM);
+	uv_close((uv_handle_t*)&process, NULL);
+	uv_run(&loop, UV_RUN_ONCE);
+
+	uv_loop_close(&loop);
+}
+
+static void _test_pattern(const tf_test_options_t* options)
+{
+	assert(tf_http_pattern_matches("/~core/test/", "/~core/test/"));
+	assert(tf_http_pattern_matches("/~core/test/*", "/~core/test/"));
+	assert(tf_http_pattern_matches("/~core/test/*", "/~core/test/blah"));
+	assert(tf_http_pattern_matches("*/~core/test/", "/~core/test/"));
+	assert(tf_http_pattern_matches("*/~core/test/", "blah/~core/test/"));
+	assert(tf_http_pattern_matches("/~*/*/", "/~core/test/"));
+	assert(tf_http_pattern_matches("/~{word}/*", "/~core/test"));
+	assert(tf_http_pattern_matches("/~{word}/{word}/", "/~core/test/"));
+	assert(tf_http_pattern_matches("/~{word}/{word}", "/~core/test"));
+	assert(!tf_http_pattern_matches("/~{word}/{word}", "/~foo/bar/baz"));
+}
+
 static void _test_auto_process_exit(uv_process_t* process, int64_t status, int termination_signal)
 {
-	tf_printf("Process exit %d signal=%d.\n", (int)WEXITSTATUS(status), termination_signal);
-	assert(WEXITSTATUS(status) == 0);
+	tf_printf("Process exit %" PRId64 " signal=%d.\n", status, termination_signal);
+	assert(status == 0);
 	process->data = NULL;
 	uv_close((uv_handle_t*)process, NULL);
 }
@@ -822,6 +962,7 @@ static void _test_auto(const tf_test_options_t* options)
 	int spawn_result = uv_spawn(&loop, &process, &process_options);
 	if (spawn_result)
 	{
+		tf_printf("uv_spawn: %s\n", uv_strerror(spawn_result));
 		abort();
 	}
 
@@ -834,6 +975,7 @@ static void _test_auto(const tf_test_options_t* options)
 	spawn_result = uv_spawn(&loop, &selenium, &process_options);
 	if (spawn_result)
 	{
+		tf_printf("uv_spawn: %s\n", uv_strerror(spawn_result));
 		abort();
 	}
 
@@ -887,6 +1029,8 @@ void tf_tests(const tf_test_options_t* options)
 #if !TARGET_OS_IPHONE
 	_tf_test_run(options, "bip39", _test_bip39, false);
 	_tf_test_run(options, "http", _test_http, false);
+	_tf_test_run(options, "httpd", _test_httpd, false);
+	_tf_test_run(options, "pattern", _test_pattern, false);
 	_tf_test_run(options, "ssb", tf_ssb_test_ssb, false);
 	_tf_test_run(options, "ssb_id", tf_ssb_test_id_conversion, false);
 	_tf_test_run(options, "ssb_following", tf_ssb_test_following, false);
@@ -917,6 +1061,7 @@ void tf_tests(const tf_test_options_t* options)
 	_tf_test_run(options, "encrypt", tf_ssb_test_encrypt, false);
 	_tf_test_run(options, "peer_exchange", tf_ssb_test_peer_exchange, false);
 	_tf_test_run(options, "publish", tf_ssb_test_publish, false);
+	_tf_test_run(options, "replicate", tf_ssb_test_replicate, false);
 	tf_printf("Tests completed.\n");
 #endif
 }

src/trace.c

@@ -16,8 +16,6 @@
 #include <time.h>
 #include <unistd.h>
 
-#define tf_countof(a) ((int)(sizeof((a)) / sizeof(*(a))))
-
 enum
 {
 	k_buffer_size = 4 * 1024 * 1024,

src/util.js.c

@@ -304,6 +304,74 @@ static JSValue _util_parseHttpResponse(JSContext* context, JSValueConst this_val
 	return result;
 }
 
+#if defined(__APPLE__)
+#include <TargetConditionals.h>
+#endif
+
+static bool _is_mobile()
+{
+#if defined(__ANDROID__) || (defined(__APPLE__) && TARGET_OS_IPHONE)
+	return true;
+#else
+	return false;
+#endif
+}
+
+static JSValue _util_defaultGlobalSettings(JSContext* context, JSValueConst this_val, int argc, JSValueConst* argv)
+{
+	typedef struct _setting_t
+	{
+		const char* name;
+		const char* type;
+		const char* description;
+		JSValue default_value;
+	} setting_t;
+
+	const setting_t k_settings[] = {
+		{ .name = "code_of_conduct", .type = "textarea", .description = "Code of conduct presented at sign-in." },
+		{ .name = "blob_fetch_age_seconds",
+			.type = "integer",
+			.description = "Only blobs mentioned more recently than this age will be automatically fetched.",
+			.default_value = _is_mobile() ? JS_NewInt32(context, (int)(0.5f * 365 * 24 * 60 * 60)) : JS_UNDEFINED },
+		{ .name = "blob_expire_age_seconds",
+			.type = "integer",
+			.description = "Blobs older than this will be automatically deleted.",
+			.default_value = _is_mobile() ? JS_NewInt32(context, (int)(1.0f * 365 * 24 * 60 * 60)) : JS_UNDEFINED },
+		{ .name = "fetch_hosts", .type = "string", .description = "Comma-separated list of host names to which HTTP fetch requests are allowed.  None if empty." },
+		{ .name = "http_redirect", .type = "string", .description = "If connecting by HTTP and HTTPS is configured, Location header prefix (ie, \"http://example.com\")" },
+		{ .name = "index", .type = "string", .description = "Default path.", .default_value = JS_NewString(context, "/~core/apps") },
+		{ .name = "index_map", .type = "textarea", .description = "Mappings from hostname to redirect path, one per line, as in: \"www.tildefriends.net=/~core/index/\"" },
+		{ .name = "peer_exchange",
+			.type = "boolean",
+			.description = "Enable discovery of, sharing of, and connecting to internet peer strangers, including announcing this instance.",
+			.default_value = JS_FALSE },
+		{ .name = "replicator", .type = "boolean", .description = "Enable message and blob replication.", .default_value = JS_TRUE },
+		{ .name = "room", .type = "boolean", .description = "Enable peers to tunnel through this instance as a room.", .default_value = JS_TRUE },
+		{ .name = "room_name", .type = "string", .description = "Name of the room.", .default_value = JS_NewString(context, "tilde friends tunnel") },
+		{ .name = "seeds_host", .type = "string", .description = "Hostname for seed connections.", .default_value = JS_NewString(context, "seeds.tildefriends.net") },
+		{ .name = "account_registration", .type = "boolean", .description = "Allow registration of new accounts.", .default_value = JS_TRUE },
+		{ .name = "replication_hops",
+			.type = "integer",
+			.description = "Number of hops to replicate (1 = direct follows, 2 = follows of follows, etc.).",
+			.default_value = JS_NewInt32(context, 2) },
+		{ .name = "delete_stale_feeds",
+			.type = "boolean",
+			.description = "Periodically delete feeds that visible from local accounts and related follows.",
+			.default_value = JS_FALSE },
+	};
+
+	JSValue settings = JS_NewObject(context);
+	for (int i = 0; i < tf_countof(k_settings); i++)
+	{
+		JSValue entry = JS_NewObject(context);
+		JS_SetPropertyStr(context, entry, "type", JS_NewString(context, k_settings[i].type));
+		JS_SetPropertyStr(context, entry, "description", JS_NewString(context, k_settings[i].description));
+		JS_SetPropertyStr(context, entry, "default_value", k_settings[i].default_value);
+		JS_SetPropertyStr(context, settings, k_settings[i].name, entry);
+	}
+	return settings;
+}
+
 JSValue tf_util_new_uint8_array(JSContext* context, const uint8_t* data, size_t size)
 {
 	JSValue array_buffer = JS_NewArrayBufferCopy(context, data, size);
@@ -327,11 +395,17 @@ void tf_util_register(JSContext* context)
 	JS_SetPropertyStr(context, global, "bip39Bytes", JS_NewCFunction(context, _util_bip39_bytes, "bip39Bytes", 1));
 	JS_SetPropertyStr(context, global, "print", JS_NewCFunction(context, _util_print, "print", 1));
 	JS_SetPropertyStr(context, global, "parseHttpResponse", JS_NewCFunction(context, _util_parseHttpResponse, "parseHttpResponse", 2));
+	JS_SetPropertyStr(context, global, "defaultGlobalSettings", JS_NewCFunction(context, _util_defaultGlobalSettings, "defaultGlobalSettings", 2));
 	JS_FreeValue(context, global);
 }
 
 int tf_util_get_length(JSContext* context, JSValue value)
 {
+	if (JS_IsUndefined(value))
+	{
+		return 0;
+	}
+
 	JSValue length = JS_GetPropertyStr(context, value, "length");
 	int result = 0;
 	JS_ToInt32(context, &result, length);

src/util.js.h

@@ -150,4 +150,11 @@ const char* tf_util_function_to_string(void* function);
 		_a > _b ? _b : _a; \
 	})
 
+/**
+** Get the number of elements in an array.
+** @param a The array.
+** @return The number of array elements.
+*/
+#define tf_countof(a) ((int)(sizeof((a)) / sizeof(*(a))))
+
 /** @} */

src/version.h

@@ -1,2 +1,2 @@
-#define VERSION_NUMBER "0.0.24"
-#define VERSION_NAME "Honey bunches of boats."
+#define VERSION_NUMBER "0.0.26"
+#define VERSION_NAME "This program kills fascists."

tools/autotest.py

@@ -6,17 +6,62 @@ import sys
 import time
 
 if sys.platform == 'haiku1':
-    print('Automation tests are disabled on Haiku.')
-    exit(0)
+	print('Automation tests are disabled on Haiku.')
+	exit(0)
 
+import selenium
 from selenium import webdriver
 from selenium.webdriver.firefox.service import Service
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions
 from selenium.webdriver.support.ui import WebDriverWait
 
-def exists_in_shadow_root(shadow_root, by, value):
-	return lambda driver: shadow_root.find_element(by, value)
+def select(driver, path, action = None, keep_trying = False):
+	start_time = time.time()
+	done = False
+	while True:
+		try:
+			driver.switch_to.default_content()
+			context = driver
+			for node in path:
+				if node.startswith('#'):
+					context = context.find_element(By.ID, node[1:])
+				elif node.startswith('/'):
+					context = context.find_element(By.XPATH, node)
+				elif node.startswith('.'):
+					context = context.find_element(By.CLASS_NAME, node[1:])
+				elif node.startswith('='):
+					context = context.find_element(By.LINK_TEXT, node[1:])
+				elif node == 'frame':
+					driver.switch_to.frame(context)
+					context = driver
+				elif node == 'shadow_root':
+					context = context.shadow_root
+				else:
+					context = context.find_element(By.TAG_NAME, node)
+			if action is not None:
+				if action[0] == 'click':
+					context.click()
+				elif action[0] == 'send_keys':
+					context.send_keys(action[1])
+				elif action[0] == 'clear':
+					context.clear()
+				else:
+					raise RuntimeError(f'Unexpected action: {action}.')
+				done = True
+				if not keep_trying:
+					break
+			else:
+				return context
+		except (selenium.common.exceptions.NoSuchElementException,
+				selenium.common.exceptions.NoSuchShadowRootException,
+				selenium.common.exceptions.StaleElementReferenceException,
+				selenium.common.exceptions.WebDriverException):
+			if done and keep_trying:
+				break
+			if time.time() - start_time < 5.0:
+				time.sleep(0.1)
+				pass
 
 success = False
 try:
@@ -27,246 +72,177 @@ try:
 	wait = WebDriverWait(driver, 10)
 
 	driver.get('http://localhost:8888')
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.LINK_TEXT, 'login').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'register_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('adminuser')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('admin_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'confirm').send_keys('admin_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-	driver.switch_to.frame(driver.find_element(By.ID, 'document'))
-	wait.until(expected_conditions.presence_of_element_located((By.LINK_TEXT, 'identity')))
-	driver.switch_to.default_content()
+	select(driver, ['tf-navigation', 'shadow_root', '=login'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#register_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'adminuser'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'admin_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#confirm'], ('send_keys', 'admin_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['#document', 'frame', '=identity'])
 
 	driver.get('http://localhost:8888/~core/admin/')
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-	driver.switch_to.frame(driver.find_element(By.ID, 'document'))
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'gs_room_name'))).send_keys('test room')
-	wait.until(expected_conditions.presence_of_element_located((By.XPATH, '//*[@id="gs_room_name"]/following-sibling::button'))).click()
+	select(driver, ['#document', 'frame', '#gs_room_name'], ('send_keys', 'test room'))
+	select(driver, ['#document', 'frame', '//*[@id="gs_room_name"]/following-sibling::button'], ('click',))
+	select(driver, ['//button[text()="✅ Allow"]'], ('click',))
 	driver.switch_to.alert.accept()
-	driver.switch_to.default_content()
 
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'identity').click()
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'logout').click()
+	select(driver, ['tf-navigation', 'shadow_root', '#identity'], ('click',))
+	select(driver, ['tf-navigation', 'shadow_root', '#logout'], ('click',))
 
 	driver.get('http://localhost:8888')
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.LINK_TEXT, 'login').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'register_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('testuser')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'confirm').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
+	select(driver, ['tf-navigation', 'shadow_root', '=login'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#register_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'testuser'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#confirm'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['#document'])
+
+	select(driver, ['tf-navigation', 'shadow_root', '#create_identity'], ('click',))
+	wait.until(expected_conditions.alert_is_present()).accept()
+
+	select(driver, ['tf-navigation', 'shadow_root', '#identity'], ('click',))
+	select(driver, ['tf-navigation', 'shadow_root', '#id_dropdown', '//button[position()=2]'], ('click',))
+
+	select(driver, ['#document', 'frame', 'tf-app', 'shadow_root', '#tf-tab-news', 'shadow_root', '.tf-profile', 'shadow_root', '#edit_profile'], ('click',), keep_trying = True)
+	select(driver, ['#document', 'frame', 'tf-app', 'shadow_root', '#tf-tab-news', 'shadow_root', '.tf-profile', 'shadow_root', '#name'], ('send_keys', 'user'))
+	select(driver, ['#document', 'frame', 'tf-app', 'shadow_root', '#tf-tab-news', 'shadow_root', '.tf-profile', 'shadow_root', '#save_profile'], ('click',))
+
+	select(driver, ['//button[text()="✅ Allow"]'], ('click',))
 
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
 	driver.get('http://localhost:8888/~testuser/test/')
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-	while True:
-		try:
-			wait.until(exists_in_shadow_root(driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root, By.ID, 'close_error')).click()
-			break
-		except:
-			pass
+	select(driver, ['#document'])
+	select(driver, ['tf-navigation', 'shadow_root', '#close_error'], ('click',))
+	select(driver, ['tf-navigation', 'shadow_root', '=edit'], ('click',))
+	select(driver, ['#editor', '.cm-content'], ('click',))
+	select(driver, ['#editor', '.cm-content'], ('send_keys', 'app.setDocument(\n\t"<div id=\'test-div\'>Hello, world!</div>"\n);'))
+	select(driver, ['#save'], ('click',))
 
-	driver.switch_to.default_content()
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.LINK_TEXT, 'edit').click()
-	editor = driver.find_element(By.ID, 'editor').find_element(By.CLASS_NAME, 'cm-content')
-	editor.click()
-	editor.send_keys('app.setDocument("<div id=\'test-div\'>Hello, world!</div>")');
-	driver.find_element(By.ID, 'save').click()
+	select(driver, ['#document', 'frame', '#test-div'])
 
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-	driver.switch_to.frame(driver.find_element(By.ID, 'document'))
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'test-div')))
+	size = driver.get_window_size()
+	driver.set_window_size(1200, 540)
+	driver.save_screenshot('out/screenshot0.png')
+	driver.set_window_size(size['width'], size['height'])
 
-	driver.switch_to.default_content()
-	editor = driver.find_element(By.ID, 'editor').find_element(By.CLASS_NAME, 'cm-content')
-	editor.click()
-	editor.clear()
-	editor.send_keys('app.setDocument("<div id=\'test-div2\'>Hello, world, again!</div>")');
-	driver.find_element(By.ID, 'save').click()
+	select(driver, ['#editor', '.cm-content'], ('click',))
+	select(driver, ['#editor', '.cm-content'], ('clear',))
+	select(driver, ['#editor', '.cm-content'], ('send_keys', 'app.setDocument("<div id=\'test-div2\'>Hello, world, again!</div>")'))
+	select(driver, ['#save'], ('click',))
+	select(driver, ['#document', 'frame', '#test-div2'])
 
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-	driver.switch_to.frame(driver.find_element(By.ID, 'document'))
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'test-div2')))
-
-	driver.switch_to.default_content()
-	driver.find_element(By.ID, 'delete').click()
+	select(driver, ['#delete'], ('click',))
 	wait.until(expected_conditions.alert_is_present()).accept()
 	wait.until(expected_conditions.alert_is_present()).dismiss()
 	driver.get('http://localhost:8888/~testuser/test/')
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-	while True:
-		try:
-			wait.until(exists_in_shadow_root(driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root, By.ID, 'close_error')).click()
-			break
-		except:
-			pass
+
+	select(driver, ['#document'])
+	select(driver, ['tf-navigation', 'shadow_root', '#close_error'], ('click',))
+	select(driver, ['tf-navigation', 'shadow_root', '=edit'], ('click',))
 
 	driver.get('http://localhost:8888')
 
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-	driver.switch_to.frame(driver.find_element(By.ID, 'document'))
-	wait.until(expected_conditions.presence_of_element_located((By.LINK_TEXT, 'identity')))
-	driver.switch_to.default_content()
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'create_identity').click()
-	wait.until(expected_conditions.alert_is_present()).accept()
+	select(driver, ['#document', 'frame', '=identity'])
 
-	driver.switch_to.frame(driver.find_element(By.ID, 'document'))
-	wait.until(expected_conditions.presence_of_element_located((By.LINK_TEXT, 'identity'))).click()
+	size = driver.get_window_size()
+	driver.set_window_size(540, 1200)
+	driver.save_screenshot('out/screenshot1.png')
+	driver.set_window_size(size['width'], size['height'])
 
-	# StaleElementReferenceException
-	while True:
-		try:
-			driver.switch_to.default_content()
-			wait.until(expected_conditions.presence_of_element_located((By.ID, 'content')))
-			driver.switch_to.frame(wait.until(expected_conditions.presence_of_element_located((By.ID, 'document'))))
-			wait.until(expected_conditions.presence_of_element_located((By.ID, 'create_id'))).click()
-			driver.switch_to.alert.accept()
-			break
-		except:
-			pass
+	select(driver, ['#document', 'frame', '=identity'], ('click',))
 
-	# StaleElementReferenceException
-	while True:
-		try:
-			id0 = wait.until(expected_conditions.presence_of_element_located((By.TAG_NAME, 'li'))).text.split(' ')[-1]
-			break
-		except:
-			pass
-	wait.until(expected_conditions.presence_of_element_located((By.XPATH, '//li/button[text()="Export Identity"]'))).click()
-	driver.switch_to.default_content()
-	wait.until(expected_conditions.presence_of_element_located((By.XPATH, '//button[text()="✅ Allow"]'))).click()
-	driver.switch_to.frame(wait.until(expected_conditions.presence_of_element_located((By.ID, 'document'))))
-	words = wait.until(expected_conditions.presence_of_element_located((By.XPATH, '//li//textarea'))).get_attribute('value')
-	wait.until(expected_conditions.presence_of_element_located((By.XPATH, '//li/button[text()="Delete Identity"]'))).click()
+	select(driver, ['#document', 'frame', '#create_id'], ('click',))
+	id0 = select(driver, ['#document', 'frame', 'li']).text.split(' ')[-1]
+
+	select(driver, ['#document', 'frame', '//li/button[text()="Export Identity"]'], ('click',))
+	select(driver, ['//button[text()="✅ Allow"]'], ('click',))
+	words = select(driver, ['#document', 'frame', '//li//textarea']).get_attribute('value')
+	select(driver, ['#document', 'frame', '//li/button[text()="Delete Identity"]'], ('click',))
 	driver.switch_to.alert.send_keys('DELETE')
 	driver.switch_to.alert.accept()
-	driver.switch_to.default_content()
-	wait.until(expected_conditions.presence_of_element_located((By.XPATH, '//button[text()="✅ Allow"]'))).click()
+	select(driver, ['//button[text()="✅ Allow"]'], ('click',))
+	driver.switch_to.alert.accept()
+	words = select(driver, ['#document', 'frame', '//textarea'], ('send_keys', words))
+	select(driver, ['#document', 'frame', '//button[text()="Import Identity"]'], ('click',))
+	select(driver, ['//button[text()="✅ Allow"]'], ('click',))
 	driver.switch_to.alert.accept()
 	driver.switch_to.frame(wait.until(expected_conditions.presence_of_element_located((By.ID, 'document'))))
-	words = wait.until(expected_conditions.presence_of_element_located((By.XPATH, '//textarea'))).send_keys(words)
-	wait.until(expected_conditions.presence_of_element_located((By.XPATH, '//button[text()="Import Identity"]'))).click()
-	driver.switch_to.default_content()
-	wait.until(expected_conditions.presence_of_element_located((By.XPATH, '//button[text()="✅ Allow"]'))).click()
-	driver.switch_to.alert.accept()
-	driver.switch_to.frame(wait.until(expected_conditions.presence_of_element_located((By.ID, 'document'))))
-	id1 = wait.until(expected_conditions.presence_of_element_located((By.TAG_NAME, 'li'))).text.split(' ')[-1]
+	id1 = select(driver, ['#document', 'frame', 'li']).text.split(' ')[-1]
+	assert id0 == id1
 
 	driver.get('http://localhost:8888')
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-	driver.switch_to.frame(driver.find_element(By.ID, 'document'))
-	wait.until(expected_conditions.presence_of_element_located((By.LINK_TEXT, 'ssb'))).click()
-	driver.switch_to.default_content()
+	select(driver, ['#document', 'frame', '=ssb'], ('click',))
+	select(driver, ['#document', 'frame', 'tf-app', 'shadow_root', '#tf-tab-news', 'shadow_root', '#tf-compose', 'shadow_root', '#edit'], ('send_keys', 'Hello, world!'))
+	select(driver, ['#document', 'frame', 'tf-app', 'shadow_root', '#tf-tab-news', 'shadow_root', '#tf-compose', 'shadow_root', '#submit'], ('click',))
+	select(driver, ['//button[text()="✅ Allow"]'], ('click',))
 
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'content')))
+	select(driver, ['tf-navigation', 'shadow_root', '#identity'], ('click',))
+	select(driver, ['tf-navigation', 'shadow_root', '#logout'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#login_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'testuser'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
 
-	# StaleElementReferenceException
-	while True:
-		try:
-			driver.switch_to.frame(wait.until(expected_conditions.presence_of_element_located((By.ID, 'document'))))
-			break
-		except:
-			pass
-	# NoSuchShadowRootException
-	while True:
-		try:
-			tf_app = wait.until(expected_conditions.presence_of_element_located((By.TAG_NAME, 'tf-app'))).shadow_root
-			break
-		except:
-			pass
+	select(driver, ['tf-navigation', 'shadow_root', '#identity'], ('click',))
+	select(driver, ['tf-navigation', 'shadow_root', '#logout'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#guest_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#guestButton'], ('click',))
+	select(driver, ['#document', 'frame', 'tf-app', 'shadow_root'])
 
-	# WebDriverException (shadow root is detached)
-	while True:
-		try:
-			tf_tab_news = wait.until(exists_in_shadow_root(tf_app, By.ID, 'tf-tab-news')).shadow_root
-			tf_tab_news.find_element(By.ID, 'tf-compose').shadow_root.find_element(By.ID, 'edit').send_keys('Hello, world!')
-			tf_tab_news.find_element(By.ID, 'tf-compose').shadow_root.find_element(By.ID, 'submit').click()
-			break
-		except:
-			pass
+	select(driver, ['tf-navigation', 'shadow_root', '#logout'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#login_label'], ('click',))
 
-	driver.switch_to.default_content()
-	driver.find_element(By.ID, 'allow').click()
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'testuser'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'wrong_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#error'])
 
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'identity').click()
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'logout').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'login_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('testuser')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'wrong_user'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#error'])
 
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'content')))
+	select(driver, ['tf-auth', 'shadow_root', '#register_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'testuser'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'wrong_test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#confirm'], ('send_keys', 'wrong_test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#error'])
 
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'identity').click()
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'logout').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'guest_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'guestButton').click()
+	select(driver, ['tf-auth', 'shadow_root', '#register_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'testuser'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#confirm'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#error'])
 
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'content')))
-	driver.switch_to.frame(wait.until(expected_conditions.presence_of_element_located((By.ID, 'document'))))
-	# NoSuchShadowRootException
-	while True:
-		try:
-			tf_app = wait.until(expected_conditions.presence_of_element_located((By.TAG_NAME, 'tf-app'))).shadow_root
-			break
-		except:
-			pass
-	driver.switch_to.default_content()
+	select(driver, ['tf-auth', 'shadow_root', '#register_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', '😁'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#confirm'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#error'])
 
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'logout').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'login_label').click()
+	select(driver, ['tf-auth', 'shadow_root', '#change_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'testuser'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#new_password'], ('send_keys', 'new_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#confirm'], ('send_keys', 'new_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['#document', 'frame'])
 
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('testuser')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('wrong_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'error')
-
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('wrong_user')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'error')
-
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'register_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('testuser')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('wrong_test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'confirm').send_keys('wrong_test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'error')
-
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'register_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('1invalid')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'confirm').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'error')
-
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'register_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('😁')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'confirm').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'error')
-
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'change_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('testuser')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'new_password').send_keys('new_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'confirm').send_keys('new_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'identity').click()
-	driver.find_element(By.TAG_NAME, 'tf-navigation').shadow_root.find_element(By.ID, 'logout').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'login_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('testuser')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('test_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'error')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'login_label').click()
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'name').send_keys('testuser')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'password').send_keys('new_password')
-	driver.find_element(By.TAG_NAME, 'tf-auth').shadow_root.find_element(By.ID, 'loginButton').click()
-	wait.until(expected_conditions.presence_of_element_located((By.ID, 'document')))
+	select(driver, ['tf-navigation', 'shadow_root', '#identity'], ('click',))
+	select(driver, ['tf-navigation', 'shadow_root', '#logout'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#login_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'testuser'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'test_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#error'])
+	select(driver, ['tf-auth', 'shadow_root', '#login_label'], ('click',))
+	select(driver, ['tf-auth', 'shadow_root', '#name'], ('send_keys', 'testuser'))
+	select(driver, ['tf-auth', 'shadow_root', '#password'], ('send_keys', 'new_password'))
+	select(driver, ['tf-auth', 'shadow_root', '#loginButton'], ('click',))
+	select(driver, ['#document', 'frame'])
 
 	success = True
 finally:

tools/buttfeed.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+
+import argparse
+import feedparser
+import json
+import re
+import subprocess
+import time
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--state_file', help = 'Path to a file in which to store state.')
+args = parser.parse_args()
+
+k_feeds = {
+    'tildefriends': 'https://dev.tildefriends.net/cory/tildefriends.rss',
+    'erlbutt': 'https://github.com/cmoid/erlbutt/commits/main.atom',
+    'habitat': 'https://tildegit.org/jeremylist/habitat.rss',
+    'manyverse': 'https://gitlab.com/staltz/manyverse/-/commits/master?format=atom',
+    'ahau': 'https://gitlab.com/ahau/ahau/-/commits/master.atom',
+}
+
+def fix_title(entry):
+    if entry.summary.startswith('<a href=') and '\n' in entry.summary and entry.summary and '\n' in entry.summary:
+        return entry.summary.split('\n')[1]
+    return entry.title.split('\n')[0]
+
+def get_entries():
+    results = []
+    for name, url in k_feeds.items():
+        feed = feedparser.parse(url)
+        for entry in feed.entries:
+            if '/issues/' in entry.link:
+                m = re.match(r'^(\d+)#(.*)#$', entry.description)
+                if m:
+                    results.append((time.mktime(entry.get('updated_parsed')), name, entry.link, f'new issue #{m.group(1)}: {m.group(2)}'))
+                    continue
+            elif '/releases/' in entry.link:
+                m = re.match(r'(.*) released <a href=".*?">(.*?)</a>', entry.title)
+                if m:
+                    results.append((time.mktime(entry.get('updated_parsed')), name, entry.link, f'{m.group(1)} released {m.group(2)}'))
+                    continue
+            if entry.summary.startswith('<a href='):
+                for m in re.findall(r'<a href="(.*?)">.*?</a>$\s*^([^\n]+)$', entry.summary, re.S | re.M):
+                    results.append((time.mktime(entry.get('updated_parsed')), name, m[0], m[1]))
+            else:
+                results.append((time.mktime(entry.get('updated_parsed')), name, entry.link, entry.title.split('\n')[0]))
+    results.sort()
+    results.reverse()
+    return results
+
+state = {}
+if args.state_file:
+    try:
+        with open(args.state_file, 'r') as f:
+            state = json.load(f)
+    except:
+        pass
+cutoff = state.get('last_update') or (time.time() - 2 * 7 * 24 * 60 * 60)
+
+entries = [entry for entry in get_entries() if entry[0] > cutoff]
+if entries:
+    text = '\n'.join([f' * [{entry[1]}] [{entry[3]}]({entry[2]})' for entry in entries])
+    state['last_update'] = entries[0][0]
+    if args.state_file:
+        content = json.dumps({'type': 'post', 'text': text, 'mentions': []})
+        subprocess.check_call(['out/debug/tildefriends', 'publish', '--user', 'cory', '--id', '@DnYDqFfmxdNkYQlpflF9Wkltk2HIhJ5u1MW5njKPLzM=.ed25519', '--content', content])
+        with open(args.state_file, 'w') as f:
+            json.dump(state, f)
+    else:
+        print(text)

tools/emojis.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import re
+import urllib.request
+
+if not os.path.exists('out/emoji-test.txt'):
+    urllib.request.urlretrieve('https://unicode.org/Public/emoji/latest/emoji-test.txt', 'out/emoji-test.txt')
+
+doc = {}
+
+with open('out/emoji-test.txt', 'r') as f:
+    for line in f:
+        line = line.strip()
+        if line.startswith('# group: '):
+            group = line[len('# group: '):]
+        elif line.startswith('# subgroup: '):
+            subgroup = line[len('# subgroup: '):]
+        else:
+            m = re.match(r'((?:\s?[0-9A-F]+)+)\s+; (\S+).*#.*E\d+\.\d+ (.*)', line)
+            if m:
+                emoji = ''.join(chr(int(g, 16)) for g in m.group(1).split(' '))
+                qualified = m.group(2)
+                if qualified == 'fully-qualified':
+                    name = m.group(3)
+                    if not group in doc:
+                        doc[group] = {}
+                    doc[group][name] = emoji
+
+with open('apps/ssb/emojis.json', 'w') as f:
+    json.dump(doc, f, ensure_ascii = False)