Compare commits

...

5 Commits

Author SHA1 Message Date
67d458bd38 ssb: prettier.
Some checks failed
Build Tilde Friends / Build-All (push) Has been cancelled
2025-04-09 22:15:51 -04:00
d9684c7d62 ssb: Fix culling feeds again. 2025-04-09 22:15:01 -04:00
5a818d2119 ssb: More incremental profile loading? 2025-04-09 22:08:33 -04:00
6f96d4ce65 ssb: More different feed culling logic. 2025-04-09 20:45:23 -04:00
f72395756a ssb: If things time out because we're following a million accounts...recover ungracefully. 2025-04-09 20:02:16 -04:00
5 changed files with 125 additions and 83 deletions

View File

@ -1,5 +1,5 @@
{ {
"type": "tildefriends-app", "type": "tildefriends-app",
"emoji": "🦀", "emoji": "🦀",
"previous": "&YZCzXrfB6j+y0sXF4KspAibwjLsSCaMoB5rdO3mQl+Q=.sha256" "previous": "&h9uzh3gVWy1KWIu6wFYCrBZkVmnQAi7QLAj5iIXffFM=.sha256"
} }

View File

@ -175,10 +175,23 @@ class TfElement extends LitElement {
} }
} }
const k_chunk_size = 1024;
let min_row_id = 0;
console.log(
'loading about for',
ids.length,
'accounts',
cache.last_row_id,
'=>',
max_row_id
);
try {
while (true) {
let abouts = await tfrpc.rpc.query( let abouts = await tfrpc.rpc.query(
` `
SELECT * FROM (
SELECT SELECT
messages.author, json(messages.content) AS content, messages.sequence messages.rowid AS rowid, messages.author, json(messages.content) AS content, messages.sequence
FROM FROM
messages, messages,
json_each(?1) AS following json_each(?1) AS following
@ -189,23 +202,28 @@ class TfElement extends LitElement {
messages.rowid <= ?4 messages.rowid <= ?4
UNION UNION
SELECT SELECT
messages.author, json(messages.content) AS content, messages.sequence messages.rowid AS rowid, messages.author, json(messages.content) AS content, messages.sequence
FROM FROM
messages, messages,
json_each(?2) AS following json_each(?2) AS following
WHERE WHERE
messages.author = following.value AND messages.author = following.value AND
messages.content ->> 'type' = 'about' AND messages.content ->> 'type' = 'about' AND
messages.rowid > ?6 AND
messages.rowid <= ?4 messages.rowid <= ?4
ORDER BY messages.author, messages.sequence )
ORDER BY rowid LIMIT ?5
`, `,
[ [
JSON.stringify(ids.filter((id) => cache.about[id])), JSON.stringify(ids.filter((id) => cache.about[id])),
JSON.stringify(ids.filter((id) => !cache.about[id])), JSON.stringify(ids.filter((id) => !cache.about[id])),
cache.last_row_id, cache.last_row_id,
max_row_id, max_row_id,
k_chunk_size,
min_row_id,
] ]
); );
let max_seen;
for (let about of abouts) { for (let about of abouts) {
let content = JSON.parse(about.content); let content = JSON.parse(about.content);
if (content.about === about.author) { if (content.about === about.author) {
@ -216,8 +234,18 @@ class TfElement extends LitElement {
content content
); );
} }
max_seen = about.rowid;
} }
cache.last_row_id = max_row_id; console.log(
'cache =',
cache.last_row_id,
'seen =',
max_seen,
'max =',
max_row_id
);
cache.last_row_id = Math.max(cache.last_row_id, max_seen ?? max_row_id);
min_row_id = Math.max(min_row_id, max_seen ?? max_row_id);
let new_cache = JSON.stringify(cache); let new_cache = JSON.stringify(cache);
if (new_cache !== original_cache) { if (new_cache !== original_cache) {
let start_time = new Date(); let start_time = new Date();
@ -233,6 +261,13 @@ class TfElement extends LitElement {
cache.about[id] cache.about[id]
); );
} }
if (cache.last_row_id >= max_row_id) {
break;
}
}
} catch (e) {
console.log(e);
}
return Object.assign({}, users); return Object.assign({}, users);
} }

View File

@ -431,12 +431,17 @@ class TfMessageElement extends LitElement {
> >
Copy ID Copy ID
</button> </button>
${this.drafts[this.message?.id] === undefined ? html` ${this.drafts[this.message?.id] === undefined
? html`
<button class="w3-button w3-bar-item" @click=${this.show_reply}> <button class="w3-button w3-bar-item" @click=${this.show_reply}>
Reply Reply
</button> </button>
` : undefined} `
<button class="w3-button w3-bar-item w3-border-bottom" @click=${this.react}> : undefined}
<button
class="w3-button w3-bar-item w3-border-bottom"
@click=${this.react}
>
👍 React 👍 React
</button> </button>
${formats.map( ${formats.map(
@ -541,8 +546,7 @@ class TfMessageElement extends LitElement {
: undefined; : undefined;
return html` return html`
<div class="w3-section w3-container"> <div class="w3-section w3-container">
${reply} ${reply} ${this.render_children()}
${this.render_children()}
</div> </div>
`; `;
} }

View File

@ -81,7 +81,9 @@ class TfTabNewsFeedElement extends LitElement {
`, `,
[JSON.stringify(combined.map((x) => x.id))] [JSON.stringify(combined.map((x) => x.id))]
); );
return [].concat(combined, await tfrpc.rpc.query( return [].concat(
combined,
await tfrpc.rpc.query(
` `
SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature SELECT FALSE AS is_primary, messages.rowid, messages.id, messages.previous, messages.author, messages.sequence, messages.timestamp, messages.hash, json(messages.content) AS content, messages.signature
FROM messages FROM messages
@ -89,8 +91,12 @@ class TfTabNewsFeedElement extends LitElement {
JOIN json_each(?1) AS following ON messages.author = following.value JOIN json_each(?1) AS following ON messages.author = following.value
WHERE messages.content ->> 'type' = 'vote' WHERE messages.content ->> 'type' = 'vote'
`, `,
[JSON.stringify(this.following), JSON.stringify(refs2.map((x) => x.ref))] [
)); JSON.stringify(this.following),
JSON.stringify(refs2.map((x) => x.ref)),
]
)
);
} }
async fetch_messages(start_time, end_time) { async fetch_messages(start_time, end_time) {
@ -253,7 +259,11 @@ class TfTabNewsFeedElement extends LitElement {
try { try {
let more = []; let more = [];
let last_start_time = this.time_range[0]; let last_start_time = this.time_range[0];
try {
more = await this.fetch_messages(null, last_start_time); more = await this.fetch_messages(null, last_start_time);
} catch (e) {
console.log(e);
}
this.update_time_range_from_messages( this.update_time_range_from_messages(
more.filter((x) => x.timestamp < last_start_time) more.filter((x) => x.timestamp < last_start_time)
); );

View File

@ -1594,27 +1594,20 @@ static void _tf_ssb_rpc_delete_feeds_work(tf_ssb_t* ssb, void* user_data)
db = tf_ssb_acquire_db_writer(ssb); db = tf_ssb_acquire_db_writer(ssb);
sqlite3_stmt* statement; sqlite3_stmt* statement;
if (sqlite3_prepare(db, if (sqlite3_prepare(db,
"DELETE FROM messages WHERE author IN (" "DELETE FROM messages WHERE id IN ("
" SELECT author FROM messages WHERE author NOT IN (SELECT value FROM json_each(?)) GROUP BY author LIMIT 1" " SELECT id FROM messages WHERE author NOT IN (SELECT value FROM json_each(?)) ORDER BY rowid DESC LIMIT 1024"
") RETURNING author", ")",
-1, &statement, NULL) == SQLITE_OK) -1, &statement, NULL) == SQLITE_OK)
{ {
int status = SQLITE_OK;
bool printed = false;
if (sqlite3_bind_text(statement, 1, arg, -1, NULL) == SQLITE_OK) if (sqlite3_bind_text(statement, 1, arg, -1, NULL) == SQLITE_OK)
{ {
while ((status = sqlite3_step(statement)) == SQLITE_ROW) if (sqlite3_step(statement) != SQLITE_DONE)
{ {
if (!printed) tf_printf("deleting messages: %s\n", sqlite3_errmsg(db));
{
tf_printf("deleting %s\n", sqlite3_column_text(statement, 0));
printed = true;
delete->deleted++;
} }
} else
if (status != SQLITE_DONE)
{ {
tf_printf("deleting feeds: %s\n", sqlite3_errmsg(db)); delete->deleted += sqlite3_changes(db);
} }
} }
sqlite3_finalize(statement); sqlite3_finalize(statement);
@ -1627,7 +1620,7 @@ static void _tf_ssb_rpc_delete_feeds_work(tf_ssb_t* ssb, void* user_data)
JS_FreeRuntime(runtime); JS_FreeRuntime(runtime);
delete->duration_ms = (uv_hrtime() - start_ns) / 1000000LL; delete->duration_ms = (uv_hrtime() - start_ns) / 1000000LL;
tf_printf("Deleted %d feeds in %d ms.\n", delete->deleted, (int)delete->duration_ms); tf_printf("Deleted %d message in %d ms.\n", delete->deleted, (int)delete->duration_ms);
_tf_ssb_rpc_checkpoint(ssb); _tf_ssb_rpc_checkpoint(ssb);
} }