ssb: Switch to a more brute force about-gathering approach. I think if I start with this and avoid querying all known accounts up-front, we will be plenty fast.
Some checks failed
Build Tilde Friends / Build-All (push) Failing after 5m12s
Some checks failed
Build Tilde Friends / Build-All (push) Failing after 5m12s
This commit is contained in:
parent
c93b8fc045
commit
d367d47c4d
@ -1,5 +1,5 @@
|
||||
{
|
||||
"type": "tildefriends-app",
|
||||
"emoji": "🦀",
|
||||
"previous": "&gO8ganUDJtyGtfKUPRTbK2d34GJXhHKvHIZheF/lXI4=.sha256"
|
||||
"previous": "&2TUIbhXFJWMc6NTZt2Q0RB004ufTD+6CuaC5i2DdpsU=.sha256"
|
||||
}
|
||||
|
@ -152,137 +152,36 @@ class TfElement extends LitElement {
|
||||
|
||||
async fetch_about(following, users) {
|
||||
let ids = Object.keys(following).sort();
|
||||
const k_cache_version = 1;
|
||||
let cache = await tfrpc.rpc.databaseGet('about');
|
||||
let original_cache = cache;
|
||||
cache = cache ? JSON.parse(cache) : {};
|
||||
if (cache.version !== k_cache_version) {
|
||||
cache = {
|
||||
version: k_cache_version,
|
||||
about: {},
|
||||
last_row_id: 0,
|
||||
};
|
||||
}
|
||||
let max_row_id = (
|
||||
await tfrpc.rpc.query(
|
||||
`
|
||||
SELECT MAX(rowid) AS max_row_id FROM messages
|
||||
`,
|
||||
[]
|
||||
)
|
||||
)[0].max_row_id;
|
||||
for (let id of Object.keys(cache.about)) {
|
||||
if (ids.indexOf(id) == -1) {
|
||||
delete cache.about[id];
|
||||
}
|
||||
}
|
||||
|
||||
const k_account_chunk_size = 4096;
|
||||
const k_chunk_size = 1024;
|
||||
let min_row_id = 0;
|
||||
console.log(
|
||||
'loading about for',
|
||||
ids.length,
|
||||
'accounts',
|
||||
cache.last_row_id,
|
||||
'=>',
|
||||
max_row_id
|
||||
);
|
||||
console.log('loading about for', ids.length, 'accounts');
|
||||
try {
|
||||
while (true) {
|
||||
let max_seen;
|
||||
for (
|
||||
let account_chunk = 0;
|
||||
account_chunk < ids.length;
|
||||
account_chunk += k_account_chunk_size
|
||||
) {
|
||||
let ids_chunk = ids.slice(
|
||||
account_chunk,
|
||||
account_chunk + k_account_chunk_size
|
||||
);
|
||||
let abouts = await tfrpc.rpc.query(
|
||||
`
|
||||
WITH
|
||||
past AS (
|
||||
SELECT
|
||||
messages.rowid AS rowid, messages.author, json(messages.content) AS content, messages.sequence
|
||||
FROM
|
||||
messages
|
||||
WHERE
|
||||
messages.rowid > ?3 AND
|
||||
messages.rowid <= ?4 AND
|
||||
messages.content ->> 'type' = 'about'
|
||||
),
|
||||
current AS (
|
||||
SELECT
|
||||
messages.rowid AS rowid, messages.author, json(messages.content) AS content, messages.sequence
|
||||
FROM
|
||||
messages
|
||||
WHERE
|
||||
messages.rowid > ?6 AND
|
||||
messages.rowid <= ?4 AND
|
||||
messages.content ->> 'type' = 'about'
|
||||
)
|
||||
SELECT * FROM past
|
||||
JOIN json_each(?1) AS following
|
||||
ON past.author = following.value
|
||||
UNION SELECT * FROM current
|
||||
JOIN json_each(?2) AS following
|
||||
ON current.author = following.value
|
||||
ORDER BY rowid LIMIT ?5
|
||||
`,
|
||||
[
|
||||
JSON.stringify(ids_chunk.filter((id) => cache.about[id])),
|
||||
JSON.stringify(ids_chunk.filter((id) => !cache.about[id])),
|
||||
cache.last_row_id,
|
||||
max_row_id,
|
||||
k_chunk_size,
|
||||
min_row_id,
|
||||
]
|
||||
);
|
||||
for (let about of abouts) {
|
||||
let content = JSON.parse(about.content);
|
||||
if (content.about === about.author) {
|
||||
delete content.type;
|
||||
delete content.about;
|
||||
cache.about[about.author] = Object.assign(
|
||||
cache.about[about.author] || {},
|
||||
content
|
||||
);
|
||||
}
|
||||
max_seen = about.rowid;
|
||||
}
|
||||
console.log(account_chunk, '/', ids.length, 'accounts');
|
||||
}
|
||||
console.log(
|
||||
'cache =',
|
||||
cache.last_row_id,
|
||||
'seen =',
|
||||
max_seen,
|
||||
'max =',
|
||||
max_row_id
|
||||
let rows = await tfrpc.rpc.query(
|
||||
`
|
||||
SELECT all_abouts.author, json(json_group_object(all_abouts.key, all_abouts.value)) AS about
|
||||
FROM (
|
||||
SELECT
|
||||
messages.author,
|
||||
fields.key,
|
||||
RANK() OVER (PARTITION BY messages.author, fields.key ORDER BY messages.sequence DESC) AS rank,
|
||||
fields.value
|
||||
FROM messages JOIN json_each(messages.content) AS fields
|
||||
WHERE
|
||||
messages.content ->> '$.type' = 'about' AND
|
||||
messages.content ->> '$.about' = messages.author AND
|
||||
NOT fields.key IN ('about', 'type')) all_abouts
|
||||
WHERE rank = 1
|
||||
GROUP BY all_abouts.author
|
||||
`,
|
||||
[JSON.stringify(ids)]
|
||||
);
|
||||
users = users || {};
|
||||
for (let row of rows) {
|
||||
users[row.author] = Object.assign(
|
||||
{follow_depth: following[row.author]?.d},
|
||||
users[row.author] || {},
|
||||
JSON.parse(row.about)
|
||||
);
|
||||
cache.last_row_id = Math.max(cache.last_row_id, max_seen ?? max_row_id);
|
||||
min_row_id = Math.max(min_row_id, max_seen ?? max_row_id);
|
||||
let new_cache = JSON.stringify(cache);
|
||||
if (new_cache !== original_cache) {
|
||||
let start_time = new Date();
|
||||
tfrpc.rpc.databaseSet('about', new_cache).then(function () {
|
||||
console.log('saving about took', (new Date() - start_time) / 1000);
|
||||
});
|
||||
}
|
||||
users = users || {};
|
||||
for (let id of Object.keys(cache.about)) {
|
||||
users[id] = Object.assign(
|
||||
{follow_depth: following[id]?.d},
|
||||
users[id] || {},
|
||||
cache.about[id]
|
||||
);
|
||||
}
|
||||
if (cache.last_row_id >= max_row_id) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
console.log('updated users');
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
}
|
||||
|
@ -98,7 +98,9 @@ class TfTabNewsElement extends LitElement {
|
||||
|
||||
unread_status(channel) {
|
||||
if (channel === undefined) {
|
||||
if (Object.keys(this.channels_unread).some(x => this.unread_status(x))) {
|
||||
if (
|
||||
Object.keys(this.channels_unread).some((x) => this.unread_status(x))
|
||||
) {
|
||||
return '✉️ ';
|
||||
}
|
||||
} else if (
|
||||
|
Loading…
x
Reference in New Issue
Block a user