keejef
/
session-desktop
mirror of https://github.com/oxen-io/session-desktop
2
0
Fork 0
Code Issues 8 Projects Releases Wiki Activity

feat: performance improvements to quote lookup

Browse Source 
getMessagesByConversation optionally returns quotes from messages in view, quoted messages that are deleted are removed from the lookup map. getMessageBySenderAndSentAt supports an array of messages and renamed to getMessagesBySenderAndSentAt
pull/2757/head
William Grant 2 years ago
parent 1ff2969750
commit 3bc187fa5e
7 changed files with 146 additions and 87 deletions
Show Stats Download Patch File Download Diff File

45
ts/data/data.ts
Unescape Escape View File

@ -18,6 +18,7 @@ import { channels } from './channels';
import * as dataInit from './dataInit'; import * as dataInit from './dataInit';
import { StorageItem } from '../node/storage_item'; import { StorageItem } from '../node/storage_item';
import { fromArrayBufferToBase64, fromBase64ToArrayBuffer } from '../session/utils/String'; import { fromArrayBufferToBase64, fromBase64ToArrayBuffer } from '../session/utils/String';
import { Quote } from '../receiver/types';
const ERASE_SQL_KEY = 'erase-sql-key'; const ERASE_SQL_KEY = 'erase-sql-key';
const ERASE_ATTACHMENTS_KEY = 'erase-attachments'; const ERASE_ATTACHMENTS_KEY = 'erase-attachments';
@ -144,7 +145,7 @@ export const Data = {
getMessageById, getMessageById,
getMessageByServerId, getMessageByServerId,
filterAlreadyFetchedOpengroupMessage, filterAlreadyFetchedOpengroupMessage,
getMessageBySenderAndSentAt, getMessagesBySenderAndSentAt,
getUnreadByConversation, getUnreadByConversation,
getUnreadCountByConversation, getUnreadCountByConversation,
markAllAsReadByConversationNoExpiration, markAllAsReadByConversationNoExpiration,
@ -451,26 +452,22 @@ async function filterAlreadyFetchedOpengroupMessage(
/** /**
* *
* @param source senders id * @param propsList An array of objects containing a source (the sender id) and timestamp of the message - not to be confused with the serverTimestamp. This is equivalent to sent_at
* @param timestamp the timestamp of the message - not to be confused with the serverTimestamp. This is equivalent to sent_at * @returns
*/ */
async function getMessageBySenderAndSentAt({ async function getMessagesBySenderAndSentAt(
source, propsList: Array<{
timestamp, source: string;
}: { timestamp: number;
source: string; }>
timestamp: number; ): Promise<MessageCollection | null> {
}): Promise<MessageModel | null> { const messages = await channels.getMessagesBySenderAndSentAt(propsList);
const messages = await channels.getMessageBySenderAndSentAt({
source,
timestamp,
});
if (!messages || !messages.length) { if (!messages || !messages.length) {
return null; return null;
} }
return new MessageModel(messages[0]); return new MessageCollection(messages);
} }
async function getUnreadByConversation(conversationId: string): Promise<MessageCollection> { async function getUnreadByConversation(conversationId: string): Promise<MessageCollection> {
@ -512,17 +509,27 @@ async function getMessageCountByType(
async function getMessagesByConversation( async function getMessagesByConversation(
conversationId: string, conversationId: string,
{ skipTimerInit = false, messageId = null }: { skipTimerInit?: false; messageId: string | null } {
): Promise<MessageCollection> { skipTimerInit = false,
const messages = await channels.getMessagesByConversation(conversationId, { returnQuotes = false,
messageId = null,
}: { skipTimerInit?: false; returnQuotes?: boolean; messageId: string | null }
): Promise<{ messages: MessageCollection; quotes: Array<Quote> }> {
const { messages, quotes } = await channels.getMessagesByConversation(conversationId, {
messageId, messageId,
returnQuotes,
}); });
if (skipTimerInit) { if (skipTimerInit) {
for (const message of messages) { for (const message of messages) {
message.skipTimerInit = skipTimerInit; message.skipTimerInit = skipTimerInit;
} }
} }
return new MessageCollection(messages);
return {
messages: new MessageCollection(messages),
quotes,
};
} }
/** /**

2
ts/data/dataInit.ts
Unescape Escape View File

@ -46,7 +46,7 @@ const channelsToMake = new Set([
'removeAllMessagesInConversation', 'removeAllMessagesInConversation',
'getMessageCount', 'getMessageCount',
'filterAlreadyFetchedOpengroupMessage', 'filterAlreadyFetchedOpengroupMessage',
'getMessageBySenderAndSentAt', 'getMessagesBySenderAndSentAt',
'getMessageIdsFromServerIds', 'getMessageIdsFromServerIds',
'getMessageById', 'getMessageById',
'getMessagesBySentAt', 'getMessagesBySentAt',

2
ts/models/conversation.ts
Unescape Escape View File

@ -1773,7 +1773,7 @@ export class ConversationModel extends Backbone.Model<ConversationAttributes> {
); );
}).length === 1; }).length === 1;
const isFirstMessageOfConvo = const isFirstMessageOfConvo =
(await Data.getMessagesByConversation(this.id, { messageId: null })).length === 1; (await Data.getMessagesByConversation(this.id, { messageId: null })).messages.length === 1;
if (hadNoRequestsPrior && isFirstMessageOfConvo) { if (hadNoRequestsPrior && isFirstMessageOfConvo) {
friendRequestText = window.i18n('youHaveANewFriendRequest'); friendRequestText = window.i18n('youHaveANewFriendRequest');
} else { } else {

83
ts/node/sql.ts
Unescape Escape View File

@ -17,6 +17,7 @@ import {
isString, isString,
last, last,
map, map,
uniq,
} from 'lodash'; } from 'lodash';
import { redactAll } from '../util/privacy'; // checked - only node import { redactAll } from '../util/privacy'; // checked - only node
import { LocaleMessagesType } from './locale'; // checked - only node import { LocaleMessagesType } from './locale'; // checked - only node
@ -55,6 +56,7 @@ import {
updateSchema, updateSchema,
} from './migration/signalMigrations'; } from './migration/signalMigrations';
import { SettingsKey } from '../data/settings-key'; import { SettingsKey } from '../data/settings-key';
import { Quote } from '../receiver/types';
// tslint:disable: no-console function-name non-literal-fs-path // tslint:disable: no-console function-name non-literal-fs-path
@ -1062,19 +1064,32 @@ function getMessagesCountBySender({ source }: { source: string }) {
return count['count(*)'] || 0; return count['count(*)'] || 0;
} }
function getMessageBySenderAndSentAt({ source, timestamp }: { source: string; timestamp: number }) { function getMessagesBySenderAndSentAt(
const rows = assertGlobalInstance() propsList: Array<{
.prepare( source: string;
`SELECT json FROM ${MESSAGES_TABLE} WHERE timestamp: number;
}>
) {
const db = assertGlobalInstance();
const rows = [];
for (let i = 0; i < propsList.length; i++) {
const { source, timestamp } = propsList[i];
const _rows = db
.prepare(
`SELECT json FROM ${MESSAGES_TABLE} WHERE
source = $source AND source = $source AND
sent_at = $timestamp;` sent_at = $timestamp;`
) )
.all({ .all({
source, source,
timestamp, timestamp,
}); });
rows.push(..._rows);
}
return map(rows, row => jsonToObject(row.json)); return uniq(map(rows, row => jsonToObject(row.json)));
} }
function filterAlreadyFetchedOpengroupMessage( function filterAlreadyFetchedOpengroupMessage(
@ -1203,7 +1218,10 @@ function getMessageCountByType(conversationId: string, type = '%') {
const orderByClause = 'ORDER BY COALESCE(serverTimestamp, sent_at, received_at) DESC'; const orderByClause = 'ORDER BY COALESCE(serverTimestamp, sent_at, received_at) DESC';
const orderByClauseASC = 'ORDER BY COALESCE(serverTimestamp, sent_at, received_at) ASC'; const orderByClauseASC = 'ORDER BY COALESCE(serverTimestamp, sent_at, received_at) ASC';
function getMessagesByConversation(conversationId: string, { messageId = null } = {}) { function getMessagesByConversation(
conversationId: string,
{ messageId = null, returnQuotes = false } = {}
): { messages: Array<Record<string, any>>; quotes: Array<Quote> } {
const absLimit = 30; const absLimit = 30;
// If messageId is given it means we are opening the conversation to that specific messageId, // If messageId is given it means we are opening the conversation to that specific messageId,
// or that we just scrolled to it by a quote click and needs to load around it. // or that we just scrolled to it by a quote click and needs to load around it.
@ -1213,6 +1231,9 @@ function getMessagesByConversation(conversationId: string, { messageId = null }
const numberOfMessagesInConvo = getMessagesCountByConversation(conversationId, globalInstance); const numberOfMessagesInConvo = getMessagesCountByConversation(conversationId, globalInstance);
const floorLoadAllMessagesInConvo = 70; const floorLoadAllMessagesInConvo = 70;
let messages: Array<Record<string, any>> = [];
let quotes = [];
if (messageId || firstUnread) { if (messageId || firstUnread) {
const messageFound = getMessageById(messageId || firstUnread); const messageFound = getMessageById(messageId || firstUnread);
@ -1244,33 +1265,39 @@ function getMessagesByConversation(conversationId: string, { messageId = null }
: absLimit, : absLimit,
}); });
return map(rows, row => jsonToObject(row.json)); messages = map(rows, row => jsonToObject(row.json));
} }
console.info( console.info(
`getMessagesByConversation: Could not find messageId ${messageId} in db with conversationId: ${conversationId}. Just fetching the convo as usual.` `getMessagesByConversation: Could not find messageId ${messageId} in db with conversationId: ${conversationId}. Just fetching the convo as usual.`
); );
} } else {
const limit =
numberOfMessagesInConvo < floorLoadAllMessagesInConvo
? floorLoadAllMessagesInConvo
: absLimit * 2;
const limit = const rows = assertGlobalInstance()
numberOfMessagesInConvo < floorLoadAllMessagesInConvo .prepare(
? floorLoadAllMessagesInConvo `
: absLimit * 2;
const rows = assertGlobalInstance()
.prepare(
`
SELECT json FROM ${MESSAGES_TABLE} WHERE SELECT json FROM ${MESSAGES_TABLE} WHERE
conversationId = $conversationId conversationId = $conversationId
${orderByClause} ${orderByClause}
LIMIT $limit; LIMIT $limit;
` `
) )
.all({ .all({
conversationId, conversationId,
limit, limit,
}); });
return map(rows, row => jsonToObject(row.json)); messages = map(rows, row => jsonToObject(row.json));
}
if (returnQuotes) {
quotes = messages.filter(message => message.quote).map(message => message.quote);
}
return { messages, quotes };
} }
function getLastMessagesByConversation(conversationId: string, limit: number) { function getLastMessagesByConversation(conversationId: string, limit: number) {
@ -2454,7 +2481,7 @@ export const sqlNode = {
getMessageCountByType, getMessageCountByType,
filterAlreadyFetchedOpengroupMessage, filterAlreadyFetchedOpengroupMessage,
getMessageBySenderAndSentAt, getMessagesBySenderAndSentAt,
getMessageIdsFromServerIds, getMessageIdsFromServerIds,
getMessageById, getMessageById,
getMessagesBySentAt, getMessagesBySentAt,

14
ts/receiver/contentMessage.ts
Unescape Escape View File

@ -565,10 +565,14 @@ async function handleUnsendMessage(envelope: EnvelopePlus, unsendMessage: Signal
return; return;
} }
const messageToDelete = await Data.getMessageBySenderAndSentAt({ const messageToDelete = (
source: messageAuthor, await Data.getMessagesBySenderAndSentAt([
timestamp: toNumber(timestamp), {
}); source: messageAuthor,
timestamp: toNumber(timestamp),
},
])
)?.models?.[0];
const messageHash = messageToDelete?.get('messageHash'); const messageHash = messageToDelete?.get('messageHash');
//#endregion //#endregion
@ -665,7 +669,7 @@ async function handleMessageRequestResponse(
) )
); );
const allMessageModels = flatten(allMessagesCollections.map(m => m.models)); const allMessageModels = flatten(allMessagesCollections.map(m => m.messages.models));
allMessageModels.forEach(messageModel => { allMessageModels.forEach(messageModel => {
messageModel.set({ conversationId: unblindedConvoId }); messageModel.set({ conversationId: unblindedConvoId });

12
ts/receiver/dataMessage.ts
Unescape Escape View File

@ -258,10 +258,14 @@ export async function isSwarmMessageDuplicate({
sentAt: number; sentAt: number;
}) { }) {
try { try {
const result = await Data.getMessageBySenderAndSentAt({ const result = (
source, await Data.getMessagesBySenderAndSentAt([
timestamp: sentAt, {
}); source,
timestamp: sentAt,
},
])
)?.models?.length;
return Boolean(result); return Boolean(result);
} catch (error) { } catch (error) {

75
ts/state/ducks/conversations.ts
Unescape Escape View File

@ -289,6 +289,7 @@ export type ConversationLookupType = {
}; };
export type QuoteLookupType = { export type QuoteLookupType = {
// key is message [timestamp]-[author-pubkey]
[key: string]: MessageModelPropsWithoutConvoProps; [key: string]: MessageModelPropsWithoutConvoProps;
}; };
@ -298,7 +299,6 @@ export type ConversationsStateType = {
// NOTE the messages that are in view // NOTE the messages that are in view
messages: Array<MessageModelPropsWithoutConvoProps>; messages: Array<MessageModelPropsWithoutConvoProps>;
// NOTE the quotes that are in view // NOTE the quotes that are in view
// key is message [timestamp]-[author-pubkey]
quotes: QuoteLookupType; quotes: QuoteLookupType;
firstUnreadMessageId: string | undefined; firstUnreadMessageId: string | undefined;
messageDetailProps?: MessagePropsDetails; messageDetailProps?: MessagePropsDetails;
@ -364,48 +364,45 @@ async function getMessages({
return { messagesProps: [], quotesProps: {} }; return { messagesProps: [], quotesProps: {} };
} }
const messageSet = await Data.getMessagesByConversation(conversationKey, { const {
messages: messagesCollection,
quotes: quotesCollection,
} = await Data.getMessagesByConversation(conversationKey, {
messageId, messageId,
returnQuotes: true,
}); });
const messagesProps: Array<MessageModelPropsWithoutConvoProps> = messageSet.models.map(m => const messagesProps: Array<MessageModelPropsWithoutConvoProps> = messagesCollection.models.map(
m.getMessageModelProps() m => m.getMessageModelProps()
); );
const time = Date.now() - beforeTimestamp; const time = Date.now() - beforeTimestamp;
window?.log?.info(`Loading ${messagesProps.length} messages took ${time}ms to load.`); window?.log?.info(`Loading ${messagesProps.length} messages took ${time}ms to load.`);
const quotesProps: QuoteLookupType = {}; const quotesProps: QuoteLookupType = {};
const quotes = messagesProps.filter(
message => message.propsForMessage?.quote?.messageId && message.propsForMessage.quote?.sender
);
for (let i = 0; i < quotes.length; i++) { if (quotesCollection?.length) {
const id = quotes[i].propsForMessage?.quote?.messageId; const quotePropsList = quotesCollection.map(quote => ({
const sender = quotes[i].propsForMessage.quote?.sender; timestamp: Number(quote?.id),
source: String(quote?.author),
if (id && sender) { }));
const timestamp = Number(id);
// See if a quoted message is already in memory if not lookup in db const quotedMessagesCollection = await Data.getMessagesBySenderAndSentAt(quotePropsList);
let results = messagesProps.filter(
message => if (quotedMessagesCollection?.length) {
message.propsForMessage.timestamp === timestamp && for (let i = 0; i < quotedMessagesCollection.length; i++) {
message.propsForMessage.sender === sender const quotedMessage = quotedMessagesCollection.models.at(i)?.getMessageModelProps();
); if (quotedMessage) {
const timestamp = Number(quotedMessage.propsForMessage.timestamp);
if (!results.length) { const sender = quotedMessage.propsForMessage.sender;
const dbResult = ( if (timestamp && sender) {
await Data.getMessageBySenderAndSentAt({ source: sender, timestamp }) quotesProps[`${timestamp}-${sender}`] = quotedMessage;
)?.getMessageModelProps(); }
if (dbResult) {
results = [dbResult];
} }
} }
quotesProps[`${timestamp}-${sender}`] = results[0];
} }
} }
// window.log.debug(`WIP: duck quoteProps`, quotesProps); // window.log.debug(`WIP: duck quotesProps`, quotesProps);
return { messagesProps, quotesProps }; return { messagesProps, quotesProps };
} }
@ -563,7 +560,26 @@ function handleMessageExpiredOrDeleted(
// search if we find this message id. // search if we find this message id.
// we might have not loaded yet, so this case might not happen // we might have not loaded yet, so this case might not happen
const messageInStoreIndex = state?.messages.findIndex(m => m.propsForMessage.id === messageId); const messageInStoreIndex = state?.messages.findIndex(m => m.propsForMessage.id === messageId);
const editedQuotes = { ...state.quotes };
if (messageInStoreIndex >= 0) { if (messageInStoreIndex >= 0) {
// Check if the message is quoted somewhere, and if so, remove it from the quotes
const msgProps = state.messages[messageInStoreIndex].propsForMessage;
// TODO check if message is a group or public group because we will need to use the server timestamp
const { timestamp, sender } = msgProps;
if (timestamp && sender) {
const message2Delete = editedQuotes[`${timestamp}-${sender}`];
window.log.debug(
`WIP: deleting quote {${timestamp}-${sender}} ${JSON.stringify(message2Delete)}`
);
window.log.debug(
`WIP: editedQuotes count before delete ${Object.keys(editedQuotes).length}`
);
delete editedQuotes[`${timestamp}-${sender}`];
window.log.debug(
`WIP: editedQuotes count after delete ${Object.keys(editedQuotes).length}`
);
}
// we cannot edit the array directly, so slice the first part, and slice the second part, // we cannot edit the array directly, so slice the first part, and slice the second part,
// keeping the index removed out // keeping the index removed out
const editedMessages = [ const editedMessages = [
@ -578,6 +594,7 @@ function handleMessageExpiredOrDeleted(
return { return {
...state, ...state,
messages: editedMessages, messages: editedMessages,
quotes: editedQuotes,
firstUnreadMessageId: firstUnreadMessageId:
state.firstUnreadMessageId === messageId ? undefined : state.firstUnreadMessageId, state.firstUnreadMessageId === messageId ? undefined : state.firstUnreadMessageId,
}; };

Write Preview
Loading…
Cancel
Save