From a98fe05e089c5a47e2c87cf397dbe2f5baadd218 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Wed, 29 Oct 2025 21:50:58 +0100 Subject: [PATCH 01/11] feat: message previews - Remove partial downloads (remove creation of the stub messages) (#7373) - Remove "Download maximum available until" and remove stock string `DC_STR_DOWNLOAD_AVAILABILITY` (#7369) - Send pre-message on messages with large attachments (#7410) - Pre messages can now get read receipts (#7433) Co-authored-by: Hocuri --- deltachat-ffi/deltachat.h | 31 +- deltachat-jsonrpc/src/api/types/message.rs | 3 + .../tests/test_chatlist_events.py | 4 +- deltachat-rpc-client/tests/test_something.py | 100 +--- python/tests/test_1_online.py | 33 -- src/calls/calls_tests.rs | 64 +-- src/chat.rs | 131 ++++- src/chat/chat_tests.rs | 4 +- src/config.rs | 6 +- src/context.rs | 7 - src/context/context_tests.rs | 1 + src/download.rs | 487 ++++++---------- src/download/pre_msg_metadata.rs | 248 +++++++++ src/headerdef.rs | 18 + src/imap.rs | 127 +++-- src/imap/session.rs | 2 + src/internals_for_benches.rs | 2 +- src/message.rs | 99 +++- src/message/message_tests.rs | 106 ---- src/mimefactory.rs | 103 +++- src/mimefactory/mimefactory_tests.rs | 8 +- src/mimeparser.rs | 111 ++-- src/mimeparser/mimeparser_tests.rs | 129 ++--- src/param.rs | 30 + src/reaction.rs | 70 +-- src/receive_imf.rs | 274 +++++---- src/receive_imf/receive_imf_tests.rs | 273 +-------- src/scheduler.rs | 41 +- src/sql/migrations.rs | 24 + src/stock_str.rs | 49 +- src/stock_str/stock_str_tests.rs | 8 - src/test_utils.rs | 44 +- src/tests.rs | 1 + src/tests/pre_messages.rs | 6 + src/tests/pre_messages/additional_text.rs | 40 ++ src/tests/pre_messages/forward_and_save.rs | 122 ++++ src/tests/pre_messages/legacy.rs | 61 ++ src/tests/pre_messages/receiving.rs | 522 ++++++++++++++++++ src/tests/pre_messages/sending.rs | 337 +++++++++++ src/tests/pre_messages/util.rs | 65 +++ src/webxdc/webxdc_tests.rs | 66 +-- 41 files changed, 2411 insertions(+), 1446 deletions(-) create mode 100644 src/download/pre_msg_metadata.rs create mode 100644 src/tests/pre_messages.rs create mode 100644 src/tests/pre_messages/additional_text.rs create mode 100644 src/tests/pre_messages/forward_and_save.rs create mode 100644 src/tests/pre_messages/legacy.rs create mode 100644 src/tests/pre_messages/receiving.rs create mode 100644 src/tests/pre_messages/sending.rs create mode 100644 src/tests/pre_messages/util.rs diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index b540cc5339..66399f2abd 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -488,11 +488,11 @@ char* dc_get_blobdir (const dc_context_t* context); * 0=use IMAP IDLE if the server supports it. * This is a developer option used for testing polling used as an IDLE fallback. * - `download_limit` = Messages up to this number of bytes are downloaded automatically. - * For larger messages, only the header is downloaded and a placeholder is shown. + * For messages with large attachments, two messages are sent: + * a Pre-Message containing metadata and a Post-Message containing the attachment. + * Pre-Messages are always downloaded and show a placeholder message. * These messages can be downloaded fully using dc_download_full_msg() later. - * The limit is compared against raw message sizes, including headers. - * The actually used limit may be corrected - * to not mess up with non-delivery-reports or read-receipts. + * Post-Messages are automatically downloaded if they are smaller than the download_limit. * 0=no limit (default). * Changes affect future messages only. * - `protect_autocrypt` = Enable Header Protection for Autocrypt header. @@ -4310,7 +4310,8 @@ char* dc_msg_get_webxdc_info (const dc_msg_t* msg); /** * Get the size of the file. Returns the size of the file associated with a - * message, if applicable. + * message, if applicable. + * If message is a pre-message, then this returns size of the to be downloaded file. * * Typically, this is used to show the size of document files, e.g. a PDF. * @@ -7263,22 +7264,9 @@ void dc_event_unref(dc_event_t* event); /// `%1$s` will be replaced by the percentage used #define DC_STR_QUOTA_EXCEEDING_MSG_BODY 98 -/// "%1$s message" -/// -/// Used as the message body when a message -/// was not yet downloaded completely -/// (dc_msg_get_download_state() is e.g. @ref DC_DOWNLOAD_AVAILABLE). -/// -/// `%1$s` will be replaced by human-readable size (e.g. "1.2 MiB"). +/// @deprecated Deprecated 2025-11-12, this string is no longer needed. #define DC_STR_PARTIAL_DOWNLOAD_MSG_BODY 99 -/// "Download maximum available until %1$s" -/// -/// Appended after some separator to @ref DC_STR_PARTIAL_DOWNLOAD_MSG_BODY. -/// -/// `%1$s` will be replaced by human-readable date and time. -#define DC_STR_DOWNLOAD_AVAILABILITY 100 - /// "Multi Device Synchronization" /// /// Used in subjects of outgoing sync messages. @@ -7776,6 +7764,11 @@ void dc_event_unref(dc_event_t* event); /// Used as the first info messages in newly created classic email threads. #define DC_STR_CHAT_UNENCRYPTED_EXPLANATON 230 +/// "Contact" +/// +/// Used in summaries. +#define DC_STR_CONTACT 231 + /** * @} */ diff --git a/deltachat-jsonrpc/src/api/types/message.rs b/deltachat-jsonrpc/src/api/types/message.rs index 567e3ffdc9..27ba8dca0e 100644 --- a/deltachat-jsonrpc/src/api/types/message.rs +++ b/deltachat-jsonrpc/src/api/types/message.rs @@ -92,6 +92,9 @@ pub struct MessageObject { file: Option, file_mime: Option, + + /// The size of the file in bytes, if applicable. + /// If message is a pre-message, then this is the size of the to be downloaded file. file_bytes: u64, file_name: Option, diff --git a/deltachat-rpc-client/tests/test_chatlist_events.py b/deltachat-rpc-client/tests/test_chatlist_events.py index 05bec0795f..f13769e071 100644 --- a/deltachat-rpc-client/tests/test_chatlist_events.py +++ b/deltachat-rpc-client/tests/test_chatlist_events.py @@ -1,7 +1,5 @@ from __future__ import annotations -import base64 -import os from typing import TYPE_CHECKING from deltachat_rpc_client import Account, EventType, const @@ -129,7 +127,7 @@ def test_download_on_demand(acfactory: ACFactory) -> None: msg.get_snapshot().chat.accept() bob.get_chat_by_id(chat_id).send_message( "Hello World, this message is bigger than 5 bytes", - html=base64.b64encode(os.urandom(300000)).decode("utf-8"), + file="../test-data/image/screenshot.jpg", ) message = alice.wait_for_incoming_msg() diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index 44cddb3070..415c135d22 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -5,13 +5,12 @@ import os import socket import subprocess -import time from unittest.mock import MagicMock import pytest -from deltachat_rpc_client import Contact, EventType, Message, events -from deltachat_rpc_client.const import DownloadState, MessageState +from deltachat_rpc_client import EventType, events +from deltachat_rpc_client.const import MessageState from deltachat_rpc_client.pytestplugin import E2EE_INFO_MSGS from deltachat_rpc_client.rpc import JsonRpcError @@ -333,7 +332,7 @@ def test_receive_imf_failure(acfactory) -> None: alice_contact_bob = alice.create_contact(bob, "Bob") alice_chat_bob = alice_contact_bob.create_chat() - bob.set_config("fail_on_receiving_full_msg", "1") + bob.set_config("simulate_receive_imf_error", "1") alice_chat_bob.send_text("Hello!") event = bob.wait_for_event(EventType.MSGS_CHANGED) assert event.chat_id == bob.get_device_chat().id @@ -342,17 +341,16 @@ def test_receive_imf_failure(acfactory) -> None: snapshot = message.get_snapshot() assert ( snapshot.text == "❌ Failed to receive a message:" - " Condition failed: `!context.get_config_bool(Config::FailOnReceivingFullMsg).await?`." + " Condition failed: `!context.get_config_bool(Config::SimulateReceiveImfError).await?`." " Please report this bug to delta@merlinux.eu or https://support.delta.chat/." ) # The failed message doesn't break the IMAP loop. - bob.set_config("fail_on_receiving_full_msg", "0") + bob.set_config("simulate_receive_imf_error", "0") alice_chat_bob.send_text("Hello again!") message = bob.wait_for_incoming_msg() snapshot = message.get_snapshot() assert snapshot.text == "Hello again!" - assert snapshot.download_state == DownloadState.DONE assert snapshot.error is None @@ -588,94 +586,6 @@ def test_mdn_doesnt_break_autocrypt(acfactory) -> None: assert snapshot.show_padlock -def test_reaction_to_partially_fetched_msg(acfactory, tmp_path): - """See https://github.com/deltachat/deltachat-core-rust/issues/3688 "Partially downloaded - messages are received out of order". - - If the Inbox contains X small messages followed by Y large messages followed by Z small - messages, Delta Chat first downloaded a batch of X+Z messages, and then a batch of Y messages. - - This bug was discovered by @Simon-Laux while testing reactions PR #3644 and can be reproduced - with online test as follows: - - Bob enables download limit and goes offline. - - Alice sends a large message to Bob and reacts to this message with a thumbs-up. - - Bob goes online - - Bob first processes a reaction message and throws it away because there is no corresponding - message, then processes a partially downloaded message. - - As a result, Bob does not see a reaction - """ - download_limit = 300000 - ac1, ac2 = acfactory.get_online_accounts(2) - ac1_addr = ac1.get_config("addr") - chat = ac1.create_chat(ac2) - ac2.set_config("download_limit", str(download_limit)) - ac2.stop_io() - - logging.info("sending small+large messages from ac1 to ac2") - msgs = [] - msgs.append(chat.send_text("hi")) - path = tmp_path / "large" - path.write_bytes(os.urandom(download_limit + 1)) - msgs.append(chat.send_file(str(path))) - for m in msgs: - m.wait_until_delivered() - - logging.info("sending a reaction to the large message from ac1 to ac2") - # TODO: Find the reason of an occasional message reordering on the server (so that the reaction - # has a lower UID than the previous message). W/a is to sleep for some time to let the reaction - # have a later INTERNALDATE. - time.sleep(1.1) - react_str = "\N{THUMBS UP SIGN}" - msgs.append(msgs[-1].send_reaction(react_str)) - msgs[-1].wait_until_delivered() - - ac2.start_io() - - logging.info("wait for ac2 to receive a reaction") - msg2 = Message(ac2, ac2.wait_for_reactions_changed().msg_id) - assert msg2.get_sender_contact().get_snapshot().address == ac1_addr - assert msg2.get_snapshot().download_state == DownloadState.AVAILABLE - reactions = msg2.get_reactions() - contacts = [Contact(ac2, int(i)) for i in reactions.reactions_by_contact] - assert len(contacts) == 1 - assert contacts[0].get_snapshot().address == ac1_addr - assert list(reactions.reactions_by_contact.values())[0] == [react_str] - - -@pytest.mark.parametrize("n_accounts", [3, 2]) -def test_download_limit_chat_assignment(acfactory, tmp_path, n_accounts): - download_limit = 300000 - - alice, *others = acfactory.get_online_accounts(n_accounts) - bob = others[0] - - alice_group = alice.create_group("test group") - for account in others: - chat = account.create_chat(alice) - chat.send_text("Hello Alice!") - assert alice.wait_for_incoming_msg().get_snapshot().text == "Hello Alice!" - - contact = alice.create_contact(account) - alice_group.add_contact(contact) - - bob.set_config("download_limit", str(download_limit)) - - alice_group.send_text("hi") - snapshot = bob.wait_for_incoming_msg().get_snapshot() - assert snapshot.text == "hi" - bob_group = snapshot.chat - - path = tmp_path / "large" - path.write_bytes(os.urandom(download_limit + 1)) - - for i in range(10): - logging.info("Sending message %s", i) - alice_group.send_file(str(path)) - snapshot = bob.wait_for_incoming_msg().get_snapshot() - assert snapshot.download_state == DownloadState.AVAILABLE - assert snapshot.chat == bob_group - - def test_markseen_contact_request(acfactory): """ Test that seen status is synchronized for contact request messages diff --git a/python/tests/test_1_online.py b/python/tests/test_1_online.py index d673dc94a2..b479af16d9 100644 --- a/python/tests/test_1_online.py +++ b/python/tests/test_1_online.py @@ -1,7 +1,6 @@ import os import queue import sys -import base64 from datetime import datetime, timezone import pytest @@ -222,38 +221,6 @@ def test_webxdc_huge_update(acfactory, data, lp): assert update["payload"] == payload -def test_webxdc_download_on_demand(acfactory, data, lp): - ac1, ac2 = acfactory.get_online_accounts(2) - acfactory.introduce_each_other([ac1, ac2]) - chat = acfactory.get_accepted_chat(ac1, ac2) - - msg1 = Message.new_empty(ac1, "webxdc") - msg1.set_text("message1") - msg1.set_file(data.get_path("webxdc/minimal.xdc")) - msg1 = chat.send_msg(msg1) - assert msg1.is_webxdc() - assert msg1.filename - - msg2 = ac2._evtracker.wait_next_incoming_message() - assert msg2.is_webxdc() - - lp.sec("ac2 sets download limit") - ac2.set_config("download_limit", "100") - assert msg1.send_status_update({"payload": base64.b64encode(os.urandom(300000))}, "some test data") - ac2_update = ac2._evtracker.wait_next_incoming_message() - assert ac2_update.download_state == dc.const.DC_DOWNLOAD_AVAILABLE - assert not msg2.get_status_updates() - - ac2_update.download_full() - ac2._evtracker.get_matching("DC_EVENT_WEBXDC_STATUS_UPDATE") - assert msg2.get_status_updates() - - # Get a event notifying that the message disappeared from the chat. - msgs_changed_event = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") - assert msgs_changed_event.data1 == msg2.chat.id - assert msgs_changed_event.data2 == 0 - - def test_enable_mvbox_move(acfactory, lp): (ac1,) = acfactory.get_online_accounts(1) diff --git a/src/calls/calls_tests.rs b/src/calls/calls_tests.rs index 3f983d8438..23b3947a2e 100644 --- a/src/calls/calls_tests.rs +++ b/src/calls/calls_tests.rs @@ -2,7 +2,7 @@ use super::*; use crate::chat::forward_msgs; use crate::config::Config; use crate::constants::DC_CHAT_ID_TRASH; -use crate::receive_imf::{receive_imf, receive_imf_from_inbox}; +use crate::receive_imf::receive_imf; use crate::test_utils::{TestContext, TestContextManager}; struct CallSetup { @@ -610,65 +610,3 @@ async fn test_end_text_call() -> Result<()> { Ok(()) } - -/// Tests that partially downloaded "call ended" -/// messages are not processed. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_no_partial_calls() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - - let seen = false; - - // The messages in the test - // have no `Date` on purpose, - // so they are treated as new. - let received_call = receive_imf( - alice, - b"From: bob@example.net\n\ - To: alice@example.org\n\ - Message-ID: \n\ - Chat-Version: 1.0\n\ - Chat-Content: call\n\ - Chat-Webrtc-Room: YWFhYWFhYWFhCg==\n\ - \n\ - Hello, this is a call\n", - seen, - ) - .await? - .unwrap(); - assert_eq!(received_call.msg_ids.len(), 1); - let call_msg = Message::load_from_db(alice, received_call.msg_ids[0]) - .await - .unwrap(); - assert_eq!(call_msg.viewtype, Viewtype::Call); - assert_eq!(call_state(alice, call_msg.id).await?, CallState::Alerting); - - let imf_raw = b"From: bob@example.net\n\ - To: alice@example.org\n\ - Message-ID: \n\ - In-Reply-To: \n\ - Chat-Version: 1.0\n\ - Chat-Content: call-ended\n\ - \n\ - Call ended\n"; - receive_imf_from_inbox( - alice, - "second@example.net", - imf_raw, - seen, - Some(imf_raw.len().try_into().unwrap()), - ) - .await?; - - // The call is still not ended. - assert_eq!(call_state(alice, call_msg.id).await?, CallState::Alerting); - - // Fully downloading the message ends the call. - receive_imf_from_inbox(alice, "second@example.net", imf_raw, seen, None) - .await - .context("Failed to fully download end call message")?; - assert_eq!(call_state(alice, call_msg.id).await?, CallState::Missed); - - Ok(()) -} diff --git a/src/chat.rs b/src/chat.rs index d3f710c183..44858f459a 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -12,6 +12,7 @@ use std::time::Duration; use anyhow::{Context as _, Result, anyhow, bail, ensure}; use chrono::TimeZone; use deltachat_contact_tools::{ContactAddress, sanitize_bidi_characters, sanitize_single_line}; +use humansize::{BINARY, format_size}; use mail_builder::mime::MimePart; use serde::{Deserialize, Serialize}; use strum_macros::EnumIter; @@ -27,7 +28,9 @@ use crate::constants::{ use crate::contact::{self, Contact, ContactId, Origin}; use crate::context::Context; use crate::debug_logging::maybe_set_logging_xdc; -use crate::download::DownloadState; +use crate::download::{ + DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD, PRE_MSG_SIZE_WARNING_THRESHOLD, +}; use crate::ephemeral::{Timer as EphemeralTimer, start_chat_ephemeral_timers}; use crate::events::EventType; use crate::key::self_fingerprint; @@ -35,7 +38,7 @@ use crate::location; use crate::log::{LogExt, warn}; use crate::logged_debug_assert; use crate::message::{self, Message, MessageState, MsgId, Viewtype}; -use crate::mimefactory::MimeFactory; +use crate::mimefactory::{MimeFactory, RenderedEmail}; use crate::mimeparser::SystemMessage; use crate::param::{Param, Params}; use crate::receive_imf::ReceivedMsg; @@ -2728,6 +2731,57 @@ async fn prepare_send_msg( Ok(row_ids) } +/// Renders the Message or splits it into Post-Message and Pre-Message. +/// +/// Pre-Message is a small message with metadata which announces a larger Post-Message. +/// Post-Messages are not downloaded in the background. +/// +/// If pre-message is not nessesary this returns a normal message instead. +async fn render_mime_message_and_pre_message( + context: &Context, + msg: &mut Message, + mimefactory: MimeFactory, +) -> Result<(RenderedEmail, Option)> { + let needs_pre_message = msg.viewtype.has_file() + && mimefactory.will_be_encrypted() // unencrypted is likely email, we don't want to spam by sending multiple messages + && msg + .get_filebytes(context) + .await? + .context("filebytes not available, even though message has attachment")? + > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD; + + if needs_pre_message { + info!( + context, + "Message is large and will be split into a pre- and a post-message.", + ); + + let mut mimefactory_post_msg = mimefactory.clone(); + mimefactory_post_msg.set_as_post_message(); + let rendered_msg = mimefactory_post_msg.render(context).await?; + + let mut mimefactory_pre_msg = mimefactory; + mimefactory_pre_msg.set_as_pre_message_for(&rendered_msg); + let rendered_pre_msg = mimefactory_pre_msg + .render(context) + .await + .context("pre-message failed to render")?; + + if rendered_pre_msg.message.len() > PRE_MSG_SIZE_WARNING_THRESHOLD { + warn!( + context, + "Pre-message for message (MsgId={}) is larger than expected: {}.", + msg.id, + rendered_pre_msg.message.len() + ); + } + + Ok((rendered_msg, Some(rendered_pre_msg))) + } else { + Ok((mimefactory.render(context).await?, None)) + } +} + /// Constructs jobs for sending a message and inserts them into the appropriate table. /// /// Updates the message `GuaranteeE2ee` parameter and persists it @@ -2799,13 +2853,29 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) - return Ok(Vec::new()); } - let rendered_msg = match mimefactory.render(context).await { - Ok(res) => Ok(res), - Err(err) => { - message::set_msg_failed(context, msg, &err.to_string()).await?; - Err(err) - } - }?; + let (rendered_msg, rendered_pre_msg) = + match render_mime_message_and_pre_message(context, msg, mimefactory).await { + Ok(res) => Ok(res), + Err(err) => { + message::set_msg_failed(context, msg, &err.to_string()).await?; + Err(err) + } + }?; + + if let (post_msg, Some(pre_msg)) = (&rendered_msg, &rendered_pre_msg) { + info!( + context, + "Message Sizes: Pre-Message {}; Post-Message: {}", + format_size(pre_msg.message.len(), BINARY), + format_size(post_msg.message.len(), BINARY) + ); + } else { + info!( + context, + "Message will be sent as normal message (no pre- and post message). Size: {}", + format_size(rendered_msg.message.len(), BINARY) + ); + } if needs_encryption && !rendered_msg.is_encrypted { /* unrecoverable */ @@ -2864,12 +2934,26 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) - } else { for recipients_chunk in recipients.chunks(chunk_size) { let recipients_chunk = recipients_chunk.join(" "); + // send pre-message before actual message + if let Some(pre_msg) = &rendered_pre_msg { + let row_id = t.execute( + "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id) + VALUES (?1, ?2, ?3, ?4)", + ( + &pre_msg.rfc724_mid, + &recipients_chunk, + &pre_msg.message, + msg.id, + ), + )?; + row_ids.push(row_id.try_into()?); + } let row_id = t.execute( - "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id) \ + "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id) VALUES (?1, ?2, ?3, ?4)", ( &rendered_msg.rfc724_mid, - recipients_chunk, + &recipients_chunk, &rendered_msg.message, msg.id, ), @@ -4256,6 +4340,14 @@ pub async fn forward_msgs_2ctx( msg.viewtype = Viewtype::Text; } + if msg.download_state != DownloadState::Done { + // we don't use Message.get_text() here, + // because it may change in future, + // when UI shows this info itself, + // then the additional_text will not be added in get_text anymore. + msg.text += &msg.additional_text; + } + let param = &mut param; msg.param.steal(param, Param::File); msg.param.steal(param, Param::Filename); @@ -4332,12 +4424,22 @@ pub(crate) async fn save_copy_in_self_talk( msg.param.remove(Param::WebxdcDocumentTimestamp); msg.param.remove(Param::WebxdcSummary); msg.param.remove(Param::WebxdcSummaryTimestamp); + msg.param.remove(Param::PostMessageFileBytes); + msg.param.remove(Param::PostMessageViewtype); + + if msg.download_state != DownloadState::Done { + // we don't use Message.get_text() here, + // because it may change in future, + // when UI shows this info itself, + // then the additional_text will not be added in get_text anymore. + msg.text += &msg.additional_text; + } if !msg.original_msg_id.is_unset() { bail!("message already saved."); } - let copy_fields = "from_id, to_id, timestamp_rcvd, type, txt, + let copy_fields = "from_id, to_id, timestamp_rcvd, type, mime_modified, mime_headers, mime_compressed, mime_in_reply_to, subject, msgrmsg"; let row_id = context .sql @@ -4345,7 +4447,7 @@ pub(crate) async fn save_copy_in_self_talk( &format!( "INSERT INTO msgs ({copy_fields}, timestamp_sent, - chat_id, rfc724_mid, state, timestamp, param, starred) + txt, chat_id, rfc724_mid, state, timestamp, param, starred) SELECT {copy_fields}, -- Outgoing messages on originating device -- have timestamp_sent == 0. @@ -4353,10 +4455,11 @@ pub(crate) async fn save_copy_in_self_talk( -- so UIs display the same timestamp -- for saved and original message. IIF(timestamp_sent == 0, timestamp, timestamp_sent), - ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ? FROM msgs WHERE id=?;" ), ( + msg.text, dest_chat_id, dest_rfc724_mid, if msg.from_id == ContactId::SELF { diff --git a/src/chat/chat_tests.rs b/src/chat/chat_tests.rs index a93ab2179a..1e8cff82fb 100644 --- a/src/chat/chat_tests.rs +++ b/src/chat/chat_tests.rs @@ -3116,7 +3116,7 @@ async fn test_broadcast_channel_protected_listid() -> Result<()> { .await? .grpid; - let parsed = mimeparser::MimeMessage::from_bytes(bob, sent.payload.as_bytes(), None).await?; + let parsed = mimeparser::MimeMessage::from_bytes(bob, sent.payload.as_bytes()).await?; assert_eq!( parsed.get_mailinglist_header().unwrap(), format!("My Channel <{}>", alice_list_id) @@ -3311,7 +3311,7 @@ async fn test_leave_broadcast_multidevice() -> Result<()> { remove_contact_from_chat(bob0, bob_chat_id, ContactId::SELF).await?; let leave_msg = bob0.pop_sent_msg().await; - let parsed = MimeMessage::from_bytes(bob1, leave_msg.payload().as_bytes(), None).await?; + let parsed = MimeMessage::from_bytes(bob1, leave_msg.payload().as_bytes()).await?; assert_eq!(parsed.parts[0].msg, "I left the group."); let rcvd = bob1.recv_msg(&leave_msg).await; diff --git a/src/config.rs b/src/config.rs index c065709bed..b133a2fe50 100644 --- a/src/config.rs +++ b/src/config.rs @@ -438,14 +438,14 @@ pub enum Config { /// using this still run unmodified code. TestHooks, - /// Return an error from `receive_imf_inner()` for a fully downloaded message. For tests. - FailOnReceivingFullMsg, - /// Enable composing emails with Header Protection as defined in /// "Header Protection for Cryptographically /// Protected Email". #[strum(props(default = "1"))] StdHeaderProtectionComposing, + + /// Return an error from `receive_imf_inner()`. For tests. + SimulateReceiveImfError, } impl Config { diff --git a/src/context.rs b/src/context.rs index c4bd71c521..95db60301c 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1083,13 +1083,6 @@ impl Context { .await? .unwrap_or_default(), ); - res.insert( - "fail_on_receiving_full_msg", - self.sql - .get_raw_config("fail_on_receiving_full_msg") - .await? - .unwrap_or_default(), - ); res.insert( "std_header_protection_composing", self.sql diff --git a/src/context/context_tests.rs b/src/context/context_tests.rs index 4a20c3af37..c4ffc1648a 100644 --- a/src/context/context_tests.rs +++ b/src/context/context_tests.rs @@ -297,6 +297,7 @@ async fn test_get_info_completeness() { "encrypted_device_token", "stats_last_update", "stats_last_old_contact_id", + "simulate_receive_imf_error", // only used in tests ]; let t = TestContext::new().await; let info = t.get_info().await.unwrap(); diff --git a/src/download.rs b/src/download.rs index a2dfce7758..fd4655381c 100644 --- a/src/download.rs +++ b/src/download.rs @@ -1,27 +1,18 @@ //! # Download large messages manually. -use std::cmp::max; use std::collections::BTreeMap; use anyhow::{Result, anyhow, bail, ensure}; use deltachat_derive::{FromSql, ToSql}; use serde::{Deserialize, Serialize}; -use crate::config::Config; use crate::context::Context; use crate::imap::session::Session; -use crate::message::{Message, MsgId, Viewtype}; -use crate::mimeparser::{MimeMessage, Part}; -use crate::tools::time; -use crate::{EventType, chatlist_events, stock_str}; +use crate::log::warn; +use crate::message::{self, Message, MsgId, rfc724_mid_exists}; +use crate::{EventType, chatlist_events}; -/// Download limits should not be used below `MIN_DOWNLOAD_LIMIT`. -/// -/// For better UX, some messages as add-member, non-delivery-reports (NDN) or read-receipts (MDN) -/// should always be downloaded completely to handle them correctly, -/// also in larger groups and if group and contact avatar are attached. -/// Most of these cases are caught by `MIN_DOWNLOAD_LIMIT`. -pub(crate) const MIN_DOWNLOAD_LIMIT: u32 = 163840; +pub(crate) mod pre_msg_metadata; /// If a message is downloaded only partially /// and `delete_server_after` is set to small timeouts (eg. "at once"), @@ -29,6 +20,15 @@ pub(crate) const MIN_DOWNLOAD_LIMIT: u32 = 163840; /// `MIN_DELETE_SERVER_AFTER` increases the timeout in this case. pub(crate) const MIN_DELETE_SERVER_AFTER: i64 = 48 * 60 * 60; +/// From this point onward outgoing messages are considered large +/// and get a Pre-Message, which announces the Post-Message. +// this is only about sending so we can modify it any time. +// current value is a bit less than the minimum auto download setting from the UIs (which is 160 KiB) +pub(crate) const PRE_MSG_ATTACHMENT_SIZE_THRESHOLD: u64 = 140_000; + +/// Max size for pre messages. A warning is emitted when this is exceeded. +pub(crate) const PRE_MSG_SIZE_WARNING_THRESHOLD: usize = 150_000; + /// Download state of the message. #[derive( Debug, @@ -64,20 +64,8 @@ pub enum DownloadState { InProgress = 1000, } -impl Context { - // Returns validated download limit or `None` for "no limit". - pub(crate) async fn download_limit(&self) -> Result> { - let download_limit = self.get_config_int(Config::DownloadLimit).await?; - if download_limit <= 0 { - Ok(None) - } else { - Ok(Some(max(MIN_DOWNLOAD_LIMIT, download_limit as u32))) - } - } -} - impl MsgId { - /// Schedules full message download for partially downloaded message. + /// Schedules Post-Message download for partially downloaded message. pub async fn download_full(self, context: &Context) -> Result<()> { let msg = Message::load_from_db(context, self).await?; match msg.download_state() { @@ -86,11 +74,17 @@ impl MsgId { } DownloadState::InProgress => return Err(anyhow!("Download already in progress.")), DownloadState::Available | DownloadState::Failure => { + if msg.rfc724_mid().is_empty() { + return Err(anyhow!("Download not possible, message has no rfc724_mid")); + } self.update_download_state(context, DownloadState::InProgress) .await?; context .sql - .execute("INSERT INTO download (msg_id) VALUES (?)", (self,)) + .execute( + "INSERT INTO download (rfc724_mid, msg_id) VALUES (?,?)", + (msg.rfc724_mid(), msg.id), + ) .await?; context.scheduler.interrupt_inbox().await; } @@ -139,25 +133,14 @@ impl Message { /// Most messages are downloaded automatically on fetch instead. pub(crate) async fn download_msg( context: &Context, - msg_id: MsgId, + rfc724_mid: String, session: &mut Session, ) -> Result<()> { - let Some(msg) = Message::load_from_db_optional(context, msg_id).await? else { - // If partially downloaded message was already deleted - // we do not know its Message-ID anymore - // so cannot download it. - // - // Probably the message expired due to `delete_device_after` - // setting or was otherwise removed from the device, - // so we don't want it to reappear anyway. - return Ok(()); - }; - let row = context .sql .query_row_optional( "SELECT uid, folder FROM imap WHERE rfc724_mid=? AND target!=''", - (&msg.rfc724_mid,), + (&rfc724_mid,), |row| { let server_uid: u32 = row.get(0)?; let server_folder: String = row.get(1)?; @@ -172,7 +155,7 @@ pub(crate) async fn download_msg( }; session - .fetch_single_msg(context, &server_folder, server_uid, msg.rfc724_mid.clone()) + .fetch_single_msg(context, &server_folder, server_uid, rfc724_mid) .await?; Ok(()) } @@ -205,7 +188,7 @@ impl Session { let mut uid_message_ids: BTreeMap = BTreeMap::new(); uid_message_ids.insert(uid, rfc724_mid); let (sender, receiver) = async_channel::unbounded(); - self.fetch_many_msgs(context, folder, vec![uid], &uid_message_ids, false, sender) + self.fetch_many_msgs(context, folder, vec![uid], &uid_message_ids, sender) .await?; if receiver.recv().await.is_err() { bail!("Failed to fetch UID {uid}"); @@ -214,41 +197,143 @@ impl Session { } } -impl MimeMessage { - /// Creates a placeholder part and add that to `parts`. - /// - /// To create the placeholder, only the outermost header can be used, - /// the mime-structure itself is not available. - /// - /// The placeholder part currently contains a text with size and availability of the message. - pub(crate) async fn create_stub_from_partial_download( - &mut self, - context: &Context, - org_bytes: u32, - ) -> Result<()> { - let mut text = format!( - "[{}]", - stock_str::partial_download_msg_body(context, org_bytes).await - ); - if let Some(delete_server_after) = context.get_config_delete_server_after().await? { - let until = stock_str::download_availability( - context, - time() + max(delete_server_after, MIN_DELETE_SERVER_AFTER), - ) - .await; - text += format!(" [{until}]").as_str(); - }; +async fn set_msg_state_to_failed(context: &Context, rfc724_mid: &str) -> Result<()> { + if let Some(msg_id) = rfc724_mid_exists(context, rfc724_mid).await? { + // Update download state to failure + // so it can be retried. + // + // On success update_download_state() is not needed + // as receive_imf() already + // set the state and emitted the event. + msg_id + .update_download_state(context, DownloadState::Failure) + .await?; + } + Ok(()) +} - info!(context, "Partial download: {}", text); +async fn available_post_msgs_contains_rfc724_mid( + context: &Context, + rfc724_mid: &str, +) -> Result { + Ok(context + .sql + .query_get_value::( + "SELECT rfc724_mid FROM available_post_msgs WHERE rfc724_mid=?", + (&rfc724_mid,), + ) + .await? + .is_some()) +} - self.do_add_single_part(Part { - typ: Viewtype::Text, - msg: text, - ..Default::default() - }); +async fn remove_from_available_post_msgs_table(context: &Context, rfc724_mid: &str) -> Result<()> { + context + .sql + .execute( + "DELETE FROM available_post_msgs WHERE rfc724_mid=?", + (&rfc724_mid,), + ) + .await?; + Ok(()) +} - Ok(()) +async fn remove_from_download_table(context: &Context, rfc724_mid: &str) -> Result<()> { + context + .sql + .execute("DELETE FROM download WHERE rfc724_mid=?", (&rfc724_mid,)) + .await?; + Ok(()) +} + +// this is a dedicated method because it is used in multiple places. +pub(crate) async fn premessage_is_downloaded_for( + context: &Context, + rfc724_mid: &str, +) -> Result { + Ok(message::rfc724_mid_exists(context, rfc724_mid) + .await? + .is_some()) +} + +pub(crate) async fn download_msgs(context: &Context, session: &mut Session) -> Result<()> { + let rfc724_mids = context + .sql + .query_map_vec("SELECT rfc724_mid FROM download", (), |row| { + let rfc724_mid: String = row.get(0)?; + Ok(rfc724_mid) + }) + .await?; + + for rfc724_mid in &rfc724_mids { + let res = download_msg(context, rfc724_mid.clone(), session).await; + if res.is_ok() { + remove_from_download_table(context, rfc724_mid).await?; + remove_from_available_post_msgs_table(context, rfc724_mid).await?; + } + if let Err(err) = res { + warn!( + context, + "Failed to download message rfc724_mid={rfc724_mid}: {:#}.", err + ); + if !premessage_is_downloaded_for(context, rfc724_mid).await? { + // This is probably a classical email that vanished before we could download it + warn!( + context, + "{rfc724_mid} is probably a classical email that vanished before we could download it" + ); + remove_from_download_table(context, rfc724_mid).await?; + } else if available_post_msgs_contains_rfc724_mid(context, rfc724_mid).await? { + warn!( + context, + "{rfc724_mid} is in available_post_msgs table but we failed to fetch it, + so set the message to DownloadState::Failure - probably it was deleted on the server in the meantime" + ); + set_msg_state_to_failed(context, rfc724_mid).await?; + remove_from_download_table(context, rfc724_mid).await?; + remove_from_available_post_msgs_table(context, rfc724_mid).await?; + } else { + // leave the message in DownloadState::InProgress; + // it will be downloaded once it arrives. + } + } } + + Ok(()) +} + +/// Download known post messages without pre_message +/// in order to guard against lost pre-messages: +pub(crate) async fn download_known_post_messages_without_pre_message( + context: &Context, + session: &mut Session, +) -> Result<()> { + let rfc724_mids = context + .sql + .query_map_vec("SELECT rfc724_mid FROM available_post_msgs", (), |row| { + let rfc724_mid: String = row.get(0)?; + Ok(rfc724_mid) + }) + .await?; + for rfc724_mid in &rfc724_mids { + if !premessage_is_downloaded_for(context, rfc724_mid).await? { + // Download the Post-Message unconditionally, + // because the Pre-Message got lost. + // The message may be in the wrong order, + // but at least we have it at all. + let res = download_msg(context, rfc724_mid.clone(), session).await; + if res.is_ok() { + remove_from_available_post_msgs_table(context, rfc724_mid).await?; + } + if let Err(err) = res { + warn!( + context, + "download_known_post_messages_without_pre_message: Failed to download message rfc724_mid={rfc724_mid}: {:#}.", + err + ); + } + } + } + Ok(()) } #[cfg(test)] @@ -256,11 +341,8 @@ mod tests { use num_traits::FromPrimitive; use super::*; - use crate::chat::{get_chat_msgs, send_msg}; - use crate::ephemeral::Timer; - use crate::message::delete_msgs; - use crate::receive_imf::receive_imf_from_inbox; - use crate::test_utils::{E2EE_INFO_MSGS, TestContext, TestContextManager}; + use crate::chat::send_msg; + use crate::test_utils::TestContext; #[test] fn test_downloadstate_values() { @@ -278,29 +360,6 @@ mod tests { ); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_download_limit() -> Result<()> { - let t = TestContext::new_alice().await; - - assert_eq!(t.download_limit().await?, None); - - t.set_config(Config::DownloadLimit, Some("200000")).await?; - assert_eq!(t.download_limit().await?, Some(200000)); - - t.set_config(Config::DownloadLimit, Some("20000")).await?; - assert_eq!(t.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT)); - - t.set_config(Config::DownloadLimit, None).await?; - assert_eq!(t.download_limit().await?, None); - - for val in &["0", "-1", "-100", "", "foo"] { - t.set_config(Config::DownloadLimit, Some(val)).await?; - assert_eq!(t.download_limit().await?, None); - } - - Ok(()) - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_update_download_state() -> Result<()> { let t = TestContext::new_alice().await; @@ -332,230 +391,4 @@ mod tests { Ok(()) } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_partial_receive_imf() -> Result<()> { - let t = TestContext::new_alice().await; - - let header = "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ - From: bob@example.com\n\ - To: alice@example.org\n\ - Subject: foo\n\ - Message-ID: \n\ - Chat-Version: 1.0\n\ - Date: Sun, 22 Mar 2020 22:37:57 +0000\ - Content-Type: text/plain"; - - receive_imf_from_inbox( - &t, - "Mr.12345678901@example.com", - header.as_bytes(), - false, - Some(100000), - ) - .await?; - let msg = t.get_last_msg().await; - assert_eq!(msg.download_state(), DownloadState::Available); - assert_eq!(msg.get_subject(), "foo"); - assert!( - msg.get_text() - .contains(&stock_str::partial_download_msg_body(&t, 100000).await) - ); - - receive_imf_from_inbox( - &t, - "Mr.12345678901@example.com", - format!("{header}\n\n100k text...").as_bytes(), - false, - None, - ) - .await?; - let msg = t.get_last_msg().await; - assert_eq!(msg.download_state(), DownloadState::Done); - assert_eq!(msg.get_subject(), "foo"); - assert_eq!(msg.get_text(), "100k text..."); - - Ok(()) - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_partial_download_and_ephemeral() -> Result<()> { - let t = TestContext::new_alice().await; - let chat_id = t - .create_chat_with_contact("bob", "bob@example.org") - .await - .id; - chat_id - .set_ephemeral_timer(&t, Timer::Enabled { duration: 60 }) - .await?; - - // download message from bob partially, this must not change the ephemeral timer - receive_imf_from_inbox( - &t, - "first@example.org", - b"From: Bob \n\ - To: Alice \n\ - Chat-Version: 1.0\n\ - Subject: subject\n\ - Message-ID: \n\ - Date: Sun, 14 Nov 2021 00:10:00 +0000\ - Content-Type: text/plain", - false, - Some(100000), - ) - .await?; - assert_eq!( - chat_id.get_ephemeral_timer(&t).await?, - Timer::Enabled { duration: 60 } - ); - - Ok(()) - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_status_update_expands_to_nothing() -> Result<()> { - let alice = TestContext::new_alice().await; - let bob = TestContext::new_bob().await; - let chat_id = alice.create_chat(&bob).await.id; - - let file = alice.get_blobdir().join("minimal.xdc"); - tokio::fs::write(&file, include_bytes!("../test-data/webxdc/minimal.xdc")).await?; - let mut instance = Message::new(Viewtype::File); - instance.set_file_and_deduplicate(&alice, &file, None, None)?; - let _sent1 = alice.send_msg(chat_id, &mut instance).await; - - alice - .send_webxdc_status_update(instance.id, r#"{"payload":7}"#) - .await?; - alice.flush_status_updates().await?; - let sent2 = alice.pop_sent_msg().await; - let sent2_rfc724_mid = sent2.load_from_db().await.rfc724_mid; - - // not downloading the status update results in an placeholder - receive_imf_from_inbox( - &bob, - &sent2_rfc724_mid, - sent2.payload().as_bytes(), - false, - Some(sent2.payload().len() as u32), - ) - .await?; - let msg = bob.get_last_msg().await; - let chat_id = msg.chat_id; - assert_eq!( - get_chat_msgs(&bob, chat_id).await?.len(), - E2EE_INFO_MSGS + 1 - ); - assert_eq!(msg.download_state(), DownloadState::Available); - - // downloading the status update afterwards expands to nothing and moves the placeholder to trash-chat - // (usually status updates are too small for not being downloaded directly) - receive_imf_from_inbox( - &bob, - &sent2_rfc724_mid, - sent2.payload().as_bytes(), - false, - None, - ) - .await?; - assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), E2EE_INFO_MSGS); - assert!( - Message::load_from_db_optional(&bob, msg.id) - .await? - .is_none() - ); - - Ok(()) - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_mdn_expands_to_nothing() -> Result<()> { - let bob = TestContext::new_bob().await; - let raw = b"Subject: Message opened\n\ - Date: Mon, 10 Jan 2020 00:00:00 +0000\n\ - Chat-Version: 1.0\n\ - Message-ID: \n\ - To: Alice \n\ - From: Bob \n\ - Content-Type: multipart/report; report-type=disposition-notification;\n\t\ - boundary=\"kJBbU58X1xeWNHgBtTbMk80M5qnV4N\"\n\ - \n\ - \n\ - --kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\ - Content-Type: text/plain; charset=utf-8\n\ - \n\ - bla\n\ - \n\ - \n\ - --kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\ - Content-Type: message/disposition-notification\n\ - \n\ - Reporting-UA: Delta Chat 1.88.0\n\ - Original-Recipient: rfc822;bob@example.org\n\ - Final-Recipient: rfc822;bob@example.org\n\ - Original-Message-ID: \n\ - Disposition: manual-action/MDN-sent-automatically; displayed\n\ - \n\ - \n\ - --kJBbU58X1xeWNHgBtTbMk80M5qnV4N--\n\ - "; - - // not downloading the mdn results in an placeholder - receive_imf_from_inbox(&bob, "bar@example.org", raw, false, Some(raw.len() as u32)).await?; - let msg = bob.get_last_msg().await; - let chat_id = msg.chat_id; - assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), 1); - assert_eq!(msg.download_state(), DownloadState::Available); - - // downloading the mdn afterwards expands to nothing and deletes the placeholder directly - // (usually mdn are too small for not being downloaded directly) - receive_imf_from_inbox(&bob, "bar@example.org", raw, false, None).await?; - assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), 0); - assert!( - Message::load_from_db_optional(&bob, msg.id) - .await? - .is_none() - ); - - Ok(()) - } - - /// Tests that fully downloading the message - /// works even if the Message-ID already exists - /// in the database assigned to the trash chat. - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_partial_download_trashed() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - - let imf_raw = b"From: Bob \n\ - To: Alice \n\ - Chat-Version: 1.0\n\ - Subject: subject\n\ - Message-ID: \n\ - Date: Sun, 14 Nov 2021 00:10:00 +0000\ - Content-Type: text/plain"; - - // Download message from Bob partially. - let partial_received_msg = - receive_imf_from_inbox(alice, "first@example.org", imf_raw, false, Some(100000)) - .await? - .unwrap(); - assert_eq!(partial_received_msg.msg_ids.len(), 1); - - // Delete the received message. - // Not it is still in the database, - // but in the trash chat. - delete_msgs(alice, &[partial_received_msg.msg_ids[0]]).await?; - - // Fully download message after deletion. - let full_received_msg = - receive_imf_from_inbox(alice, "first@example.org", imf_raw, false, None).await?; - - // The message does not reappear. - // However, `receive_imf` should not fail. - assert!(full_received_msg.is_none()); - - Ok(()) - } } diff --git a/src/download/pre_msg_metadata.rs b/src/download/pre_msg_metadata.rs new file mode 100644 index 0000000000..cc19b9e1f7 --- /dev/null +++ b/src/download/pre_msg_metadata.rs @@ -0,0 +1,248 @@ +use anyhow::{Context as _, Result}; +use num_traits::ToPrimitive; +use serde::{Deserialize, Serialize}; + +use crate::context::Context; +use crate::log::warn; +use crate::message::Message; +use crate::message::Viewtype; +use crate::param::{Param, Params}; + +/// Metadata contained in Pre-Message that describes the Post-Message. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PreMsgMetadata { + /// size of the attachment in bytes + pub(crate) size: u64, + /// Real viewtype of message + pub(crate) viewtype: Viewtype, + /// the original file name + pub(crate) filename: String, + /// Dimensions: width and height of image or video + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) dimensions: Option<(i32, i32)>, + /// Duration of audio file or video in milliseconds + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) duration: Option, +} + +impl PreMsgMetadata { + // Returns PreMsgMetadata for messages with files and None for messages without file attachment + pub(crate) async fn from_msg(context: &Context, message: &Message) -> Result> { + if !message.viewtype.has_file() { + return Ok(None); + } + + let size = message + .get_filebytes(context) + .await? + .context("Unexpected: file has no size")?; + let filename = message + .param + .get(Param::Filename) + .unwrap_or_default() + .to_owned(); + let dimensions = { + match ( + message.param.get_int(Param::Width), + message.param.get_int(Param::Height), + ) { + (None, None) => None, + (Some(width), Some(height)) => Some((width, height)), + _ => { + warn!(context, "Message misses either width or height."); + None + } + } + }; + let duration = message.param.get_int(Param::Duration); + + Ok(Some(Self { + size, + filename, + viewtype: message.viewtype, + dimensions, + duration, + })) + } + + pub(crate) fn to_header_value(&self) -> Result { + Ok(serde_json::to_string(&self)?) + } + + pub(crate) fn try_from_header_value(value: &str) -> Result { + Ok(serde_json::from_str(value)?) + } +} + +impl Params { + /// Applies data from pre_msg_metadata to Params + pub(crate) fn apply_from_pre_msg_metadata( + &mut self, + pre_msg_metadata: &PreMsgMetadata, + ) -> &mut Self { + self.set(Param::PostMessageFileBytes, pre_msg_metadata.size); + if !pre_msg_metadata.filename.is_empty() { + self.set(Param::Filename, &pre_msg_metadata.filename); + } + self.set_i64( + Param::PostMessageViewtype, + pre_msg_metadata.viewtype.to_i64().unwrap_or_default(), + ); + if let Some((width, height)) = pre_msg_metadata.dimensions { + self.set(Param::Width, width); + self.set(Param::Height, height); + } + if let Some(duration) = pre_msg_metadata.duration { + self.set(Param::Duration, duration); + } + + self + } +} + +#[cfg(test)] +mod tests { + use anyhow::Result; + use pretty_assertions::assert_eq; + + use crate::{ + message::{Message, Viewtype}, + test_utils::{TestContextManager, create_test_image}, + }; + + use super::PreMsgMetadata; + + /// Build from message with file attachment + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_build_from_file_msg() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + + let mut file_msg = Message::new(Viewtype::File); + file_msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 1_000_000], None)?; + let pre_mesage_metadata = PreMsgMetadata::from_msg(alice, &file_msg).await?; + assert_eq!( + pre_mesage_metadata, + Some(PreMsgMetadata { + size: 1_000_000, + viewtype: Viewtype::File, + filename: "test.bin".to_string(), + dimensions: None, + duration: None, + }) + ); + Ok(()) + } + + /// Build from message with image attachment + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_build_from_image_msg() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let mut image_msg = Message::new(Viewtype::Image); + + let (width, height) = (1080, 1920); + let test_img = create_test_image(width, height)?; + image_msg.set_file_from_bytes(alice, "vacation.png", &test_img, None)?; + // this is usually done while sending, + // but we don't send it here, so we need to call it ourself + image_msg.try_calc_and_set_dimensions(alice).await?; + let pre_mesage_metadata = PreMsgMetadata::from_msg(alice, &image_msg).await?; + assert_eq!( + pre_mesage_metadata, + Some(PreMsgMetadata { + size: 1816098, + viewtype: Viewtype::Image, + filename: "vacation.png".to_string(), + dimensions: Some((width as i32, height as i32)), + duration: None, + }) + ); + + Ok(()) + } + + /// Test that serialisation results in expected format + #[test] + fn test_serialize_to_header() -> Result<()> { + assert_eq!( + PreMsgMetadata { + size: 1_000_000, + viewtype: Viewtype::File, + filename: "test.bin".to_string(), + dimensions: None, + duration: None, + } + .to_header_value()?, + "{\"size\":1000000,\"viewtype\":\"File\",\"filename\":\"test.bin\"}" + ); + assert_eq!( + PreMsgMetadata { + size: 5_342_765, + viewtype: Viewtype::Image, + filename: "vacation.png".to_string(), + dimensions: Some((1080, 1920)), + duration: None, + } + .to_header_value()?, + "{\"size\":5342765,\"viewtype\":\"Image\",\"filename\":\"vacation.png\",\"dimensions\":[1080,1920]}" + ); + assert_eq!( + PreMsgMetadata { + size: 5_000, + viewtype: Viewtype::Audio, + filename: "audio-DD-MM-YY.ogg".to_string(), + dimensions: None, + duration: Some(152_310), + } + .to_header_value()?, + "{\"size\":5000,\"viewtype\":\"Audio\",\"filename\":\"audio-DD-MM-YY.ogg\",\"duration\":152310}" + ); + + Ok(()) + } + + /// Test that deserialisation from expected format works + /// This test will become important for compatibility between versions in the future + #[test] + fn test_deserialize_from_header() -> Result<()> { + assert_eq!( + serde_json::from_str::( + "{\"size\":1000000,\"viewtype\":\"File\",\"filename\":\"test.bin\",\"dimensions\":null,\"duration\":null}" + )?, + PreMsgMetadata { + size: 1_000_000, + viewtype: Viewtype::File, + filename: "test.bin".to_string(), + dimensions: None, + duration: None, + } + ); + assert_eq!( + serde_json::from_str::( + "{\"size\":5342765,\"viewtype\":\"Image\",\"filename\":\"vacation.png\",\"dimensions\":[1080,1920]}" + )?, + PreMsgMetadata { + size: 5_342_765, + viewtype: Viewtype::Image, + filename: "vacation.png".to_string(), + dimensions: Some((1080, 1920)), + duration: None, + } + ); + assert_eq!( + serde_json::from_str::( + "{\"size\":5000,\"viewtype\":\"Audio\",\"filename\":\"audio-DD-MM-YY.ogg\",\"duration\":152310}" + )?, + PreMsgMetadata { + size: 5_000, + viewtype: Viewtype::Audio, + filename: "audio-DD-MM-YY.ogg".to_string(), + dimensions: None, + duration: Some(152_310), + } + ); + + Ok(()) + } +} diff --git a/src/headerdef.rs b/src/headerdef.rs index c57f05033f..4baee1c57c 100644 --- a/src/headerdef.rs +++ b/src/headerdef.rs @@ -102,6 +102,21 @@ pub enum HeaderDef { /// used to encrypt and decrypt messages. /// This secret is sent to a new member in the member-addition message. ChatBroadcastSecret, + /// A message with a large attachment is split into two MIME messages: + /// A pre-message, which contains everything but the attachment, + /// and a Post-Message. + /// The Pre-Message gets a `Chat-Post-Message-Id` header + /// referencing the Post-Message's rfc724_mid. + ChatPostMessageId, + + /// Announce Post-Message metadata in a Pre-Message. + /// contains serialized PreMsgMetadata struct + ChatPostMessageMetadata, + + /// This message is preceded by a Pre-Message + /// and thus this message can be skipped while fetching messages. + /// This is a cleartext / unproteced header. + ChatIsPostMessage, /// [Autocrypt](https://autocrypt.org/) header. Autocrypt, @@ -147,6 +162,9 @@ pub enum HeaderDef { impl HeaderDef { /// Returns the corresponding header string. + /// + /// Format is lower-kebab-case for easy comparisons. + /// This method is used in message receiving and testing. pub fn get_headername(&self) -> &'static str { self.into() } diff --git a/src/imap.rs b/src/imap.rs index 006314225e..57549584a3 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -67,7 +67,6 @@ const RFC724MID_UID: &str = "(UID BODY.PEEK[HEADER.FIELDS (\ X-MICROSOFT-ORIGINAL-MESSAGE-ID\ )])"; const BODY_FULL: &str = "(FLAGS BODY.PEEK[])"; -const BODY_PARTIAL: &str = "(FLAGS RFC822.SIZE BODY.PEEK[HEADER])"; #[derive(Debug)] pub(crate) struct Imap { @@ -615,12 +614,23 @@ impl Imap { .context("prefetch")?; let read_cnt = msgs.len(); - let download_limit = context.download_limit().await?; - let mut uids_fetch = Vec::<(u32, bool /* partially? */)>::with_capacity(msgs.len() + 1); + let mut uids_fetch = Vec::::with_capacity(msgs.len() + 1); + let mut available_post_msgs = Vec::::with_capacity(msgs.len()); + let mut download_when_normal_starts = Vec::::with_capacity(msgs.len()); let mut uid_message_ids = BTreeMap::new(); let mut largest_uid_skipped = None; let delete_target = context.get_delete_msgs_target().await?; + let download_limit = { + let download_limit: Option = + context.get_config_parsed(Config::DownloadLimit).await?; + if download_limit == Some(0) { + None + } else { + download_limit + } + }; + // Store the info about IMAP messages in the database. for (uid, ref fetch_response) in msgs { let headers = match get_fetch_headers(fetch_response) { @@ -632,6 +642,9 @@ impl Imap { }; let message_id = prefetch_get_message_id(&headers); + let size = fetch_response + .size + .context("imap fetch response does not contain size")?; // Determine the target folder where the message should be moved to. // @@ -706,14 +719,23 @@ impl Imap { ) .await.context("prefetch_should_download")? { - match download_limit { - Some(download_limit) => uids_fetch.push(( - uid, - fetch_response.size.unwrap_or_default() > download_limit, - )), - None => uids_fetch.push((uid, false)), - } - uid_message_ids.insert(uid, message_id); + if headers + .get_header_value(HeaderDef::ChatIsPostMessage) + .is_some() + { + info!(context, "{message_id:?} is a post-message."); + available_post_msgs.push(message_id.clone()); + + // whether it fits download size limit + if download_limit.is_none_or(|download_limit| size < download_limit) { + download_when_normal_starts.push(message_id.clone()); + } + } else { + info!(context, "{message_id:?} is not a post-message."); + + uids_fetch.push(uid); + uid_message_ids.insert(uid, message_id); + }; } else { largest_uid_skipped = Some(uid); } @@ -747,29 +769,10 @@ impl Imap { }; let actually_download_messages_future = async { - let sender = sender; - let mut uids_fetch_in_batch = Vec::with_capacity(max(uids_fetch.len(), 1)); - let mut fetch_partially = false; - uids_fetch.push((0, !uids_fetch.last().unwrap_or(&(0, false)).1)); - for (uid, fp) in uids_fetch { - if fp != fetch_partially { - session - .fetch_many_msgs( - context, - folder, - uids_fetch_in_batch.split_off(0), - &uid_message_ids, - fetch_partially, - sender.clone(), - ) - .await - .context("fetch_many_msgs")?; - fetch_partially = fp; - } - uids_fetch_in_batch.push(uid); - } - - anyhow::Ok(()) + session + .fetch_many_msgs(context, folder, uids_fetch, &uid_message_ids, sender) + .await + .context("fetch_many_msgs") }; let (largest_uid_fetched, fetch_res) = @@ -804,6 +807,30 @@ impl Imap { chat::mark_old_messages_as_noticed(context, received_msgs).await?; + if fetch_res.is_ok() { + info!( + context, + "available_post_msgs: {}, download_when_normal_starts: {}", + available_post_msgs.len(), + download_when_normal_starts.len() + ); + for rfc724_mid in available_post_msgs { + context + .sql + .insert("INSERT INTO available_post_msgs VALUES (?)", (rfc724_mid,)) + .await?; + } + for rfc724_mid in download_when_normal_starts { + context + .sql + .insert( + "INSERT INTO download (rfc724_mid, msg_id) VALUES (?,0)", + (rfc724_mid,), + ) + .await?; + } + } + // Now fail if fetching failed, so we will // establish a new session if this one is broken. fetch_res?; @@ -1373,7 +1400,6 @@ impl Session { folder: &str, request_uids: Vec, uid_message_ids: &BTreeMap, - fetch_partially: bool, received_msgs_channel: Sender<(u32, Option)>, ) -> Result<()> { if request_uids.is_empty() { @@ -1381,25 +1407,10 @@ impl Session { } for (request_uids, set) in build_sequence_sets(&request_uids)? { - info!( - context, - "Starting a {} FETCH of message set \"{}\".", - if fetch_partially { "partial" } else { "full" }, - set - ); - let mut fetch_responses = self - .uid_fetch( - &set, - if fetch_partially { - BODY_PARTIAL - } else { - BODY_FULL - }, - ) - .await - .with_context(|| { - format!("fetching messages {} from folder \"{}\"", &set, folder) - })?; + info!(context, "Starting a full FETCH of message set \"{}\".", set); + let mut fetch_responses = self.uid_fetch(&set, BODY_FULL).await.with_context(|| { + format!("fetching messages {} from folder \"{}\"", &set, folder) + })?; // Map from UIDs to unprocessed FETCH results. We put unprocessed FETCH results here // when we want to process other messages first. @@ -1456,11 +1467,7 @@ impl Session { count += 1; let is_deleted = fetch_response.flags().any(|flag| flag == Flag::Deleted); - let (body, partial) = if fetch_partially { - (fetch_response.header(), fetch_response.size) // `BODY.PEEK[HEADER]` goes to header() ... - } else { - (fetch_response.body(), None) // ... while `BODY.PEEK[]` goes to body() - and includes header() - }; + let body = fetch_response.body(); if is_deleted { info!(context, "Not processing deleted msg {}.", request_uid); @@ -1494,7 +1501,7 @@ impl Session { context, "Passing message UID {} to receive_imf().", request_uid ); - let res = receive_imf_inner(context, rfc724_mid, body, is_seen, partial).await; + let res = receive_imf_inner(context, rfc724_mid, body, is_seen).await; let received_msg = match res { Err(err) => { warn!(context, "receive_imf error: {err:#}."); diff --git a/src/imap/session.rs b/src/imap/session.rs index 0da1d7936f..1d8c2d7110 100644 --- a/src/imap/session.rs +++ b/src/imap/session.rs @@ -17,6 +17,7 @@ use crate::tools; /// - Chat-Version to check if a message is a chat message /// - Autocrypt-Setup-Message to check if a message is an autocrypt setup message, /// not necessarily sent by Delta Chat. +/// - Chat-Is-Post-Message to skip it in background fetch or when it is too large const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIELDS (\ MESSAGE-ID \ DATE \ @@ -24,6 +25,7 @@ const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIE FROM \ IN-REPLY-TO REFERENCES \ CHAT-VERSION \ + CHAT-IS-POST-MESSAGE \ AUTO-SUBMITTED \ AUTOCRYPT-SETUP-MESSAGE\ )])"; diff --git a/src/internals_for_benches.rs b/src/internals_for_benches.rs index 1fd80c5516..61fc9d7fca 100644 --- a/src/internals_for_benches.rs +++ b/src/internals_for_benches.rs @@ -21,7 +21,7 @@ pub async fn store_self_keypair(context: &Context, keypair: &KeyPair) -> Result< } pub async fn parse_and_get_text(context: &Context, imf_raw: &[u8]) -> Result { - let mime_parser = MimeMessage::from_bytes(context, imf_raw, None).await?; + let mime_parser = MimeMessage::from_bytes(context, imf_raw).await?; Ok(mime_parser.parts.into_iter().next().unwrap().msg) } diff --git a/src/message.rs b/src/message.rs index 8dd19c11ec..b06fdf6087 100644 --- a/src/message.rs +++ b/src/message.rs @@ -8,6 +8,9 @@ use std::str; use anyhow::{Context as _, Result, ensure, format_err}; use deltachat_contact_tools::{VcardContact, parse_vcard}; use deltachat_derive::{FromSql, ToSql}; +use humansize::BINARY; +use humansize::format_size; +use num_traits::FromPrimitive; use serde::{Deserialize, Serialize}; use tokio::{fs, io}; @@ -425,6 +428,10 @@ pub struct Message { pub(crate) ephemeral_timer: EphemeralTimer, pub(crate) ephemeral_timestamp: i64, pub(crate) text: String, + /// Text that is added to the end of Message.text + /// + /// Currently used for adding the download information on pre-messages + pub(crate) additional_text: String, /// Message subject. /// @@ -483,7 +490,7 @@ impl Message { !id.is_special(), "Can not load special message ID {id} from DB" ); - let msg = context + let mut msg = context .sql .query_row_optional( concat!( @@ -565,6 +572,7 @@ impl Message { original_msg_id: row.get("original_msg_id")?, mime_modified: row.get("mime_modified")?, text, + additional_text: String::new(), subject: row.get("subject")?, param: row.get::<_, String>("param")?.parse().unwrap_or_default(), hidden: row.get("hidden")?, @@ -579,9 +587,48 @@ impl Message { .await .with_context(|| format!("failed to load message {id} from the database"))?; + if let Some(msg) = &mut msg { + msg.additional_text = + Self::get_additional_text(context, msg.download_state, &msg.param).await?; + } + Ok(msg) } + /// Returns additional text which is appended to the message's text field + /// when it is loaded from the database. + /// Currently this is used to add infomation to pre-messages of what the download will be and how large it is + async fn get_additional_text( + context: &Context, + download_state: DownloadState, + param: &Params, + ) -> Result { + if download_state != DownloadState::Done { + let file_size = param + .get(Param::PostMessageFileBytes) + .and_then(|s| s.parse().ok()) + .map(|file_size: usize| format_size(file_size, BINARY)) + .unwrap_or("?".to_owned()); + let viewtype = param + .get_i64(Param::PostMessageViewtype) + .and_then(Viewtype::from_i64) + .unwrap_or(Viewtype::Unknown); + let file_name = param + .get(Param::Filename) + .map(sanitize_filename) + .unwrap_or("?".to_owned()); + + return match viewtype { + Viewtype::File => Ok(format!(" [{file_name} - {file_size}]")), + _ => { + let translated_viewtype = viewtype.to_locale_string(context).await; + Ok(format!(" [{translated_viewtype} - {file_size}]")) + } + }; + } + Ok(String::new()) + } + /// Returns the MIME type of an attached file if it exists. /// /// If the MIME type is not known, the function guesses the MIME type @@ -764,7 +811,7 @@ impl Message { /// Returns the text of the message. pub fn get_text(&self) -> String { - self.text.clone() + self.text.clone() + &self.additional_text } /// Returns message subject. @@ -786,7 +833,17 @@ impl Message { } /// Returns the size of the file in bytes, if applicable. + /// If message is a pre-message, then this returns size of the to be downloaded file. pub async fn get_filebytes(&self, context: &Context) -> Result> { + // if download state is not downloaded then return value from from params metadata + if self.download_state != DownloadState::Done + && let Some(file_size) = self + .param + .get(Param::PostMessageFileBytes) + .and_then(|s| s.parse().ok()) + { + return Ok(Some(file_size)); + } if let Some(path) = self.param.get_file_path(context)? { Ok(Some(get_filebytes(context, &path).await.with_context( || format!("failed to get {} size in bytes", path.display()), @@ -796,6 +853,21 @@ impl Message { } } + /// If message is a Pre-Message, + /// then this returns the viewtype it will have when it is downloaded. + #[cfg(test)] + pub(crate) fn get_post_message_viewtype(&self) -> Option { + if self.download_state != DownloadState::Done + && let Some(viewtype) = self + .param + .get_i64(Param::PostMessageViewtype) + .and_then(Viewtype::from_i64) + { + return Some(viewtype); + } + None + } + /// Returns width of associated image or video file. pub fn get_width(&self) -> i32 { self.param.get_int(Param::Width).unwrap_or_default() @@ -1676,9 +1748,17 @@ pub async fn delete_msgs_ex( let update_db = |trans: &mut rusqlite::Transaction| { trans.execute( "UPDATE imap SET target=? WHERE rfc724_mid=?", - (target, msg.rfc724_mid), + (target, &msg.rfc724_mid), )?; trans.execute("DELETE FROM smtp WHERE msg_id=?", (msg_id,))?; + trans.execute( + "DELETE FROM download WHERE rfc724_mid=?", + (&msg.rfc724_mid,), + )?; + trans.execute( + "DELETE FROM available_post_msgs WHERE rfc724_mid=?", + (&msg.rfc724_mid,), + )?; Ok(()) }; if let Err(e) = context.sql.transaction(update_db).await { @@ -1746,7 +1826,6 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec) -> Result<()> "SELECT m.chat_id AS chat_id, m.state AS state, - m.download_state as download_state, m.ephemeral_timer AS ephemeral_timer, m.param AS param, m.from_id AS from_id, @@ -1759,7 +1838,6 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec) -> Result<()> |row| { let chat_id: ChatId = row.get("chat_id")?; let state: MessageState = row.get("state")?; - let download_state: DownloadState = row.get("download_state")?; let param: Params = row.get::<_, String>("param")?.parse().unwrap_or_default(); let from_id: ContactId = row.get("from_id")?; let rfc724_mid: String = row.get("rfc724_mid")?; @@ -1771,7 +1849,6 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec) -> Result<()> id, chat_id, state, - download_state, param, from_id, rfc724_mid, @@ -1804,7 +1881,6 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec) -> Result<()> id, curr_chat_id, curr_state, - curr_download_state, curr_param, curr_from_id, curr_rfc724_mid, @@ -1814,14 +1890,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec) -> Result<()> _curr_ephemeral_timer, ) in msgs { - if curr_download_state != DownloadState::Done { - if curr_state == MessageState::InFresh { - // Don't mark partially downloaded messages as seen or send a read receipt since - // they are not really seen by the user. - update_msg_state(context, id, MessageState::InNoticed).await?; - updated_chat_ids.insert(curr_chat_id); - } - } else if curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed { + if curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed { update_msg_state(context, id, MessageState::InSeen).await?; info!(context, "Seen message {}.", id); diff --git a/src/message/message_tests.rs b/src/message/message_tests.rs index be3ed01d9a..d43d21220e 100644 --- a/src/message/message_tests.rs +++ b/src/message/message_tests.rs @@ -326,112 +326,6 @@ async fn test_markseen_msgs() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_markseen_not_downloaded_msg() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - alice.set_config(Config::DownloadLimit, Some("1")).await?; - let bob = &tcm.bob().await; - let bob_chat_id = bob.create_chat(alice).await.id; - alice.create_chat(bob).await; // Make sure the chat is accepted. - - tcm.section("Bob sends a large message to Alice"); - let file_bytes = include_bytes!("../../test-data/image/screenshot.png"); - let mut msg = Message::new(Viewtype::Image); - msg.set_file_from_bytes(bob, "a.jpg", file_bytes, None)?; - let sent_msg = bob.send_msg(bob_chat_id, &mut msg).await; - - tcm.section("Alice receives a large message from Bob"); - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Available); - assert!(!msg.param.get_bool(Param::WantsMdn).unwrap_or_default()); - assert_eq!(msg.state, MessageState::InFresh); - markseen_msgs(alice, vec![msg.id]).await?; - // A not downloaded message can be seen only if it's seen on another device. - assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed); - // Marking the message as seen again is a no op. - markseen_msgs(alice, vec![msg.id]).await?; - assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed); - - msg.id - .update_download_state(alice, DownloadState::InProgress) - .await?; - markseen_msgs(alice, vec![msg.id]).await?; - assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed); - msg.id - .update_download_state(alice, DownloadState::Failure) - .await?; - markseen_msgs(alice, vec![msg.id]).await?; - assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed); - msg.id - .update_download_state(alice, DownloadState::Undecipherable) - .await?; - markseen_msgs(alice, vec![msg.id]).await?; - assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed); - - assert!( - !alice - .sql - .exists("SELECT COUNT(*) FROM smtp_mdns", ()) - .await? - ); - - alice.set_config(Config::DownloadLimit, None).await?; - // Let's assume that Alice and Bob resolved the problem with encryption. - let old_msg = msg; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.chat_id, old_msg.chat_id); - assert_eq!(msg.download_state, DownloadState::Done); - assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default()); - assert!(msg.get_showpadlock()); - // The message state mustn't be downgraded to `InFresh`. - assert_eq!(msg.state, MessageState::InNoticed); - markseen_msgs(alice, vec![msg.id]).await?; - let msg = Message::load_from_db(alice, msg.id).await?; - assert_eq!(msg.state, MessageState::InSeen); - assert_eq!( - alice - .sql - .count("SELECT COUNT(*) FROM smtp_mdns", ()) - .await?, - 1 - ); - Ok(()) -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_msg_seen_on_imap_when_downloaded() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - alice.set_config(Config::DownloadLimit, Some("1")).await?; - let bob = &tcm.bob().await; - let bob_chat_id = tcm.send_recv_accept(alice, bob, "hi").await.chat_id; - - let file_bytes = include_bytes!("../../test-data/image/screenshot.png"); - let mut msg = Message::new(Viewtype::Image); - msg.set_file_from_bytes(bob, "a.jpg", file_bytes, None)?; - let sent_msg = bob.send_msg(bob_chat_id, &mut msg).await; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Available); - assert_eq!(msg.state, MessageState::InFresh); - - alice.set_config(Config::DownloadLimit, None).await?; - let seen = true; - let rcvd_msg = receive_imf(alice, sent_msg.payload().as_bytes(), seen) - .await - .unwrap() - .unwrap(); - assert_eq!(rcvd_msg.chat_id, msg.chat_id); - let msg = Message::load_from_db(alice, *rcvd_msg.msg_ids.last().unwrap()) - .await - .unwrap(); - assert_eq!(msg.download_state, DownloadState::Done); - assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default()); - assert!(msg.get_showpadlock()); - assert_eq!(msg.state, MessageState::InSeen); - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_get_state() -> Result<()> { let alice = TestContext::new_alice().await; diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 732fff545a..049114847a 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -21,6 +21,7 @@ use crate::constants::{ASM_SUBJECT, BROADCAST_INCOMPATIBILITY_MSG}; use crate::constants::{Chattype, DC_FROM_HANDSHAKE}; use crate::contact::{Contact, ContactId, Origin}; use crate::context::Context; +use crate::download::pre_msg_metadata::PreMsgMetadata; use crate::e2ee::EncryptHelper; use crate::ensure_and_debug_assert; use crate::ephemeral::Timer as EphemeralTimer; @@ -59,6 +60,15 @@ pub enum Loaded { }, } +#[derive(Debug, Clone, PartialEq)] +pub enum PreMessageMode { + /// adds the Chat-Is-Post-Message header in unprotected part + PostMessage, + /// adds the Chat-Post-Message-ID header to protected part + /// also adds metadata and explicitly excludes attachment + PreMessage { post_msg_rfc724_mid: String }, +} + /// Helper to construct mime messages. #[derive(Debug, Clone)] pub struct MimeFactory { @@ -146,6 +156,9 @@ pub struct MimeFactory { /// This field is used to sustain the topic id of webxdcs needed for peer channels. webxdc_topic: Option, + + /// This field is used when this is either a pre-message or a Post-Message. + pre_message_mode: Option, } /// Result of rendering a message, ready to be submitted to a send job. @@ -500,6 +513,7 @@ impl MimeFactory { sync_ids_to_delete: None, attach_selfavatar, webxdc_topic, + pre_message_mode: None, }; Ok(factory) } @@ -548,6 +562,7 @@ impl MimeFactory { sync_ids_to_delete: None, attach_selfavatar: false, webxdc_topic: None, + pre_message_mode: None, }; Ok(res) @@ -779,7 +794,10 @@ impl MimeFactory { headers.push(("Date", mail_builder::headers::raw::Raw::new(date).into())); let rfc724_mid = match &self.loaded { - Loaded::Message { msg, .. } => msg.rfc724_mid.clone(), + Loaded::Message { msg, .. } => match &self.pre_message_mode { + Some(PreMessageMode::PreMessage { .. }) => create_outgoing_rfc724_mid(), + _ => msg.rfc724_mid.clone(), + }, Loaded::Mdn { .. } => create_outgoing_rfc724_mid(), }; headers.push(( @@ -893,7 +911,7 @@ impl MimeFactory { )); } - let is_encrypted = self.encryption_pubkeys.is_some(); + let is_encrypted = self.will_be_encrypted(); // Add ephemeral timer for non-MDN messages. // For MDNs it does not matter because they are not visible @@ -978,6 +996,22 @@ impl MimeFactory { "MIME-Version", mail_builder::headers::raw::Raw::new("1.0").into(), )); + + if self.pre_message_mode == Some(PreMessageMode::PostMessage) { + unprotected_headers.push(( + "Chat-Is-Post-Message", + mail_builder::headers::raw::Raw::new("1").into(), + )); + } else if let Some(PreMessageMode::PreMessage { + post_msg_rfc724_mid, + }) = self.pre_message_mode.clone() + { + protected_headers.push(( + "Chat-Post-Message-ID", + mail_builder::headers::message_id::MessageId::new(post_msg_rfc724_mid).into(), + )); + } + for header @ (original_header_name, _header_value) in &headers { let header_name = original_header_name.to_lowercase(); if header_name == "message-id" { @@ -1119,6 +1153,10 @@ impl MimeFactory { for (addr, key) in &encryption_pubkeys { let fingerprint = key.dc_fingerprint().hex(); let cmd = msg.param.get_cmd(); + if self.pre_message_mode == Some(PreMessageMode::PostMessage) { + continue; + } + let should_do_gossip = cmd == SystemMessage::MemberAddedToGroup || cmd == SystemMessage::SecurejoinMessage || multiple_recipients && { @@ -1831,19 +1869,23 @@ impl MimeFactory { let footer = if is_reaction { "" } else { &self.selfstatus }; - let message_text = format!( - "{}{}{}{}{}{}", - fwdhint.unwrap_or_default(), - quoted_text.unwrap_or_default(), - escape_message_footer_marks(final_text), - if !final_text.is_empty() && !footer.is_empty() { - "\r\n\r\n" - } else { - "" - }, - if !footer.is_empty() { "-- \r\n" } else { "" }, - footer - ); + let message_text = if self.pre_message_mode == Some(PreMessageMode::PostMessage) { + "".to_string() + } else { + format!( + "{}{}{}{}{}{}", + fwdhint.unwrap_or_default(), + quoted_text.unwrap_or_default(), + escape_message_footer_marks(final_text), + if !final_text.is_empty() && !footer.is_empty() { + "\r\n\r\n" + } else { + "" + }, + if !footer.is_empty() { "-- \r\n" } else { "" }, + footer + ) + }; let mut main_part = MimePart::new("text/plain", message_text); if is_reaction { @@ -1875,8 +1917,19 @@ impl MimeFactory { // add attachment part if msg.viewtype.has_file() { - let file_part = build_body_file(context, &msg).await?; - parts.push(file_part); + if let Some(PreMessageMode::PreMessage { .. }) = self.pre_message_mode { + let Some(metadata) = PreMsgMetadata::from_msg(context, &msg).await? else { + bail!("Failed to generate metadata for pre-message") + }; + + headers.push(( + HeaderDef::ChatPostMessageMetadata.into(), + mail_builder::headers::raw::Raw::new(metadata.to_header_value()?).into(), + )); + } else { + let file_part = build_body_file(context, &msg).await?; + parts.push(file_part); + } } if let Some(msg_kml_part) = self.get_message_kml_part() { @@ -1921,6 +1974,8 @@ impl MimeFactory { } } + self.attach_selfavatar = + self.attach_selfavatar && self.pre_message_mode != Some(PreMessageMode::PostMessage); if self.attach_selfavatar { match context.get_config(Config::Selfavatar).await? { Some(path) => match build_avatar_file(context, &path).await { @@ -1990,6 +2045,20 @@ impl MimeFactory { Ok(message) } + + pub fn will_be_encrypted(&self) -> bool { + self.encryption_pubkeys.is_some() + } + + pub fn set_as_post_message(&mut self) { + self.pre_message_mode = Some(PreMessageMode::PostMessage); + } + + pub fn set_as_pre_message_for(&mut self, post_message: &RenderedEmail) { + self.pre_message_mode = Some(PreMessageMode::PreMessage { + post_msg_rfc724_mid: post_message.rfc724_mid.clone(), + }); + } } fn hidden_recipients() -> Address<'static> { diff --git a/src/mimefactory/mimefactory_tests.rs b/src/mimefactory/mimefactory_tests.rs index d29fe54645..bd79d92c5f 100644 --- a/src/mimefactory/mimefactory_tests.rs +++ b/src/mimefactory/mimefactory_tests.rs @@ -559,7 +559,7 @@ async fn test_render_reply() { "1.0" ); - let _mime_msg = MimeMessage::from_bytes(context, rendered_msg.message.as_bytes(), None) + let _mime_msg = MimeMessage::from_bytes(context, rendered_msg.message.as_bytes()) .await .unwrap(); } @@ -757,7 +757,7 @@ async fn test_protected_headers_directive() -> Result<()> { assert!(msg.get_showpadlock()); assert!(sent.payload.contains("\r\nSubject: [...]\r\n")); - let mime = MimeMessage::from_bytes(&alice, sent.payload.as_bytes(), None).await?; + let mime = MimeMessage::from_bytes(&alice, sent.payload.as_bytes()).await?; let mut payload = str::from_utf8(&mime.decoded_data)?.splitn(2, "\r\n\r\n"); let part = payload.next().unwrap(); assert_eq!( @@ -781,7 +781,7 @@ async fn test_hp_outer_headers() -> Result<()> { .await?; chat::send_text_msg(t, chat_id, "hi!".to_string()).await?; let sent_msg = t.pop_sent_msg().await; - let msg = MimeMessage::from_bytes(t, sent_msg.payload.as_bytes(), None).await?; + let msg = MimeMessage::from_bytes(t, sent_msg.payload.as_bytes()).await?; assert_eq!(msg.header_exists(HeaderDef::HpOuter), std_hp_composing); for hdr in ["Date", "From", "Message-ID"] { assert_eq!( @@ -811,7 +811,7 @@ async fn test_dont_remove_self() -> Result<()> { .await; println!("{}", sent.payload); - let mime_message = MimeMessage::from_bytes(alice, sent.payload.as_bytes(), None) + let mime_message = MimeMessage::from_bytes(alice, sent.payload.as_bytes()) .await .unwrap(); assert!(!mime_message.header_exists(HeaderDef::ChatGroupPastMembers)); diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 69bb198554..1c7080944f 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -23,6 +23,7 @@ use crate::contact::ContactId; use crate::context::Context; use crate::decrypt::{try_decrypt, validate_detached_signature}; use crate::dehtml::dehtml; +use crate::download::pre_msg_metadata::PreMsgMetadata; use crate::events::EventType; use crate::headerdef::{HeaderDef, HeaderDefMap}; use crate::key::{self, DcKey, Fingerprint, SignedPublicKey, load_self_secret_keyring}; @@ -147,6 +148,23 @@ pub(crate) struct MimeMessage { /// Sender timestamp in secs since epoch. Allowed to be in the future due to unsynchronized /// clocks, but not too much. pub(crate) timestamp_sent: i64, + + pub(crate) pre_message: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum PreMessageMode { + /// This is Post-Message + /// it replaces it's Pre-Message attachment if it exists already, + /// and if the Pre-Message does not exist it is treated as normal message + PostMessage, + /// This is a Pre-Message, + /// it adds a message preview for a Post-Message + /// and it is ignored if the Post-Message was downloaded already + PreMessage { + post_msg_rfc724_mid: String, + metadata: Option, + }, } #[derive(Debug, PartialEq)] @@ -240,12 +258,9 @@ const MIME_AC_SETUP_FILE: &str = "application/autocrypt-setup"; impl MimeMessage { /// Parse a mime message. /// - /// If `partial` is set, it contains the full message size in bytes. - pub(crate) async fn from_bytes( - context: &Context, - body: &[u8], - partial: Option, - ) -> Result { + /// This method has some side-effects, + /// such as saving blobs and saving found public keys to the database. + pub(crate) async fn from_bytes(context: &Context, body: &[u8]) -> Result { let mail = mailparse::parse_mail(body)?; let timestamp_rcvd = smeared_time(context); @@ -302,7 +317,7 @@ impl MimeMessage { ); (part, part.ctype.mimetype.parse::()?) } else { - // If it's a partially fetched message, there are no subparts. + // Not a valid signed message, handle it as plaintext. (&mail, mimetype) } } else { @@ -352,6 +367,16 @@ impl MimeMessage { let mut aheader_values = mail.headers.get_all_values(HeaderDef::Autocrypt.into()); + let mut pre_message = if mail + .headers + .get_header_value(HeaderDef::ChatIsPostMessage) + .is_some() + { + Some(PreMessageMode::PostMessage) + } else { + None + }; + let mail_raw; // Memory location for a possible decrypted message. let decrypted_msg; // Decrypted signed OpenPGP message. let secrets: Vec = context @@ -580,6 +605,36 @@ impl MimeMessage { signatures.clear(); } + if let (Ok(mail), true) = (mail, is_encrypted) + && let Some(post_msg_rfc724_mid) = + mail.headers.get_header_value(HeaderDef::ChatPostMessageId) + { + let post_msg_rfc724_mid = parse_message_id(&post_msg_rfc724_mid)?; + let metadata = if let Some(value) = mail + .headers + .get_header_value(HeaderDef::ChatPostMessageMetadata) + { + match PreMsgMetadata::try_from_header_value(&value) { + Ok(metadata) => Some(metadata), + Err(error) => { + error!( + context, + "failed to parse metadata header in pre-message: {error:#?}" + ); + None + } + } + } else { + warn!(context, "expected pre-message to have metadata header"); + None + }; + + pre_message = Some(PreMessageMode::PreMessage { + post_msg_rfc724_mid, + metadata, + }); + } + let mut parser = MimeMessage { parts: Vec::new(), headers, @@ -615,33 +670,27 @@ impl MimeMessage { is_bot: None, timestamp_rcvd, timestamp_sent, + pre_message, }; - match partial { - Some(org_bytes) => { - parser - .create_stub_from_partial_download(context, org_bytes) - .await?; + match mail { + Ok(mail) => { + parser.parse_mime_recursive(context, mail, false).await?; + } + Err(err) => { + let txt = "[This message cannot be decrypted.\n\n• It might already help to simply reply to this message and ask the sender to send the message again.\n\n• If you just re-installed Delta Chat then it is best if you re-setup Delta Chat now and choose \"Add as second device\" or import a backup.]"; + + let part = Part { + typ: Viewtype::Text, + msg_raw: Some(txt.to_string()), + msg: txt.to_string(), + // Don't change the error prefix for now, + // receive_imf.rs:lookup_chat_by_reply() checks it. + error: Some(format!("Decrypting failed: {err:#}")), + ..Default::default() + }; + parser.do_add_single_part(part); } - None => match mail { - Ok(mail) => { - parser.parse_mime_recursive(context, mail, false).await?; - } - Err(err) => { - let txt = "[This message cannot be decrypted.\n\n• It might already help to simply reply to this message and ask the sender to send the message again.\n\n• If you just re-installed Delta Chat then it is best if you re-setup Delta Chat now and choose \"Add as second device\" or import a backup.]"; - - let part = Part { - typ: Viewtype::Text, - msg_raw: Some(txt.to_string()), - msg: txt.to_string(), - // Don't change the error prefix for now, - // receive_imf.rs:lookup_chat_by_reply() checks it. - error: Some(format!("Decrypting failed: {err:#}")), - ..Default::default() - }; - parser.do_add_single_part(part); - } - }, }; let is_location_only = parser.location_kml.is_some() && parser.parts.is_empty(); diff --git a/src/mimeparser/mimeparser_tests.rs b/src/mimeparser/mimeparser_tests.rs index 6d198075b9..42edbba7a8 100644 --- a/src/mimeparser/mimeparser_tests.rs +++ b/src/mimeparser/mimeparser_tests.rs @@ -25,58 +25,54 @@ impl AvatarAction { async fn test_mimeparser_fromheader() { let ctx = TestContext::new_alice().await; - let mimemsg = MimeMessage::from_bytes(&ctx, b"From: g@c.de\n\nhi", None) + let mimemsg = MimeMessage::from_bytes(&ctx, b"From: g@c.de\n\nhi") .await .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, None); - let mimemsg = MimeMessage::from_bytes(&ctx, b"From: g@c.de \n\nhi", None) + let mimemsg = MimeMessage::from_bytes(&ctx, b"From: g@c.de \n\nhi") .await .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, None); - let mimemsg = MimeMessage::from_bytes(&ctx, b"From: \n\nhi", None) + let mimemsg = MimeMessage::from_bytes(&ctx, b"From: \n\nhi") .await .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, None); - let mimemsg = MimeMessage::from_bytes(&ctx, b"From: Goetz C \n\nhi", None) + let mimemsg = MimeMessage::from_bytes(&ctx, b"From: Goetz C \n\nhi") .await .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, Some("Goetz C".to_string())); - let mimemsg = MimeMessage::from_bytes(&ctx, b"From: \"Goetz C\" \n\nhi", None) + let mimemsg = MimeMessage::from_bytes(&ctx, b"From: \"Goetz C\" \n\nhi") .await .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, Some("Goetz C".to_string())); - let mimemsg = - MimeMessage::from_bytes(&ctx, b"From: =?utf-8?q?G=C3=B6tz?= C \n\nhi", None) - .await - .unwrap(); + let mimemsg = MimeMessage::from_bytes(&ctx, b"From: =?utf-8?q?G=C3=B6tz?= C \n\nhi") + .await + .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, Some("Götz C".to_string())); // although RFC 2047 says, encoded-words shall not appear inside quoted-string, // this combination is used in the wild eg. by MailMate - let mimemsg = MimeMessage::from_bytes( - &ctx, - b"From: \"=?utf-8?q?G=C3=B6tz?= C\" \n\nhi", - None, - ) - .await - .unwrap(); + let mimemsg = + MimeMessage::from_bytes(&ctx, b"From: \"=?utf-8?q?G=C3=B6tz?= C\" \n\nhi") + .await + .unwrap(); let contact = mimemsg.from; assert_eq!(contact.addr, "g@c.de"); assert_eq!(contact.display_name, Some("Götz C".to_string())); @@ -86,7 +82,7 @@ async fn test_mimeparser_fromheader() { async fn test_mimeparser_crash() { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/issue_523.txt"); - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); @@ -98,7 +94,7 @@ async fn test_mimeparser_crash() { async fn test_get_rfc724_mid_exists() { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/mail_with_message_id.txt"); - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); @@ -112,7 +108,7 @@ async fn test_get_rfc724_mid_exists() { async fn test_get_rfc724_mid_not_exists() { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/issue_523.txt"); - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(mimeparser.get_rfc724_mid(), None); @@ -324,7 +320,7 @@ async fn test_mailparse_0_16_0_panic() { // There should be an error, but no panic. assert!( - MimeMessage::from_bytes(&context.ctx, &raw[..], None) + MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .is_err() ); @@ -341,7 +337,7 @@ async fn test_parse_first_addr() { test1\n\ "; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None).await; + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).await; assert!(mimeparser.is_err()); } @@ -356,7 +352,7 @@ async fn test_get_parent_timestamp() { \n\ Some reply\n\ "; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -402,7 +398,7 @@ async fn test_mimeparser_with_context() { --==break==--\n\ \n"; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); @@ -438,26 +434,26 @@ async fn test_mimeparser_with_avatars() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/mail_attach_txt.eml"); - let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap(); assert_eq!(mimeparser.user_avatar, None); assert_eq!(mimeparser.group_avatar, None); let raw = include_bytes!("../../test-data/message/mail_with_user_avatar.eml"); - let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap(); assert_eq!(mimeparser.parts.len(), 1); assert_eq!(mimeparser.parts[0].typ, Viewtype::Text); assert!(mimeparser.user_avatar.unwrap().is_change()); assert_eq!(mimeparser.group_avatar, None); let raw = include_bytes!("../../test-data/message/mail_with_user_avatar_deleted.eml"); - let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap(); assert_eq!(mimeparser.parts.len(), 1); assert_eq!(mimeparser.parts[0].typ, Viewtype::Text); assert_eq!(mimeparser.user_avatar, Some(AvatarAction::Delete)); assert_eq!(mimeparser.group_avatar, None); let raw = include_bytes!("../../test-data/message/mail_with_user_and_group_avatars.eml"); - let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap(); assert_eq!(mimeparser.parts.len(), 1); assert_eq!(mimeparser.parts[0].typ, Viewtype::Text); assert!(mimeparser.user_avatar.unwrap().is_change()); @@ -467,9 +463,7 @@ async fn test_mimeparser_with_avatars() { let raw = include_bytes!("../../test-data/message/mail_with_user_and_group_avatars.eml"); let raw = String::from_utf8_lossy(raw).to_string(); let raw = raw.replace("Chat-User-Avatar:", "Xhat-Xser-Xvatar:"); - let mimeparser = MimeMessage::from_bytes(&t, raw.as_bytes(), None) - .await - .unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, raw.as_bytes()).await.unwrap(); assert_eq!(mimeparser.parts.len(), 1); assert_eq!(mimeparser.parts[0].typ, Viewtype::Image); assert_eq!(mimeparser.user_avatar, None); @@ -485,7 +479,7 @@ async fn test_mimeparser_with_videochat() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/videochat_invitation.eml"); - let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap(); assert_eq!(mimeparser.parts.len(), 1); assert_eq!(mimeparser.parts[0].typ, Viewtype::Text); assert_eq!(mimeparser.parts[0].param.get(Param::WebrtcRoom), None); @@ -528,7 +522,7 @@ Content-Disposition: attachment; filename=\"message.kml\"\n\ --==break==--\n\ ;"; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -578,7 +572,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\ --kJBbU58X1xeWNHgBtTbMk80M5qnV4N--\n\ "; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -659,7 +653,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\ --outer--\n\ "; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -706,7 +700,7 @@ Additional-Message-IDs: \n\ --kJBbU58X1xeWNHgBtTbMk80M5qnV4N--\n\ "; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -753,7 +747,7 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg== ------=_Part_25_46172632.1581201680436-- "#; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -797,7 +791,7 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg== ------=_Part_25_46172632.1581201680436-- "#; - let message = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap(); + let message = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap(); assert_eq!(message.parts.len(), 1); assert_eq!(message.parts[0].typ, Viewtype::File); @@ -839,7 +833,7 @@ iVBORw0KGgoAAAANSUhEUgAAACAAAAAeCAAAAABoYUP1AAAAAXNSR0IArs4c6QAAAo1JREFUKJFdkdFu ----11019878869865180-- "#; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(message.get_subject(), Some("example".to_string())); @@ -903,7 +897,7 @@ iVBORw0KGgoAAAANSUhEUgAAACAAAAAeCAAAAABoYUP1AAAAAXNSR0IArs4c6QAAAo1JREFUKJFdkdFu --------------779C1631600DF3DB8C02E53A--"#; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(message.get_subject(), Some("Test subject".to_string())); @@ -966,7 +960,7 @@ iVBORw0KGgoAAAANSUhEUgAAACAAAAAeCAAAAABoYUP1AAAAAXNSR0IArs4c6QAAAo1JREFUKJFdkdFu ------=_NextPart_000_0003_01D622B3.CA753E60-- "#; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -1064,7 +1058,7 @@ From: alice Reply "##; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -1096,7 +1090,7 @@ From: alice > Just a quote. "##; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -1130,7 +1124,7 @@ On 2020-10-25, Bob wrote: > A quote. "##; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(message.get_subject(), Some("Re: top posting".to_string())); @@ -1148,7 +1142,7 @@ On 2020-10-25, Bob wrote: async fn test_attachment_quote() { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/quote_attach.eml"); - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); @@ -1166,7 +1160,7 @@ async fn test_attachment_quote() { async fn test_quote_div() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/gmx-quote.eml"); - let mimeparser = MimeMessage::from_bytes(&t, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, raw).await.unwrap(); assert_eq!(mimeparser.parts[0].msg, "YIPPEEEEEE\n\nMulti-line"); assert_eq!(mimeparser.parts[0].param.get(Param::Quote).unwrap(), "Now?"); } @@ -1176,7 +1170,7 @@ async fn test_allinkl_blockquote() { // all-inkl.com puts quotes into `
`. let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/allinkl-quote.eml"); - let mimeparser = MimeMessage::from_bytes(&t, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t, raw).await.unwrap(); assert!(mimeparser.parts[0].msg.starts_with("It's 1.0.")); assert_eq!( mimeparser.parts[0].param.get(Param::Quote).unwrap(), @@ -1217,7 +1211,7 @@ async fn test_add_subj_to_multimedia_msg() { async fn test_mime_modified_plain() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/text_plain_unspecified.eml"); - let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap(); assert!(!mimeparser.is_mime_modified); assert_eq!( mimeparser.parts[0].msg, @@ -1229,7 +1223,7 @@ async fn test_mime_modified_plain() { async fn test_mime_modified_alt_plain_html() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/text_alt_plain_html.eml"); - let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap(); assert!(mimeparser.is_mime_modified); assert_eq!( mimeparser.parts[0].msg, @@ -1241,7 +1235,7 @@ async fn test_mime_modified_alt_plain_html() { async fn test_mime_modified_alt_plain() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/text_alt_plain.eml"); - let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap(); assert!(!mimeparser.is_mime_modified); assert_eq!( mimeparser.parts[0].msg, @@ -1256,7 +1250,7 @@ async fn test_mime_modified_alt_plain() { async fn test_mime_modified_alt_html() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/text_alt_html.eml"); - let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap(); assert!(mimeparser.is_mime_modified); assert_eq!( mimeparser.parts[0].msg, @@ -1268,7 +1262,7 @@ async fn test_mime_modified_alt_html() { async fn test_mime_modified_html() { let t = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/text_html.eml"); - let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap(); + let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap(); assert!(mimeparser.is_mime_modified); assert_eq!( mimeparser.parts[0].msg, @@ -1288,7 +1282,7 @@ async fn test_mime_modified_large_plain() -> Result<()> { assert!(long_txt.len() > DC_DESIRED_TEXT_LEN); { - let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref(), None).await?; + let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref()).await?; assert!(mimemsg.is_mime_modified); assert!( mimemsg.parts[0].msg.matches("just repeated").count() @@ -1321,7 +1315,7 @@ async fn test_mime_modified_large_plain() -> Result<()> { t.set_config(Config::Bot, Some("1")).await?; { - let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref(), None).await?; + let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref()).await?; assert!(!mimemsg.is_mime_modified); assert_eq!( format!("{}\n", mimemsg.parts[0].msg), @@ -1368,7 +1362,7 @@ async fn test_x_microsoft_original_message_id() { MIME-Version: 1.0\n\ \n\ Does it work with outlook now?\n\ - ", None) + ") .await .unwrap(); assert_eq!( @@ -1418,7 +1412,7 @@ async fn test_extra_imf_headers() -> Result<()> { "Message-ID:", "Chat-Forty-Two: 42\r\nForty-Two: 42\r\nMessage-ID:", ); - let msg = MimeMessage::from_bytes(t, payload.as_bytes(), None).await?; + let msg = MimeMessage::from_bytes(t, payload.as_bytes()).await?; assert!(msg.headers.contains_key("chat-version")); assert!(!msg.headers.contains_key("chat-forty-two")); assert_ne!(msg.headers.contains_key("forty-two"), std_hp_composing); @@ -1582,7 +1576,7 @@ async fn test_ms_exchange_mdn() -> Result<()> { // 1. Test mimeparser directly let mdn = include_bytes!("../../test-data/message/ms_exchange_report_disposition_notification.eml"); - let mimeparser = MimeMessage::from_bytes(&t.ctx, mdn, None).await?; + let mimeparser = MimeMessage::from_bytes(&t.ctx, mdn).await?; assert_eq!(mimeparser.mdn_reports.len(), 1); assert_eq!( mimeparser.mdn_reports[0].original_message_id.as_deref(), @@ -1608,7 +1602,6 @@ async fn test_receive_eml() -> Result<()> { let mime_message = MimeMessage::from_bytes( &alice, include_bytes!("../../test-data/message/attached-eml.eml"), - None, ) .await?; @@ -1651,7 +1644,6 @@ Content-Disposition: reaction\n\ \n\ \u{1F44D}" .as_bytes(), - None, ) .await?; @@ -1673,7 +1665,7 @@ async fn test_jpeg_as_application_octet_stream() -> Result<()> { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/jpeg-as-application-octet-stream.eml"); - let msg = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let msg = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(msg.parts.len(), 1); @@ -1691,7 +1683,7 @@ async fn test_schleuder() -> Result<()> { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/schleuder.eml"); - let msg = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let msg = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(msg.parts.len(), 2); @@ -1711,7 +1703,7 @@ async fn test_tlsrpt() -> Result<()> { let context = TestContext::new_alice().await; let raw = include_bytes!("../../test-data/message/tlsrpt.eml"); - let msg = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let msg = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(msg.parts.len(), 1); @@ -1744,7 +1736,6 @@ async fn test_time_in_future() -> Result<()> { Content-Type: text/plain; charset=utf-8\n\ \n\ Hi", - None, ) .await?; @@ -1806,7 +1797,7 @@ Content-Type: text/plain; charset=utf-8 /help "#; - let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(message.get_subject(), Some("Some subject".to_string())); @@ -1847,7 +1838,7 @@ async fn test_take_last_header() { Hello\n\ "; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None) + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!( @@ -1900,9 +1891,7 @@ It DOES end with a linebreak.\r \r This is the epilogue. It is also to be ignored."; - let mimeparser = MimeMessage::from_bytes(&context, &raw[..], None) - .await - .unwrap(); + let mimeparser = MimeMessage::from_bytes(&context, &raw[..]).await.unwrap(); assert_eq!(mimeparser.parts.len(), 2); @@ -1948,7 +1937,7 @@ Message with a correct Message-ID hidden header --luTiGu6GBoVLCvTkzVtmZmwsmhkNMw-- "#; - let message = MimeMessage::from_bytes(t, &raw[..], None).await.unwrap(); + let message = MimeMessage::from_bytes(t, &raw[..]).await.unwrap(); assert_eq!(message.get_rfc724_mid().unwrap(), "foo@example.org"); } @@ -2126,9 +2115,7 @@ Third alternative. --boundary-- "#; - let message = MimeMessage::from_bytes(context, &raw[..], None) - .await - .unwrap(); + let message = MimeMessage::from_bytes(context, &raw[..]).await.unwrap(); assert_eq!(message.parts.len(), 1); assert_eq!(message.parts[0].typ, Viewtype::Text); assert_eq!(message.parts[0].msg, "Third alternative."); diff --git a/src/param.rs b/src/param.rs index 0640b551ef..01f23c7271 100644 --- a/src/param.rs +++ b/src/param.rs @@ -251,6 +251,13 @@ pub enum Param { /// For info messages: Contact ID in added or removed to a group. ContactAddedRemoved = b'5', + + /// For (pre-)Message: ViewType of the Post-Message, + /// because pre message is always `Viewtype::Text`. + PostMessageViewtype = b'8', + + /// For (pre-)Message: File byte size of Post-Message attachment + PostMessageFileBytes = b'9', } /// An object for handling key=value parameter lists. @@ -441,6 +448,15 @@ impl Params { } self } + + /// Merge in parameters from other Params struct, + /// overwriting the keys that are in both + /// with the values from the new Params struct. + pub fn merge_in_from_params(&mut self, new_params: Self) -> &mut Self { + let mut new_params = new_params; + self.inner.append(&mut new_params.inner); + self + } } #[cfg(test)] @@ -503,4 +519,18 @@ mod tests { assert_eq!(p.get(Param::Height), Some("14")); Ok(()) } + + #[test] + fn test_merge() -> Result<()> { + let mut p = Params::from_str("w=12\na=5\nh=14")?; + let p2 = Params::from_str("L=1\nh=17")?; + assert_eq!(p.len(), 3); + p.merge_in_from_params(p2); + assert_eq!(p.len(), 4); + assert_eq!(p.get(Param::Width), Some("12")); + assert_eq!(p.get(Param::Height), Some("17")); + assert_eq!(p.get(Param::Forwarded), Some("5")); + assert_eq!(p.get(Param::IsEdited), Some("1")); + Ok(()) + } } diff --git a/src/reaction.rs b/src/reaction.rs index 0a00f7aeb1..82a75ed546 100644 --- a/src/reaction.rs +++ b/src/reaction.rs @@ -392,9 +392,8 @@ mod tests { use crate::chatlist::Chatlist; use crate::config::Config; use crate::contact::{Contact, Origin}; - use crate::download::DownloadState; use crate::message::{MessageState, Viewtype, delete_msgs}; - use crate::receive_imf::{receive_imf, receive_imf_from_inbox}; + use crate::receive_imf::receive_imf; use crate::sql::housekeeping; use crate::test_utils::E2EE_INFO_MSGS; use crate::test_utils::TestContext; @@ -924,73 +923,6 @@ Content-Disposition: reaction\n\ Ok(()) } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_partial_download_and_reaction() -> Result<()> { - let alice = TestContext::new_alice().await; - let bob = TestContext::new_bob().await; - - alice - .create_chat_with_contact("Bob", "bob@example.net") - .await; - - let msg_header = "From: Bob \n\ - To: Alice \n\ - Chat-Version: 1.0\n\ - Subject: subject\n\ - Message-ID: \n\ - Date: Sun, 14 Nov 2021 00:10:00 +0000\ - Content-Type: text/plain"; - let msg_full = format!("{msg_header}\n\n100k text..."); - - // Alice downloads message from Bob partially. - let alice_received_message = receive_imf_from_inbox( - &alice, - "first@example.org", - msg_header.as_bytes(), - false, - Some(100000), - ) - .await? - .unwrap(); - let alice_msg_id = *alice_received_message.msg_ids.first().unwrap(); - - // Bob downloads own message on the other device. - let bob_received_message = receive_imf(&bob, msg_full.as_bytes(), false) - .await? - .unwrap(); - let bob_msg_id = *bob_received_message.msg_ids.first().unwrap(); - - // Bob reacts to own message. - send_reaction(&bob, bob_msg_id, "👍").await.unwrap(); - let bob_reaction_msg = bob.pop_sent_msg().await; - - // Alice receives a reaction. - alice.recv_msg_hidden(&bob_reaction_msg).await; - - let reactions = get_msg_reactions(&alice, alice_msg_id).await?; - assert_eq!(reactions.to_string(), "👍1"); - let msg = Message::load_from_db(&alice, alice_msg_id).await?; - assert_eq!(msg.download_state(), DownloadState::Available); - - // Alice downloads full message. - receive_imf_from_inbox( - &alice, - "first@example.org", - msg_full.as_bytes(), - false, - None, - ) - .await?; - - // Check that reaction is still on the message after full download. - let msg = Message::load_from_db(&alice, alice_msg_id).await?; - assert_eq!(msg.download_state(), DownloadState::Done); - let reactions = get_msg_reactions(&alice, alice_msg_id).await?; - assert_eq!(reactions.to_string(), "👍1"); - - Ok(()) - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_send_reaction_multidevice() -> Result<()> { let mut tcm = TestContextManager::new(); diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 0013443fc3..f6257ca113 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -20,16 +20,14 @@ use crate::constants::{self, Blocked, Chattype, DC_CHAT_ID_TRASH, EDITED_PREFIX, use crate::contact::{self, Contact, ContactId, Origin, mark_contact_id_as_verified}; use crate::context::Context; use crate::debug_logging::maybe_set_logging_xdc_inner; -use crate::download::DownloadState; +use crate::download::{DownloadState, premessage_is_downloaded_for}; use crate::ephemeral::{Timer as EphemeralTimer, stock_ephemeral_timer_changed}; use crate::events::EventType; use crate::headerdef::{HeaderDef, HeaderDefMap}; use crate::imap::{GENERATED_PREFIX, markseen_on_imap_table}; use crate::key::{DcKey, Fingerprint}; use crate::key::{self_fingerprint, self_fingerprint_opt}; -use crate::log::LogExt; -use crate::log::warn; -use crate::logged_debug_assert; +use crate::log::{LogExt as _, warn}; use crate::message::{ self, Message, MessageState, MessengerMessage, MsgId, Viewtype, rfc724_mid_exists, }; @@ -47,6 +45,7 @@ use crate::tools::{ self, buf_compress, normalize_text, remove_subject_prefix, validate_broadcast_secret, }; use crate::{chatlist_events, ensure_and_debug_assert, ensure_and_debug_assert_eq, location}; +use crate::{logged_debug_assert, mimeparser}; /// This is the struct that is returned after receiving one email (aka MIME message). /// @@ -157,24 +156,7 @@ pub async fn receive_imf( let mail = mailparse::parse_mail(imf_raw).context("can't parse mail")?; let rfc724_mid = crate::imap::prefetch_get_message_id(&mail.headers) .unwrap_or_else(crate::imap::create_message_id); - if let Some(download_limit) = context.download_limit().await? { - let download_limit: usize = download_limit.try_into()?; - if imf_raw.len() > download_limit { - let head = std::str::from_utf8(imf_raw)? - .split("\r\n\r\n") - .next() - .context("No empty line in the message")?; - return receive_imf_from_inbox( - context, - &rfc724_mid, - head.as_bytes(), - seen, - Some(imf_raw.len().try_into()?), - ) - .await; - } - } - receive_imf_from_inbox(context, &rfc724_mid, imf_raw, seen, None).await + receive_imf_from_inbox(context, &rfc724_mid, imf_raw, seen).await } /// Emulates reception of a message from "INBOX". @@ -186,9 +168,8 @@ pub(crate) async fn receive_imf_from_inbox( rfc724_mid: &str, imf_raw: &[u8], seen: bool, - is_partial_download: Option, ) -> Result> { - receive_imf_inner(context, rfc724_mid, imf_raw, seen, is_partial_download).await + receive_imf_inner(context, rfc724_mid, imf_raw, seen).await } /// Inserts a tombstone into `msgs` table @@ -211,7 +192,6 @@ async fn get_to_and_past_contact_ids( context: &Context, mime_parser: &MimeMessage, chat_assignment: &ChatAssignment, - is_partial_download: Option, parent_message: &Option, incoming_origin: Origin, ) -> Result<(Vec>, Vec>)> { @@ -254,7 +234,7 @@ async fn get_to_and_past_contact_ids( ChatAssignment::ExistingChat { chat_id, .. } => Some(*chat_id), ChatAssignment::MailingListOrBroadcast => None, ChatAssignment::OneOneChat => { - if is_partial_download.is_none() && !mime_parser.incoming { + if !mime_parser.incoming { parent_message.as_ref().map(|m| m.chat_id) } else { None @@ -484,15 +464,17 @@ async fn get_to_and_past_contact_ids( /// downloaded again, sets `chat_id=DC_CHAT_ID_TRASH` and returns `Ok(Some(…))`. /// If the message is so wrong that we didn't even create a database entry, /// returns `Ok(None)`. -/// -/// If `is_partial_download` is set, it contains the full message size in bytes. pub(crate) async fn receive_imf_inner( context: &Context, rfc724_mid: &str, imf_raw: &[u8], seen: bool, - is_partial_download: Option, ) -> Result> { + ensure!( + !context + .get_config_bool(Config::SimulateReceiveImfError) + .await? + ); if std::env::var(crate::DCC_MIME_DEBUG).is_ok() { info!( context, @@ -500,16 +482,8 @@ pub(crate) async fn receive_imf_inner( String::from_utf8_lossy(imf_raw), ); } - if is_partial_download.is_none() { - ensure!( - !context - .get_config_bool(Config::FailOnReceivingFullMsg) - .await? - ); - } - let mut mime_parser = match MimeMessage::from_bytes(context, imf_raw, is_partial_download).await - { + let mut mime_parser = match MimeMessage::from_bytes(context, imf_raw).await { Err(err) => { warn!(context, "receive_imf: can't parse MIME: {err:#}."); if rfc724_mid.starts_with(GENERATED_PREFIX) { @@ -542,7 +516,15 @@ pub(crate) async fn receive_imf_inner( // check, if the mail is already in our database. // make sure, this check is done eg. before securejoin-processing. let (replace_msg_id, replace_chat_id); - if let Some(old_msg_id) = message::rfc724_mid_exists(context, rfc724_mid).await? { + if mime_parser.pre_message == Some(mimeparser::PreMessageMode::PostMessage) { + // Post-Message just replace the attachment and mofified Params, not the whole message + // This is done in the `handle_post_message` method. + replace_msg_id = None; + replace_chat_id = None; + } else if let Some(old_msg_id) = message::rfc724_mid_exists(context, rfc724_mid).await? { + // This code handles the download of old partial download stub messages + // It will be removed after a transitioning period, + // after we have released a few versions with pre-messages replace_msg_id = Some(old_msg_id); replace_chat_id = if let Some(msg) = Message::load_from_db_optional(context, old_msg_id) .await? @@ -615,11 +597,7 @@ pub(crate) async fn receive_imf_inner( &mime_parser.from, fingerprint, prevent_rename, - is_partial_download.is_some() - && mime_parser - .get_header(HeaderDef::ContentType) - .unwrap_or_default() - .starts_with("multipart/encrypted"), + false, ) .await? { @@ -651,22 +629,14 @@ pub(crate) async fn receive_imf_inner( .await? .filter(|p| Some(p.id) != replace_msg_id); - let chat_assignment = decide_chat_assignment( - context, - &mime_parser, - &parent_message, - rfc724_mid, - from_id, - &is_partial_download, - ) - .await?; + let chat_assignment = + decide_chat_assignment(context, &mime_parser, &parent_message, rfc724_mid, from_id).await?; info!(context, "Chat assignment is {chat_assignment:?}."); let (to_ids, past_ids) = get_to_and_past_contact_ids( context, &mime_parser, &chat_assignment, - is_partial_download, &parent_message, incoming_origin, ) @@ -773,7 +743,6 @@ pub(crate) async fn receive_imf_inner( to_id, allow_creation, &mut mime_parser, - is_partial_download, parent_message, ) .await?; @@ -789,7 +758,6 @@ pub(crate) async fn receive_imf_inner( rfc724_mid_orig, from_id, seen, - is_partial_download, replace_msg_id, prevent_rename, chat_id, @@ -957,9 +925,7 @@ pub(crate) async fn receive_imf_inner( let delete_server_after = context.get_config_delete_server_after().await?; if !received_msg.msg_ids.is_empty() { - let target = if received_msg.needs_delete_job - || (delete_server_after == Some(0) && is_partial_download.is_none()) - { + let target = if received_msg.needs_delete_job || delete_server_after == Some(0) { Some(context.get_delete_msgs_target().await?) } else { None @@ -988,7 +954,7 @@ pub(crate) async fn receive_imf_inner( } } - if is_partial_download.is_none() && mime_parser.is_call() { + if mime_parser.is_call() { context .handle_call_msg(insert_msg_id, &mime_parser, from_id) .await?; @@ -1037,7 +1003,7 @@ pub(crate) async fn receive_imf_inner( /// * `find_key_contact_by_addr`: if true, we only know the e-mail address /// of the contact, but not the fingerprint, /// yet want to assign the message to some key-contact. -/// This can happen during prefetch or when the message is partially downloaded. +/// This can happen during prefetch. /// If we get it wrong, the message will be placed into the correct /// chat after downloading. /// @@ -1131,7 +1097,6 @@ async fn decide_chat_assignment( parent_message: &Option, rfc724_mid: &str, from_id: ContactId, - is_partial_download: &Option, ) -> Result { let should_trash = if !mime_parser.mdn_reports.is_empty() { info!(context, "Message is an MDN (TRASH)."); @@ -1147,9 +1112,39 @@ async fn decide_chat_assignment( { info!(context, "Chat edit/delete/iroh/sync message (TRASH)."); true - } else if is_partial_download.is_none() - && (mime_parser.is_system_message == SystemMessage::CallAccepted - || mime_parser.is_system_message == SystemMessage::CallEnded) + } else if let Some(pre_message) = &mime_parser.pre_message { + use crate::mimeparser::PreMessageMode::*; + match pre_message { + PostMessage => { + // if pre message exist, then trash after replacing, otherwise treat as normal message + let pre_message_exists = premessage_is_downloaded_for(context, rfc724_mid).await?; + info!( + context, + "Message is a Post-Message ({}).", + if pre_message_exists { + "pre-message exists already, so trash after replacing attachment" + } else { + "no pre-message -> Keep" + } + ); + pre_message_exists + } + PreMessage { + post_msg_rfc724_mid, + .. + } => { + // if post message already exists, then trash/ignore + let post_msg_exists = + premessage_is_downloaded_for(context, post_msg_rfc724_mid).await?; + info!( + context, + "Message is a Pre-Message (post_msg_exists:{post_msg_exists})." + ); + post_msg_exists + } + } + } else if mime_parser.is_system_message == SystemMessage::CallAccepted + || mime_parser.is_system_message == SystemMessage::CallEnded { info!(context, "Call state changed (TRASH)."); true @@ -1250,7 +1245,7 @@ async fn decide_chat_assignment( } } else if let Some(parent) = &parent_message { if let Some((chat_id, chat_id_blocked)) = - lookup_chat_by_reply(context, mime_parser, parent, is_partial_download).await? + lookup_chat_by_reply(context, mime_parser, parent).await? { // Try to assign to a chat based on In-Reply-To/References. ChatAssignment::ExistingChat { @@ -1272,7 +1267,7 @@ async fn decide_chat_assignment( } } else if let Some(parent) = &parent_message { if let Some((chat_id, chat_id_blocked)) = - lookup_chat_by_reply(context, mime_parser, parent, is_partial_download).await? + lookup_chat_by_reply(context, mime_parser, parent).await? { // Try to assign to a chat based on In-Reply-To/References. ChatAssignment::ExistingChat { @@ -1314,7 +1309,6 @@ async fn do_chat_assignment( to_id: ContactId, allow_creation: bool, mime_parser: &mut MimeMessage, - is_partial_download: Option, parent_message: Option, ) -> Result<(ChatId, Blocked, bool)> { let is_bot = context.get_config_bool(Config::Bot).await?; @@ -1365,7 +1359,6 @@ async fn do_chat_assignment( && let Some((new_chat_id, new_chat_id_blocked)) = create_group( context, mime_parser, - is_partial_download.is_some(), create_blocked, from_id, to_ids, @@ -1414,7 +1407,6 @@ async fn do_chat_assignment( to_ids, allow_creation || test_normal_chat.is_some(), create_blocked, - is_partial_download.is_some(), ) .await? { @@ -1496,7 +1488,6 @@ async fn do_chat_assignment( && let Some((new_chat_id, new_chat_id_blocked)) = create_group( context, mime_parser, - is_partial_download.is_some(), Blocked::Not, from_id, to_ids, @@ -1560,7 +1551,6 @@ async fn do_chat_assignment( to_ids, allow_creation, Blocked::Not, - is_partial_download.is_some(), ) .await? { @@ -1641,7 +1631,6 @@ async fn add_parts( rfc724_mid: &str, from_id: ContactId, seen: bool, - is_partial_download: Option, mut replace_msg_id: Option, prevent_rename: bool, mut chat_id: ChatId, @@ -1713,10 +1702,9 @@ async fn add_parts( .get_rfc724_mid() .unwrap_or(rfc724_mid.to_string()); - // Extract ephemeral timer from the message or use the existing timer if the message is not fully downloaded. - let mut ephemeral_timer = if is_partial_download.is_some() { - chat_id.get_ephemeral_timer(context).await? - } else if let Some(value) = mime_parser.get_header(HeaderDef::EphemeralTimer) { + // Extract ephemeral timer from the message + let mut ephemeral_timer = if let Some(value) = mime_parser.get_header(HeaderDef::EphemeralTimer) + { match value.parse::() { Ok(timer) => timer, Err(err) => { @@ -1919,7 +1907,6 @@ async fn add_parts( let chat_id = if better_msg .as_ref() .is_some_and(|better_msg| better_msg.is_empty()) - && is_partial_download.is_none() { DC_CHAT_ID_TRASH } else { @@ -1968,10 +1955,10 @@ async fn add_parts( } handle_edit_delete(context, mime_parser, from_id).await?; + handle_post_message(context, mime_parser, from_id).await?; - if is_partial_download.is_none() - && (mime_parser.is_system_message == SystemMessage::CallAccepted - || mime_parser.is_system_message == SystemMessage::CallEnded) + if mime_parser.is_system_message == SystemMessage::CallAccepted + || mime_parser.is_system_message == SystemMessage::CallEnded { if let Some(field) = mime_parser.get_header(HeaderDef::InReplyTo) { if let Some(call) = @@ -2052,6 +2039,14 @@ async fn add_parts( } }; + if let Some(mimeparser::PreMessageMode::PreMessage { + metadata: Some(metadata), + .. + }) = &mime_parser.pre_message + { + param.apply_from_pre_msg_metadata(metadata); + }; + // If you change which information is skipped if the message is trashed, // also change `MsgId::trash()` and `delete_expired_messages()` let trash = chat_id.is_trash() || (is_location_kml && part_is_empty && !save_mime_modified); @@ -2095,14 +2090,20 @@ RETURNING id "#)?; let row_id: MsgId = stmt.query_row(params![ replace_msg_id, - rfc724_mid_orig, + if let Some(mimeparser::PreMessageMode::PreMessage {post_msg_rfc724_mid, .. }) = &mime_parser.pre_message { + post_msg_rfc724_mid + } else { rfc724_mid_orig }, if trash { DC_CHAT_ID_TRASH } else { chat_id }, if trash { ContactId::UNDEFINED } else { from_id }, if trash { ContactId::UNDEFINED } else { to_id }, sort_timestamp, if trash { 0 } else { mime_parser.timestamp_sent }, if trash { 0 } else { mime_parser.timestamp_rcvd }, - if trash { Viewtype::Unknown } else { typ }, + if trash { + Viewtype::Unknown + } else if let Some(mimeparser::PreMessageMode::PreMessage {..}) = mime_parser.pre_message { + Viewtype::Text + } else { typ }, if trash { MessageState::Undefined } else { state }, if trash { MessengerMessage::No } else { is_dc_message }, if trash || hidden { "" } else { msg }, @@ -2114,7 +2115,11 @@ RETURNING id param.to_string() }, !trash && hidden, - if trash { 0 } else { part.bytes as isize }, + if trash { + 0 + } else { + part.bytes as isize + }, if save_mime_modified && !(trash || hidden) { mime_headers.clone() } else { @@ -2128,10 +2133,10 @@ RETURNING id if trash { 0 } else { ephemeral_timestamp }, if trash { DownloadState::Done - } else if is_partial_download.is_some() { - DownloadState::Available } else if mime_parser.decrypting_failed { DownloadState::Undecipherable + } else if let Some(mimeparser::PreMessageMode::PreMessage {..}) = mime_parser.pre_message { + DownloadState::Available } else { DownloadState::Done }, @@ -2324,6 +2329,82 @@ async fn handle_edit_delete( Ok(()) } +async fn handle_post_message( + context: &Context, + mime_parser: &MimeMessage, + from_id: ContactId, +) -> Result<()> { + if let Some(mimeparser::PreMessageMode::PostMessage) = &mime_parser.pre_message { + // if Pre-Message exist, replace attachment + // only replacing attachment ensures that doesn't overwrite the text if it was edited before. + let rfc724_mid = mime_parser + .get_rfc724_mid() + .context("expected Post-Message to have a message id")?; + + let Some(msg_id) = message::rfc724_mid_exists(context, &rfc724_mid).await? else { + warn!( + context, + "Download Post-Message: Database entry does not exist." + ); + return Ok(()); + }; + let Some(original_msg) = Message::load_from_db_optional(context, msg_id).await? else { + // else: message is processed like a normal message + warn!( + context, + "Download Post-Message: pre message was not downloaded, yet so treat as normal message" + ); + return Ok(()); + }; + + if original_msg.from_id != from_id { + warn!(context, "Download Post-Message: Bad sender."); + return Ok(()); + } + if let Some(part) = mime_parser.parts.first() { + if !part.typ.has_file() { + warn!( + context, + "Download Post-Message: First mime part's message-viewtype has no file" + ); + return Ok(()); + } + + let edit_msg_showpadlock = part + .param + .get_bool(Param::GuaranteeE2ee) + .unwrap_or_default(); + + if edit_msg_showpadlock || !original_msg.get_showpadlock() { + let mut new_params = original_msg.param.clone(); + new_params + .merge_in_from_params(part.param.clone()) + .remove(Param::PostMessageFileBytes) + .remove(Param::PostMessageViewtype); + context + .sql + .execute( + "UPDATE msgs SET param=?, type=?, bytes=?, error=?, download_state=? WHERE id=?", + ( + new_params.to_string(), + part.typ, + part.bytes as isize, + part.error.as_deref().unwrap_or_default(), + DownloadState::Done as u32, + original_msg.id, + ), + ) + .await?; + context.emit_msgs_changed(original_msg.chat_id, original_msg.id); + } else { + warn!(context, "Download Post-Message: Not encrypted."); + } + } + } + + Ok(()) +} + async fn tweak_sort_timestamp( context: &Context, mime_parser: &mut MimeMessage, @@ -2413,7 +2494,6 @@ async fn lookup_chat_by_reply( context: &Context, mime_parser: &MimeMessage, parent: &Message, - is_partial_download: &Option, ) -> Result> { // If the message is encrypted and has group ID, // lookup by reply should never be needed @@ -2445,10 +2525,7 @@ async fn lookup_chat_by_reply( } // Do not assign unencrypted messages to encrypted chats. - if is_partial_download.is_none() - && parent_chat.is_encrypted(context).await? - && !mime_parser.was_encrypted() - { + if parent_chat.is_encrypted(context).await? && !mime_parser.was_encrypted() { return Ok(None); } @@ -2465,18 +2542,7 @@ async fn lookup_or_create_adhoc_group( to_ids: &[Option], allow_creation: bool, create_blocked: Blocked, - is_partial_download: bool, ) -> Result> { - // Partial download may be an encrypted message with protected Subject header. We do not want to - // create a group with "..." or "Encrypted message" as a subject. The same is for undecipherable - // messages. Instead, assign the message to 1:1 chat with the sender. - if is_partial_download { - info!( - context, - "Ad-hoc group cannot be created from partial download." - ); - return Ok(None); - } if mime_parser.decrypting_failed { warn!( context, @@ -2612,11 +2678,9 @@ async fn is_probably_private_reply( /// than two members, a new ad hoc group is created. /// /// On success the function returns the created (chat_id, chat_blocked) tuple. -#[expect(clippy::too_many_arguments)] async fn create_group( context: &Context, mime_parser: &mut MimeMessage, - is_partial_download: bool, create_blocked: Blocked, from_id: ContactId, to_ids: &[Option], @@ -2698,7 +2762,7 @@ async fn create_group( if let Some(chat_id) = chat_id { Ok(Some((chat_id, chat_id_blocked))) - } else if is_partial_download || mime_parser.decrypting_failed { + } else if mime_parser.decrypting_failed { // It is possible that the message was sent to a valid, // yet unknown group, which was rejected because // Chat-Group-Name, which is in the encrypted part, was diff --git a/src/receive_imf/receive_imf_tests.rs b/src/receive_imf/receive_imf_tests.rs index c7317a30aa..014784cc17 100644 --- a/src/receive_imf/receive_imf_tests.rs +++ b/src/receive_imf/receive_imf_tests.rs @@ -10,7 +10,6 @@ use crate::chat::{ use crate::chatlist::Chatlist; use crate::constants::DC_GCL_FOR_FORWARDING; use crate::contact; -use crate::download::MIN_DOWNLOAD_LIMIT; use crate::imap::prefetch_should_download; use crate::imex::{ImexMode, imex}; use crate::securejoin::get_securejoin_qr; @@ -19,8 +18,6 @@ use crate::test_utils::{ }; use crate::tools::{SystemTime, time}; -use rand::distr::SampleString; - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_outgoing() -> Result<()> { let context = TestContext::new_alice().await; @@ -28,7 +25,7 @@ async fn test_outgoing() -> Result<()> { From: alice@example.org\n\ \n\ hello"; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None).await?; + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).await?; assert_eq!(mimeparser.incoming, false); Ok(()) } @@ -43,7 +40,7 @@ async fn test_bad_from() { References: \n\ \n\ hello\x00"; - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None).await; + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).await; assert!(mimeparser.is_err()); } @@ -2842,7 +2839,7 @@ References: Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no Message with references."#; - let mime_parser = MimeMessage::from_bytes(&t, &mime[..], None).await?; + let mime_parser = MimeMessage::from_bytes(&t, &mime[..]).await?; let parent = get_parent_message(&t, &mime_parser).await?.unwrap(); assert_eq!(parent.id, first.id); @@ -4385,37 +4382,6 @@ async fn test_adhoc_grp_name_no_prefix() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_download_later() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = tcm.alice().await; - alice.set_config(Config::DownloadLimit, Some("1")).await?; - assert_eq!(alice.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT)); - - let bob = tcm.bob().await; - let bob_chat = bob.create_chat(&alice).await; - - // Generate a random string so OpenPGP does not compress it. - let text = - rand::distr::Alphanumeric.sample_string(&mut rand::rng(), MIN_DOWNLOAD_LIMIT as usize); - - let sent_msg = bob.send_text(bob_chat.id, &text).await; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Available); - assert_eq!(msg.state, MessageState::InFresh); - - let hi_msg = tcm.send_recv(&bob, &alice, "hi").await; - - alice.set_config(Config::DownloadLimit, None).await?; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Done); - assert_eq!(msg.state, MessageState::InFresh); - assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, hi_msg.id); - assert!(msg.timestamp_sort <= hi_msg.timestamp_sort); - - Ok(()) -} - /// Malice can pretend they have the same address as Alice and sends a message encrypted to Alice's /// key but signed with another one. Alice must detect that this message is wrongly signed and not /// treat it as Autocrypt-encrypted. @@ -4450,162 +4416,6 @@ async fn test_outgoing_msg_forgery() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_create_group_with_big_msg() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = tcm.alice().await; - let bob = tcm.bob().await; - let ba_contact = bob.add_or_lookup_contact_id(&alice).await; - let ab_chat_id = alice.create_chat(&bob).await.id; - - let file_bytes = include_bytes!("../../test-data/image/screenshot.png"); - - let bob_grp_id = create_group(&bob, "Group").await?; - add_contact_to_chat(&bob, bob_grp_id, ba_contact).await?; - let mut msg = Message::new(Viewtype::Image); - msg.set_file_from_bytes(&bob, "a.jpg", file_bytes, None)?; - let sent_msg = bob.send_msg(bob_grp_id, &mut msg).await; - assert!(msg.get_showpadlock()); - - alice.set_config(Config::DownloadLimit, Some("1")).await?; - assert_eq!(alice.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT)); - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Available); - let alice_chat = Chat::load_from_db(&alice, msg.chat_id).await?; - // Incomplete message is assigned to 1:1 chat. - assert_eq!(alice_chat.typ, Chattype::Single); - - alice.set_config(Config::DownloadLimit, None).await?; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Done); - assert_eq!(msg.state, MessageState::InFresh); - assert_eq!(msg.viewtype, Viewtype::Image); - assert_ne!(msg.chat_id, alice_chat.id); - let alice_grp = Chat::load_from_db(&alice, msg.chat_id).await?; - assert_eq!(alice_grp.typ, Chattype::Group); - assert_eq!(alice_grp.name, "Group"); - assert_eq!( - chat::get_chat_contacts(&alice, alice_grp.id).await?.len(), - 2 - ); - - // Now Bob can send encrypted messages to Alice. - - let bob_grp_id = create_group(&bob, "Group1").await?; - add_contact_to_chat(&bob, bob_grp_id, ba_contact).await?; - let mut msg = Message::new(Viewtype::Image); - msg.set_file_from_bytes(&bob, "a.jpg", file_bytes, None)?; - let sent_msg = bob.send_msg(bob_grp_id, &mut msg).await; - assert!(msg.get_showpadlock()); - - alice.set_config(Config::DownloadLimit, Some("1")).await?; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Available); - // Until fully downloaded, an encrypted message must sit in the 1:1 chat. - assert_eq!(msg.chat_id, ab_chat_id); - - alice.set_config(Config::DownloadLimit, None).await?; - let msg = alice.recv_msg(&sent_msg).await; - assert_eq!(msg.download_state, DownloadState::Done); - assert_eq!(msg.state, MessageState::InFresh); - assert_eq!(msg.viewtype, Viewtype::Image); - assert_ne!(msg.chat_id, ab_chat_id); - let alice_grp = Chat::load_from_db(&alice, msg.chat_id).await?; - assert_eq!(alice_grp.typ, Chattype::Group); - assert_eq!(alice_grp.name, "Group1"); - assert_eq!( - chat::get_chat_contacts(&alice, alice_grp.id).await?.len(), - 2 - ); - - // The big message must go away from the 1:1 chat. - let msgs = chat::get_chat_msgs(&alice, ab_chat_id).await?; - assert_eq!(msgs.len(), E2EE_INFO_MSGS); - - Ok(()) -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_partial_group_consistency() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = tcm.alice().await; - let bob = tcm.bob().await; - let fiona = tcm.fiona().await; - let bob_id = alice.add_or_lookup_contact_id(&bob).await; - let alice_chat_id = create_group(&alice, "foos").await?; - add_contact_to_chat(&alice, alice_chat_id, bob_id).await?; - - send_text_msg(&alice, alice_chat_id, "populate".to_string()).await?; - let add = alice.pop_sent_msg().await; - bob.recv_msg(&add).await; - let bob_chat_id = bob.get_last_msg().await.chat_id; - let contacts = get_chat_contacts(&bob, bob_chat_id).await?; - assert_eq!(contacts.len(), 2); - - // Bob receives partial message. - let msg_id = receive_imf_from_inbox( - &bob, - "first@example.org", - b"From: Alice \n\ -To: , \n\ -Chat-Version: 1.0\n\ -Subject: subject\n\ -Message-ID: \n\ -Date: Sun, 14 Nov 2021 00:10:00 +0000\ -Content-Type: text/plain -Chat-Group-Member-Added: charlie@example.com", - false, - Some(100000), - ) - .await? - .context("no received message")?; - - let msg = Message::load_from_db(&bob, msg_id.msg_ids[0]).await?; - - // Partial download does not change the member list. - assert_eq!(msg.download_state, DownloadState::Available); - assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?, contacts); - - // Alice sends normal message to bob, adding fiona. - add_contact_to_chat( - &alice, - alice_chat_id, - alice.add_or_lookup_contact_id(&fiona).await, - ) - .await?; - - bob.recv_msg(&alice.pop_sent_msg().await).await; - - let contacts = get_chat_contacts(&bob, bob_chat_id).await?; - assert_eq!(contacts.len(), 3); - - // Bob fully receives the partial message. - let msg_id = receive_imf_from_inbox( - &bob, - "first@example.org", - b"From: Alice \n\ -To: Bob \n\ -Chat-Version: 1.0\n\ -Subject: subject\n\ -Message-ID: \n\ -Date: Sun, 14 Nov 2021 00:10:00 +0000\ -Content-Type: text/plain -Chat-Group-Member-Added: charlie@example.com", - false, - None, - ) - .await? - .context("no received message")?; - - let msg = Message::load_from_db(&bob, msg_id.msg_ids[0]).await?; - - // After full download, the old message should not change group state. - assert_eq!(msg.download_state, DownloadState::Done); - assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?, contacts); - - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_protected_group_add_remove_member_missing_key() -> Result<()> { let mut tcm = TestContextManager::new(); @@ -4844,48 +4654,6 @@ async fn test_references() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_prefer_references_to_downloaded_msgs() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - let bob = &tcm.bob().await; - bob.set_config(Config::DownloadLimit, Some("1")).await?; - let fiona = &tcm.fiona().await; - let alice_bob_id = tcm.send_recv(bob, alice, "hi").await.from_id; - let alice_fiona_id = tcm.send_recv(fiona, alice, "hi").await.from_id; - let alice_chat_id = create_group(alice, "Group").await?; - add_contact_to_chat(alice, alice_chat_id, alice_bob_id).await?; - // W/o fiona the test doesn't work -- the last message is assigned to the 1:1 chat due to - // `is_probably_private_reply()`. - add_contact_to_chat(alice, alice_chat_id, alice_fiona_id).await?; - let sent = alice.send_text(alice_chat_id, "Hi").await; - let received = bob.recv_msg(&sent).await; - assert_eq!(received.download_state, DownloadState::Done); - let bob_chat_id = received.chat_id; - - let file_bytes = include_bytes!("../../test-data/image/screenshot.gif"); - let mut msg = Message::new(Viewtype::File); - msg.set_file_from_bytes(alice, "file", file_bytes, None)?; - let mut sent = alice.send_msg(alice_chat_id, &mut msg).await; - sent.payload = sent - .payload - .replace("References:", "X-Microsoft-Original-References:") - .replace("In-Reply-To:", "X-Microsoft-Original-In-Reply-To:"); - let received = bob.recv_msg(&sent).await; - assert_eq!(received.download_state, DownloadState::Available); - assert_ne!(received.chat_id, bob_chat_id); - assert_eq!(received.chat_id, bob.get_chat(alice).await.id); - - let mut msg = Message::new(Viewtype::File); - msg.set_file_from_bytes(alice, "file", file_bytes, None)?; - let sent = alice.send_msg(alice_chat_id, &mut msg).await; - let received = bob.recv_msg(&sent).await; - assert_eq!(received.download_state, DownloadState::Available); - assert_eq!(received.chat_id, bob_chat_id); - - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_list_from() -> Result<()> { let t = &TestContext::new_alice().await; @@ -5363,41 +5131,6 @@ async fn test_outgoing_plaintext_two_member_group() -> Result<()> { Ok(()) } -/// Tests that large messages are assigned -/// to non-key-contacts if the type is not `multipart/encrypted`. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_partial_download_key_contact_lookup() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - let bob = &tcm.bob().await; - - // Create two chats with Alice, both with key-contact and email address contact. - let encrypted_chat = bob.create_chat(alice).await; - let unencrypted_chat = bob.create_email_chat(alice).await; - - let seen = false; - let is_partial_download = Some(9999); - let received = receive_imf_from_inbox( - bob, - "3333@example.org", - b"From: alice@example.org\n\ - To: bob@example.net\n\ - Message-ID: <3333@example.org>\n\ - Date: Sun, 22 Mar 2020 22:37:57 +0000\n\ - \n\ - hello\n", - seen, - is_partial_download, - ) - .await? - .unwrap(); - - assert_ne!(received.chat_id, encrypted_chat.id); - assert_eq!(received.chat_id, unencrypted_chat.id); - - Ok(()) -} - /// Tests that outgoing unencrypted message /// is assigned to a chat with email-contact. /// diff --git a/src/scheduler.rs b/src/scheduler.rs index b8bfd7b759..caf968fb16 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -14,13 +14,12 @@ pub(crate) use self::connectivity::ConnectivityStore; use crate::config::{self, Config}; use crate::contact::{ContactId, RecentlySeenLoop}; use crate::context::Context; -use crate::download::{DownloadState, download_msg}; +use crate::download::{download_known_post_messages_without_pre_message, download_msgs}; use crate::ephemeral::{self, delete_expired_imap_messages}; use crate::events::EventType; use crate::imap::{FolderMeaning, Imap, session::Session}; use crate::location; use crate::log::{LogExt, warn}; -use crate::message::MsgId; use crate::smtp::{Smtp, send_smtp_messages}; use crate::sql; use crate::stats::maybe_send_stats; @@ -351,38 +350,6 @@ pub(crate) struct Scheduler { recently_seen_loop: RecentlySeenLoop, } -async fn download_msgs(context: &Context, session: &mut Session) -> Result<()> { - let msg_ids = context - .sql - .query_map_vec("SELECT msg_id FROM download", (), |row| { - let msg_id: MsgId = row.get(0)?; - Ok(msg_id) - }) - .await?; - - for msg_id in msg_ids { - if let Err(err) = download_msg(context, msg_id, session).await { - warn!(context, "Failed to download message {msg_id}: {:#}.", err); - - // Update download state to failure - // so it can be retried. - // - // On success update_download_state() is not needed - // as receive_imf() already - // set the state and emitted the event. - msg_id - .update_download_state(context, DownloadState::Failure) - .await?; - } - context - .sql - .execute("DELETE FROM download WHERE msg_id=?", (msg_id,)) - .await?; - } - - Ok(()) -} - async fn inbox_loop( ctx: Context, started: oneshot::Sender<()>, @@ -619,6 +586,11 @@ async fn fetch_idle( delete_expired_imap_messages(ctx) .await .context("delete_expired_imap_messages")?; + + //------- + // TODO: verify that this is the correct position for this call + // in order to guard against lost pre-messages: + download_known_post_messages_without_pre_message(ctx, &mut session).await?; } else if folder_config == Config::ConfiguredInboxFolder { session.last_full_folder_scan.lock().await.take(); } @@ -704,6 +676,7 @@ async fn fetch_idle( Ok(session) } +/// The simplified IMAP IDLE loop to watch non primary folders (non-inbox folders) async fn simple_imap_loop( ctx: Context, started: oneshot::Sender<()>, diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index 80ac2c564a..bfec5b69bc 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -1466,6 +1466,30 @@ ALTER TABLE contacts ADD COLUMN name_normalized TEXT; .await?; } + inc_and_check(&mut migration_version, 144)?; + if dbversion < migration_version { + // `msg_id` in `download` table is not needed anymore, + // but we still keep it so that it's possible to import a backup into an older DC version, + // because we don't always release at the same time on all platforms. + sql.execute_migration( + "CREATE TABLE download_new ( + rfc724_mid TEXT NOT NULL DEFAULT '', + msg_id INTEGER NOT NULL DEFAULT 0 + ) STRICT; + INSERT OR IGNORE INTO download_new (rfc724_mid, msg_id) + SELECT m.rfc724_mid, d.msg_id FROM download d + JOIN msgs m ON d.msg_id = m.id + WHERE m.rfc724_mid IS NOT NULL AND m.rfc724_mid != ''; + DROP TABLE download; + ALTER TABLE download_new RENAME TO download; + CREATE TABLE available_post_msgs ( + rfc724_mid TEXT NOT NULL + );", + migration_version, + ) + .await?; + } + let new_version = sql .get_raw_config_int(VERSION_CFG) .await? diff --git a/src/stock_str.rs b/src/stock_str.rs index 291141f0d9..b9388f8f38 100644 --- a/src/stock_str.rs +++ b/src/stock_str.rs @@ -4,7 +4,6 @@ use std::collections::HashMap; use std::sync::Arc; use anyhow::{Result, bail}; -use humansize::{BINARY, format_size}; use strum::EnumProperty as EnumPropertyTrait; use strum_macros::EnumProperty; use tokio::sync::RwLock; @@ -17,7 +16,6 @@ use crate::contact::{Contact, ContactId}; use crate::context::Context; use crate::message::{Message, Viewtype}; use crate::param::Param; -use crate::tools::timestamp_to_str; /// Storage for string translations. #[derive(Debug, Clone)] @@ -167,12 +165,6 @@ pub enum StockMessage { ))] QuotaExceedingMsgBody = 98, - #[strum(props(fallback = "%1$s message"))] - PartialDownloadMsgBody = 99, - - #[strum(props(fallback = "Download maximum available until %1$s"))] - DownloadAvailability = 100, - #[strum(props(fallback = "Multi Device Synchronization"))] SyncMsgSubject = 101, @@ -423,6 +415,9 @@ https://delta.chat/donate"))] #[strum(props(fallback = "Messages in this chat use classic email and are not encrypted."))] ChatUnencryptedExplanation = 230, + + #[strum(props(fallback = "Contact"))] + Contact = 231, } impl StockMessage { @@ -890,6 +885,11 @@ pub(crate) async fn sticker(context: &Context) -> String { translated(context, StockMessage::Sticker).await } +/// Stock string: `Contact`. +pub(crate) async fn contact(context: &Context) -> String { + translated(context, StockMessage::Contact).await +} + /// Stock string: `Device messages`. pub(crate) async fn device_messages(context: &Context) -> String { translated(context, StockMessage::DeviceMessages).await @@ -1119,21 +1119,6 @@ pub(crate) async fn quota_exceeding(context: &Context, highest_usage: u64) -> St .replace("%%", "%") } -/// Stock string: `%1$s message` with placeholder replaced by human-readable size. -pub(crate) async fn partial_download_msg_body(context: &Context, org_bytes: u32) -> String { - let size = &format_size(org_bytes, BINARY); - translated(context, StockMessage::PartialDownloadMsgBody) - .await - .replace1(size) -} - -/// Stock string: `Download maximum available until %1$s`. -pub(crate) async fn download_availability(context: &Context, timestamp: i64) -> String { - translated(context, StockMessage::DownloadAvailability) - .await - .replace1(×tamp_to_str(timestamp)) -} - /// Stock string: `Incoming Messages`. pub(crate) async fn incoming_messages(context: &Context) -> String { translated(context, StockMessage::IncomingMessages).await @@ -1262,6 +1247,24 @@ pub(crate) async fn chat_unencrypted_explanation(context: &Context) -> String { translated(context, StockMessage::ChatUnencryptedExplanation).await } +impl Viewtype { + /// returns Localized name for message viewtype + pub async fn to_locale_string(&self, context: &Context) -> String { + match self { + Viewtype::Image => image(context).await, + Viewtype::Gif => gif(context).await, + Viewtype::Sticker => sticker(context).await, + Viewtype::Audio => audio(context).await, + Viewtype::Voice => voice_message(context).await, + Viewtype::Video => video(context).await, + Viewtype::File => file(context).await, + Viewtype::Webxdc => "Mini App".to_owned(), + Viewtype::Vcard => contact(context).await, + Viewtype::Unknown | Viewtype::Text | Viewtype::Call => self.to_string(), + } + } +} + impl Context { /// Set the stock string for the [StockMessage]. /// diff --git a/src/stock_str/stock_str_tests.rs b/src/stock_str/stock_str_tests.rs index 8af98be53f..37c03efabf 100644 --- a/src/stock_str/stock_str_tests.rs +++ b/src/stock_str/stock_str_tests.rs @@ -118,14 +118,6 @@ async fn test_quota_exceeding_stock_str() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_partial_download_msg_body() -> Result<()> { - let t = TestContext::new().await; - let str = partial_download_msg_body(&t, 1024 * 1024).await; - assert_eq!(str, "1 MiB message"); - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_update_device_chats() { let t = TestContext::new_alice().await; diff --git a/src/test_utils.rs b/src/test_utils.rs index 8f837ebf11..0503e2e274 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -10,6 +10,7 @@ use std::path::Path; use std::sync::{Arc, LazyLock}; use std::time::{Duration, Instant}; +use anyhow::Result; use async_channel::{self as channel, Receiver, Sender}; use chat::ChatItem; use deltachat_contact_tools::{ContactAddress, EmailAddress}; @@ -711,6 +712,32 @@ impl TestContext { }) } + pub async fn get_smtp_rows_for_msg<'a>(&'a self, msg_id: MsgId) -> Vec> { + self.ctx + .sql + .query_map_vec( + "SELECT id, msg_id, mime, recipients FROM smtp WHERE msg_id=?", + (msg_id,), + |row| { + let _id: MsgId = row.get(0)?; + let msg_id: MsgId = row.get(1)?; + let mime: String = row.get(2)?; + let recipients: String = row.get(3)?; + Ok((msg_id, mime, recipients)) + }, + ) + .await + .unwrap() + .into_iter() + .map(|(msg_id, mime, recipients)| SentMessage { + payload: mime, + sender_msg_id: msg_id, + sender_context: &self.ctx, + recipients, + }) + .collect() + } + /// Retrieves a sent sync message from the db. /// /// This retrieves and removes a sync message which has been scheduled to send from the jobs @@ -759,7 +786,7 @@ impl TestContext { /// unlikely to be affected as the message would be processed again in exactly the /// same way. pub(crate) async fn parse_msg(&self, msg: &SentMessage<'_>) -> MimeMessage { - MimeMessage::from_bytes(&self.ctx, msg.payload().as_bytes(), None) + MimeMessage::from_bytes(&self.ctx, msg.payload().as_bytes()) .await .unwrap() } @@ -1702,6 +1729,21 @@ Until the false-positive is fixed: } } +/// Method to create a test image file +pub(crate) fn create_test_image(width: u32, height: u32) -> Result> { + use image::{ImageBuffer, Rgb, RgbImage}; + use std::io::Cursor; + + let mut img: RgbImage = ImageBuffer::new(width, height); + // fill with some pattern so it stays large after compression + for (x, y, pixel) in img.enumerate_pixels_mut() { + *pixel = Rgb([(x % 255) as u8, (x + y % 255) as u8, (y % 255) as u8]); + } + let mut bytes: Vec = Vec::new(); + img.write_to(&mut Cursor::new(&mut bytes), image::ImageFormat::Png)?; + Ok(bytes) +} + mod tests { use super::*; diff --git a/src/tests.rs b/src/tests.rs index 6e642dce74..b7ae08fbad 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,3 +1,4 @@ mod account_events; mod aeap; +mod pre_messages; mod verified_chats; diff --git a/src/tests/pre_messages.rs b/src/tests/pre_messages.rs new file mode 100644 index 0000000000..a0387ddee4 --- /dev/null +++ b/src/tests/pre_messages.rs @@ -0,0 +1,6 @@ +mod additional_text; +mod forward_and_save; +mod legacy; +mod receiving; +mod sending; +mod util; diff --git a/src/tests/pre_messages/additional_text.rs b/src/tests/pre_messages/additional_text.rs new file mode 100644 index 0000000000..b894d34e98 --- /dev/null +++ b/src/tests/pre_messages/additional_text.rs @@ -0,0 +1,40 @@ +use anyhow::Result; +use pretty_assertions::assert_eq; + +use crate::message::Viewtype; +use crate::test_utils::TestContextManager; +use crate::tests::pre_messages::util::{ + send_large_file_message, send_large_image_message, send_large_webxdc_message, +}; + +/// Test the addition of the download info to message text +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_additional_text_on_different_viewtypes() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let a_group_id = alice.create_group_with_members("test group", &[bob]).await; + + tcm.section("Test metadata preview text for File"); + let (pre_message, _, _) = + send_large_file_message(alice, a_group_id, Viewtype::File, &vec![0u8; 1_000_000]).await?; + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.text, "test".to_owned()); + assert_eq!(msg.get_text(), "test [test.bin - 976.56 KiB]".to_owned()); + + tcm.section("Test metadata preview text for webxdc app"); + let (pre_message, _, _) = send_large_webxdc_message(alice, a_group_id).await?; + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.text, "test".to_owned()); + assert_eq!(msg.get_post_message_viewtype(), Some(Viewtype::Webxdc)); + assert_eq!(msg.get_text(), "test [Mini App - 976.68 KiB]".to_owned()); + + tcm.section("Test metadata preview text for Image"); + + let (pre_message, _, _) = send_large_image_message(alice, a_group_id).await?; + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.text, "test".to_owned()); + assert_eq!(msg.get_text(), "test [Image - 146.12 KiB]".to_owned()); + + Ok(()) +} diff --git a/src/tests/pre_messages/forward_and_save.rs b/src/tests/pre_messages/forward_and_save.rs new file mode 100644 index 0000000000..a3ca16ed8a --- /dev/null +++ b/src/tests/pre_messages/forward_and_save.rs @@ -0,0 +1,122 @@ +//! Tests about forwarding and saving Pre-Messages +use anyhow::Result; +use pretty_assertions::assert_eq; + +use crate::chat::{self}; +use crate::chat::{forward_msgs, save_msgs}; +use crate::chatlist::get_last_message_for_chat; +use crate::download::{DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD}; +use crate::message::{Message, Viewtype}; +use crate::test_utils::TestContextManager; + +/// Test that forwarding Pre-Message should forward additional text to not be empty +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_forwarding_pre_message_empty_text() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let pre_message = { + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 1_000_000], None)?; + assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + let msg_id = chat::send_msg(alice, alice_group_id, &mut msg).await?; + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + assert_eq!(smtp_rows.len(), 2); + smtp_rows.first().expect("Pre-Message exists").to_owned() + }; + + let bob_msg = bob.recv_msg(&pre_message).await; + assert_eq!(bob_msg.download_state, DownloadState::Available); + bob_msg.chat_id.accept(bob).await?; + tcm.section("forward pre message and check it on bobs side"); + forward_msgs(bob, &[bob_msg.id], bob_msg.chat_id).await?; + let forwarded_msg_id = get_last_message_for_chat(bob, bob_msg.chat_id) + .await? + .unwrap(); + let forwarded_msg = Message::load_from_db(bob, forwarded_msg_id).await?; + assert_eq!(forwarded_msg.is_forwarded(), true); + assert_eq!(forwarded_msg.download_state(), DownloadState::Done); + assert_eq!( + forwarded_msg + .param + .exists(crate::param::Param::PostMessageFileBytes), + false, + "PostMessageFileBytes not set" + ); + assert_eq!( + forwarded_msg + .param + .exists(crate::param::Param::PostMessageViewtype), + false, + "PostMessageViewtype not set" + ); + assert_eq!( + forwarded_msg.get_text(), + " [test.bin - 976.56 KiB]".to_owned() + ); + assert_eq!(forwarded_msg.get_viewtype(), Viewtype::Text); + assert!(forwarded_msg.additional_text.is_empty()); + tcm.section("check it on alices side"); + let sent_forward_msg = bob.pop_sent_msg().await; + let alice_forwarded_msg = alice.recv_msg(&sent_forward_msg).await; + assert!(alice_forwarded_msg.additional_text.is_empty()); + assert_eq!(alice_forwarded_msg.is_forwarded(), true); + assert_eq!(alice_forwarded_msg.download_state(), DownloadState::Done); + assert_eq!( + alice_forwarded_msg + .param + .exists(crate::param::Param::PostMessageFileBytes), + false, + "PostMessageFileBytes not set" + ); + assert_eq!( + alice_forwarded_msg + .param + .exists(crate::param::Param::PostMessageViewtype), + false, + "PostMessageViewtype not set" + ); + assert_eq!( + alice_forwarded_msg.get_text(), + " [test.bin - 976.56 KiB]".to_owned() + ); + + Ok(()) +} + +/// Test that forwarding Pre-Message should forward additional text to not be empty +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_saving_pre_message_empty_text() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let pre_message = { + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 1_000_000], None)?; + assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + let msg_id = chat::send_msg(alice, alice_group_id, &mut msg).await?; + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + assert_eq!(smtp_rows.len(), 2); + smtp_rows.first().expect("Pre-Message exists").to_owned() + }; + + let bob_msg = bob.recv_msg(&pre_message).await; + assert_eq!(bob_msg.download_state, DownloadState::Available); + bob_msg.chat_id.accept(bob).await?; + tcm.section("save pre message and check it"); + save_msgs(bob, &[bob_msg.id]).await?; + let saved_msg_id = get_last_message_for_chat(bob, bob.get_self_chat().await.id) + .await? + .unwrap(); + let saved_msg = Message::load_from_db(bob, saved_msg_id).await?; + assert!(saved_msg.additional_text.is_empty()); + assert!(saved_msg.get_original_msg_id(bob).await?.is_some()); + assert_eq!(saved_msg.download_state(), DownloadState::Done); + assert_eq!(saved_msg.get_text(), " [test.bin - 976.56 KiB]".to_owned()); + + Ok(()) +} diff --git a/src/tests/pre_messages/legacy.rs b/src/tests/pre_messages/legacy.rs new file mode 100644 index 0000000000..f8f086d1bd --- /dev/null +++ b/src/tests/pre_messages/legacy.rs @@ -0,0 +1,61 @@ +//! Test that downloading old stub messages still works +use anyhow::Result; + +use crate::download::DownloadState; +use crate::receive_imf::receive_imf_from_inbox; +use crate::test_utils::TestContext; + +// The code for downloading stub messages stays +// during the transition perios to pre-messages +// so people can still download their files shortly after they updated. +// After there are a few release with pre-message rolled out, +// we will remove the ability to download stub messages and replace the following test +// so it checks that it doesn't crash or that the messages are replaced by sth. +// like "download failed/expired, please ask sender to send it again" +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_download_stub_message() -> Result<()> { + let t = TestContext::new_alice().await; + + let header = "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: bob@example.com\n\ + To: alice@example.org\n\ + Subject: foo\n\ + Message-ID: \n\ + Chat-Version: 1.0\n\ + Date: Sun, 22 Mar 2020 22:37:57 +0000\ + Content-Type: text/plain"; + + t.sql + .execute( + r#"INSERT INTO chats VALUES( + 11001,100,'bob@example.com',0,'',2,'', + replace('C=1763151754\nt=foo','\n',char(10)),0,0,0,0,0,1763151754,0,NULL,0,''); + "#, + (), + ) + .await?; + t.sql.execute(r#"INSERT INTO msgs VALUES( + 11001,'Mr.12345678901@example.com','',0, + 11001,11001,1,1763151754,10,10,1,0, + '[97.66 KiB message]','','',0,1763151754,1763151754,0,X'', + '','',1,0,'',0,0,0,'foo',10,replace('Hop: From: userid; Date: Mon, 4 Dec 2006 13:51:39 +0000\n\nDKIM Results: Passed=true','\n',char(10)),1,NULL,0); + "#, ()).await?; + let msg = t.get_last_msg().await; + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.get_subject(), "foo"); + assert!(msg.get_text().contains("[97.66 KiB message]")); + + receive_imf_from_inbox( + &t, + "Mr.12345678901@example.com", + format!("{header}\n\n100k text...").as_bytes(), + false, + ) + .await?; + let msg = t.get_last_msg().await; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.get_subject(), "foo"); + assert_eq!(msg.get_text(), "100k text..."); + + Ok(()) +} diff --git a/src/tests/pre_messages/receiving.rs b/src/tests/pre_messages/receiving.rs new file mode 100644 index 0000000000..fd9c376c7a --- /dev/null +++ b/src/tests/pre_messages/receiving.rs @@ -0,0 +1,522 @@ +//! Tests about receiving Pre-Messages and Post-Message +use anyhow::Result; +use pretty_assertions::assert_eq; + +use crate::EventType; +use crate::chat; +use crate::contact; +use crate::download::{ + DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD, pre_msg_metadata::PreMsgMetadata, +}; +use crate::message::{Message, MessageState, Viewtype, delete_msgs, markseen_msgs}; +use crate::mimeparser::MimeMessage; +use crate::param::Param; +use crate::reaction::{get_msg_reactions, send_reaction}; +use crate::test_utils::TestContextManager; +use crate::tests::pre_messages::util::{ + send_large_file_message, send_large_image_message, send_large_webxdc_message, +}; +use crate::webxdc::StatusUpdateSerial; + +/// Test that mimeparser can correctly detect and parse pre-messages and Post-Messages +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_mimeparser_pre_message_and_post_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, post_message, _alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + let parsed_pre_message = MimeMessage::from_bytes(bob, pre_message.payload.as_bytes()).await?; + let parsed_post_message = MimeMessage::from_bytes(bob, post_message.payload.as_bytes()).await?; + + assert_eq!( + parsed_post_message.pre_message, + Some(crate::mimeparser::PreMessageMode::PostMessage) + ); + + assert_eq!( + parsed_pre_message.pre_message, + Some(crate::mimeparser::PreMessageMode::PreMessage { + post_msg_rfc724_mid: parsed_post_message.get_rfc724_mid().unwrap(), + metadata: Some(PreMsgMetadata { + size: 1_000_000, + viewtype: Viewtype::File, + filename: "test.bin".to_string(), + dimensions: None, + duration: None + }) + }) + ); + + Ok(()) +} + +/// Test receiving pre-messages and creation of the placeholder message with the metadata +/// for file attachment +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_receive_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, _post_message, _alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + let msg = bob.recv_msg(&pre_message).await; + + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.viewtype, Viewtype::Text); + assert_eq!(msg.text, "test".to_owned()); + + // test that metadata is correctly returned by methods + assert_eq!(msg.get_filebytes(bob).await?, Some(1_000_000)); + assert_eq!(msg.get_post_message_viewtype(), Some(Viewtype::File)); + assert_eq!(msg.get_filename(), Some("test.bin".to_owned())); + + Ok(()) +} + +/// Test receiving the Post-Message after receiving the pre-message +/// for file attachment +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_receive_pre_message_and_dl_post_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, post_message, _alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.viewtype, Viewtype::Text); + assert!(msg.param.exists(Param::PostMessageViewtype)); + assert!(msg.param.exists(Param::PostMessageFileBytes)); + assert_eq!(msg.text, "test".to_owned()); + let _ = bob.recv_msg_trash(&post_message).await; + let msg = Message::load_from_db(bob, msg.id).await?; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.viewtype, Viewtype::File); + assert_eq!(msg.param.exists(Param::PostMessageViewtype), false); + assert_eq!(msg.param.exists(Param::PostMessageFileBytes), false); + assert_eq!(msg.text, "test".to_owned()); + Ok(()) +} + +/// Test out of order receiving. Post-Message is received & downloaded before pre-message. +/// In that case pre-message shall be trashed. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_out_of_order_receiving() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, post_message, _alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + let msg = bob.recv_msg(&post_message).await; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.viewtype, Viewtype::File); + let _ = bob.recv_msg_trash(&pre_message).await; + Ok(()) +} + +/// Test receiving the Post-Message after receiving an edit after receiving the pre-message +/// for file attachment +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_receive_pre_message_then_edit_and_then_dl_post_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, post_message, alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + chat::send_edit_request(alice, alice_msg_id, "new_text".to_owned()).await?; + let edit_request = alice.pop_sent_msg().await; + + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.text, "test".to_owned()); + let _ = bob.recv_msg_trash(&edit_request).await; + let msg = Message::load_from_db(bob, msg.id).await?; + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.text, "new_text".to_owned()); + let _ = bob.recv_msg_trash(&post_message).await; + let msg = Message::load_from_db(bob, msg.id).await?; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.viewtype, Viewtype::File); + assert_eq!(msg.text, "new_text".to_owned()); + Ok(()) +} + +/// Process normal message with file attachment (neither post nor pre message) +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_receive_normal_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes( + alice, + "test.bin", + &vec![0u8; (PRE_MSG_ATTACHMENT_SIZE_THRESHOLD - 10_000) as usize], + None, + )?; + msg.set_text("test".to_owned()); + let msg_id = chat::send_msg(alice, alice_group_id, &mut msg).await?; + + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + assert_eq!(smtp_rows.len(), 1); + let message = smtp_rows.first().expect("message exists"); + + let msg = bob.recv_msg(message).await; + assert_eq!(msg.download_state(), DownloadState::Done); + assert_eq!(msg.viewtype, Viewtype::File); + assert_eq!(msg.text, "test".to_owned()); + Ok(()) +} + +/// Test receiving pre-messages and creation of the placeholder message with the metadata +/// for image attachment +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_receive_pre_message_image() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, _post_message, _alice_msg_id) = + send_large_image_message(alice, alice_group_id).await?; + + let msg = bob.recv_msg(&pre_message).await; + + assert_eq!(msg.download_state(), DownloadState::Available); + assert_eq!(msg.viewtype, Viewtype::Text); + assert_eq!(msg.text, "test".to_owned()); + + // test that metadata is correctly returned by methods + assert_eq!(msg.get_post_message_viewtype(), Some(Viewtype::Image)); + // recoded image dimensions + assert_eq!(msg.get_filebytes(bob).await?, Some(149632)); + assert_eq!(msg.get_height(), 1280); + assert_eq!(msg.get_width(), 720); + + Ok(()) +} + +/// Test receiving reaction on pre-message +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_reaction_on_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + let (pre_message, post_message, alice_msg_id) = + send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000]) + .await?; + + // Bob receives pre-message + let bob_msg = bob.recv_msg(&pre_message).await; + assert_eq!(bob_msg.download_state(), DownloadState::Available); + + // Alice sends reaction to her own message + send_reaction(alice, alice_msg_id, "👍").await?; + + // Bob receives the reaction + bob.recv_msg_hidden(&alice.pop_sent_msg().await).await; + + // Test if Bob sees reaction + let reactions = get_msg_reactions(bob, bob_msg.id).await?; + assert_eq!(reactions.to_string(), "👍1"); + + // Bob downloads Post-Message + bob.recv_msg_trash(&post_message).await; + let msg = Message::load_from_db(bob, bob_msg.id).await?; + assert_eq!(msg.download_state(), DownloadState::Done); + + // Test if Bob still sees reaction + let reactions = get_msg_reactions(bob, bob_msg.id).await?; + assert_eq!(reactions.to_string(), "👍1"); + + Ok(()) +} + +/// Tests that fully downloading the message +/// works but does not reappear when it was already deleted +/// (as in the Message-ID already exists in the database +/// and is assigned to the trash chat). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_full_download_after_trashed() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let bob_group_id = bob.create_group_with_members("test group", &[alice]).await; + + let (pre_message, post_message, _bob_msg_id) = + send_large_file_message(bob, bob_group_id, Viewtype::File, &vec![0u8; 1_000_000]).await?; + + // Download message from Bob partially. + let alice_msg = alice.recv_msg(&pre_message).await; + + // Delete the received message. + // Note that it remains in the database in the trash chat. + delete_msgs(alice, &[alice_msg.id]).await?; + + // Fully download message after deletion. + alice.recv_msg_trash(&post_message).await; + + // The message does not reappear. + let msg = Message::load_from_db_optional(bob, alice_msg.id).await?; + assert!(msg.is_none()); + + Ok(()) +} + +/// Test that webxdc updates are received for pre-messages +/// and available when the Post-Message is downloaded +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_webxdc_update_for_not_downloaded_instance() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let alice_group_id = alice.create_group_with_members("test group", &[bob]).await; + + // Alice sends a larger instance and an update + let (pre_message, post_message, alice_sent_instance_msg_id) = + send_large_webxdc_message(alice, alice_group_id).await?; + alice + .send_webxdc_status_update( + alice_sent_instance_msg_id, + r#"{"payload": 7, "summary":"sum", "document":"doc"}"#, + ) + .await?; + alice.flush_status_updates().await?; + let webxdc_update = alice.pop_sent_msg().await; + + // Bob does not download instance but already receives update + let bob_instance = bob.recv_msg(&pre_message).await; + assert_eq!(bob_instance.download_state, DownloadState::Available); + bob.recv_msg_trash(&webxdc_update).await; + + // Bob downloads instance, updates should be assigned correctly + bob.recv_msg_trash(&post_message).await; + + let bob_instance = bob.get_last_msg().await; + assert_eq!(bob_instance.viewtype, Viewtype::Webxdc); + assert_eq!(bob_instance.download_state, DownloadState::Done); + assert_eq!( + bob.get_webxdc_status_updates(bob_instance.id, StatusUpdateSerial::new(0)) + .await?, + r#"[{"payload":7,"document":"doc","summary":"sum","serial":1,"max_serial":1}]"# + ); + let info = bob_instance.get_webxdc_info(bob).await?; + assert_eq!(info.document, "doc"); + assert_eq!(info.summary, "sum"); + + Ok(()) +} + +/// Test mark seen pre-message +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_markseen_pre_msg() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let bob_chat_id = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + + tcm.section("Bob sends a large message to Alice"); + let (pre_message, post_message, _bob_msg_id) = + send_large_file_message(bob, bob_chat_id, Viewtype::File, &vec![0u8; 1_000_000]).await?; + + tcm.section("Alice receives a pre-message message from Bob"); + let msg = alice.recv_msg(&pre_message).await; + assert_eq!(msg.download_state, DownloadState::Available); + assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default()); + assert_eq!(msg.state, MessageState::InFresh); + + tcm.section("Alice marks the pre-message as read and sends a MDN"); + markseen_msgs(alice, vec![msg.id]).await?; + assert_eq!(msg.id.get_state(alice).await?, MessageState::InSeen); + assert_eq!( + alice + .sql + .count("SELECT COUNT(*) FROM smtp_mdns", ()) + .await?, + 1 + ); + + tcm.section("Alice downloads message"); + alice.recv_msg_trash(&post_message).await; + let msg = Message::load_from_db(alice, msg.id).await?; + assert_eq!(msg.download_state, DownloadState::Done); + assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default()); + assert_eq!( + msg.state, + MessageState::InSeen, + "The message state mustn't be downgraded to `InFresh`" + ); + + Ok(()) +} + +/// Test that pre-message can start a chat +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_pre_msg_can_start_chat() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + + tcm.section("establishing a DM chat between alice and bob"); + let bob_alice_dm_chat_id = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + + tcm.section("Alice prepares chat"); + let chat_id = chat::create_group(alice, "my group").await?; + let contacts = contact::Contact::get_all(alice, 0, None).await?; + let alice_bob_id = contacts.first().expect("contact exists"); + chat::add_contact_to_chat(alice, chat_id, *alice_bob_id).await?; + + tcm.section("Alice sends large message to promote/start chat"); + let (pre_message, _post_message, _alice_msg_id) = + send_large_file_message(alice, chat_id, Viewtype::File, &vec![0u8; 1_000_000]).await?; + + tcm.section("Bob receives the pre-message message from Alice"); + let msg = bob.recv_msg(&pre_message).await; + assert_eq!(msg.download_state, DownloadState::Available); + assert_ne!(msg.chat_id, bob_alice_dm_chat_id); + let chat = chat::Chat::load_from_db(bob, msg.chat_id).await?; + assert_eq!(chat.name, "my group"); + + Ok(()) +} + +/// Test that Post-Message can start a chat +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_post_msg_can_start_chat() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + + tcm.section("establishing a DM chat between alice and bob"); + let bob_alice_dm_chat_id = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + + tcm.section("Alice prepares chat"); + let chat_id = chat::create_group(alice, "my group").await?; + let contacts = contact::Contact::get_all(alice, 0, None).await?; + let alice_bob_id = contacts.first().expect("contact exists"); + chat::add_contact_to_chat(alice, chat_id, *alice_bob_id).await?; + + tcm.section("Alice sends large message to promote/start chat"); + let (_pre_message, post_message, _bob_msg_id) = + send_large_file_message(alice, chat_id, Viewtype::File, &vec![0u8; 1_000_000]).await?; + + tcm.section("Bob receives the pre-message message from Alice"); + let msg = bob.recv_msg(&post_message).await; + assert_eq!(msg.download_state, DownloadState::Done); + assert_ne!(msg.chat_id, bob_alice_dm_chat_id); + let chat = chat::Chat::load_from_db(bob, msg.chat_id).await?; + assert_eq!(chat.name, "my group"); + + Ok(()) +} + +/// Test that message ordering is still correct after downloading +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_download_later_keeps_message_order() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + + tcm.section( + "establishing a DM chat between alice and bob and bob sends large message to alice", + ); + let bob_alice_dm_chat = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + let (pre_message, post_message, _bob_msg_id) = send_large_file_message( + bob, + bob_alice_dm_chat, + Viewtype::File, + &vec![0u8; 1_000_000], + ) + .await?; + + tcm.section("Alice downloads pre-message"); + let msg = alice.recv_msg(&pre_message).await; + assert_eq!(msg.download_state, DownloadState::Available); + assert_eq!(msg.state, MessageState::InFresh); + assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, msg.id); + + tcm.section("Bob sends hi to Alice"); + let hi_msg = tcm.send_recv(bob, alice, "hi").await; + assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, hi_msg.id); + + tcm.section("Alice downloads Post-Message"); + alice.recv_msg_trash(&post_message).await; + let msg = Message::load_from_db(alice, msg.id).await?; + assert_eq!(msg.download_state, DownloadState::Done); + assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, hi_msg.id); + assert!(msg.timestamp_sort <= hi_msg.timestamp_sort); + + Ok(()) +} + +/// Test that ChatlistItemChanged event is emitted when downloading Post-Message +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_chatlist_event_on_post_msg_download() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + + tcm.section( + "establishing a DM chat between alice and bob and bob sends large message to alice", + ); + let bob_alice_dm_chat = bob.create_chat(alice).await.id; + alice.create_chat(bob).await; // Make sure the chat is accepted. + let (pre_message, post_message, _bob_msg_id) = send_large_file_message( + bob, + bob_alice_dm_chat, + Viewtype::File, + &vec![0u8; 1_000_000], + ) + .await?; + + tcm.section("Alice downloads pre-message"); + let msg = alice.recv_msg(&pre_message).await; + assert_eq!(msg.download_state, DownloadState::Available); + assert_eq!(msg.state, MessageState::InFresh); + assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, msg.id); + + tcm.section("Alice downloads Post-Message and waits for ChatlistItemChanged event "); + alice.evtracker.clear_events(); + alice.recv_msg_trash(&post_message).await; + let msg = Message::load_from_db(alice, msg.id).await?; + assert_eq!(msg.download_state, DownloadState::Done); + alice + .evtracker + .get_matching(|e| { + e == &EventType::ChatlistItemChanged { + chat_id: Some(msg.chat_id), + } + }) + .await; + + Ok(()) +} diff --git a/src/tests/pre_messages/sending.rs b/src/tests/pre_messages/sending.rs new file mode 100644 index 0000000000..1d04a4a4d8 --- /dev/null +++ b/src/tests/pre_messages/sending.rs @@ -0,0 +1,337 @@ +//! Tests about sending pre-messages +//! - When to send a pre-message and post-message instead of a normal message +//! - Test that sent pre- and post-message contain the right Headers +//! and that they are send in the correct order (pre-message is sent first.) +use anyhow::Result; +use mailparse::MailHeaderMap; +use tokio::fs; + +use crate::chat::{self, create_group, send_msg}; +use crate::config::Config; +use crate::download::PRE_MSG_ATTACHMENT_SIZE_THRESHOLD; +use crate::headerdef::{HeaderDef, HeaderDefMap}; +use crate::message::{Message, Viewtype}; +use crate::test_utils::{self, TestContext, TestContextManager}; +/// Tests that Pre-Message is sent for attachment larger than `PRE_MSG_ATTACHMENT_SIZE_THRESHOLD` +/// Also test that Pre-Message is sent first, before the Post-Message +/// And that Autocrypt-gossip and selfavatar never go into Post-Messages +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_sending_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let fiona = &tcm.fiona().await; + let group_id = alice + .create_group_with_members("test group", &[bob, fiona]) + .await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?; + msg.set_text("test".to_owned()); + + // assert that test attachment is bigger than limit + assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + + let msg_id = chat::send_msg(alice, group_id, &mut msg).await?; + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + // Pre-Message and Post-Message should be present + // and test that correct headers are present on both messages + assert_eq!(smtp_rows.len(), 2); + let pre_message = smtp_rows.first().expect("first element exists"); + let pre_message_parsed = mailparse::parse_mail(pre_message.payload.as_bytes())?; + let post_message = smtp_rows.get(1).expect("second element exists"); + let post_message_parsed = mailparse::parse_mail(post_message.payload.as_bytes())?; + + assert!( + pre_message_parsed + .headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_none() + ); + assert!( + post_message_parsed + .headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_some() + ); + + assert_eq!( + post_message_parsed + .headers + .get_header_value(HeaderDef::MessageId), + Some(format!("<{}>", msg.rfc724_mid)), + "Post-Message should have the rfc message id of the database message" + ); + + assert_ne!( + pre_message_parsed + .headers + .get_header_value(HeaderDef::MessageId), + post_message_parsed + .headers + .get_header_value(HeaderDef::MessageId), + "message ids of Pre-Message and Post-Message should be different" + ); + + let decrypted_post_message = bob.parse_msg(post_message).await; + assert_eq!(decrypted_post_message.decrypting_failed, false); + assert_eq!( + decrypted_post_message.header_exists(HeaderDef::ChatPostMessageId), + false + ); + + let decrypted_pre_message = bob.parse_msg(pre_message).await; + assert_eq!( + decrypted_pre_message + .get_header(HeaderDef::ChatPostMessageId) + .map(String::from), + post_message_parsed + .headers + .get_header_value(HeaderDef::MessageId) + ); + assert!( + pre_message_parsed + .headers + .get_header_value(HeaderDef::ChatPostMessageId) + .is_none(), + "no Chat-Post-Message-ID header in unprotected headers of Pre-Message" + ); + + Ok(()) +} + +/// Tests that Pre-Message has autocrypt gossip headers and self avatar +/// and Post-Message doesn't have these headers +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_selfavatar_and_autocrypt_gossip_goto_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let fiona = &tcm.fiona().await; + let group_id = alice + .create_group_with_members("test group", &[bob, fiona]) + .await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?; + msg.set_text("test".to_owned()); + + // assert that test attachment is bigger than limit + assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + + // simulate conditions for sending self avatar + let avatar_src = alice.get_blobdir().join("avatar.png"); + fs::write(&avatar_src, test_utils::AVATAR_900x900_BYTES).await?; + alice + .set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap())) + .await?; + + let msg_id = chat::send_msg(alice, group_id, &mut msg).await?; + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 2); + let pre_message = smtp_rows.first().expect("first element exists"); + let post_message = smtp_rows.get(1).expect("second element exists"); + let post_message_parsed = mailparse::parse_mail(post_message.payload.as_bytes())?; + + let decrypted_pre_message = bob.parse_msg(pre_message).await; + assert!( + decrypted_pre_message + .get_header(HeaderDef::ChatPostMessageId) + .is_some(), + "tested message is not a pre-message, sending order may be broken" + ); + assert_ne!(decrypted_pre_message.gossiped_keys.len(), 0); + assert_ne!(decrypted_pre_message.user_avatar, None); + + let decrypted_post_message = bob.parse_msg(post_message).await; + assert!( + post_message_parsed + .headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_some(), + "tested message is not a Post-Message, sending order may be broken" + ); + assert_eq!(decrypted_post_message.gossiped_keys.len(), 0); + assert_eq!(decrypted_post_message.user_avatar, None); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_unecrypted_gets_no_pre_message() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + + let chat = alice + .create_chat_with_contact("example", "email@example.org") + .await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 300_000], None)?; + msg.set_text("test".to_owned()); + + let msg_id = chat::send_msg(alice, chat.id, &mut msg).await?; + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 1); + let message_bytes = smtp_rows + .first() + .expect("first element exists") + .payload + .as_bytes(); + let message = mailparse::parse_mail(message_bytes)?; + assert!( + message + .headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_none(), + ); + Ok(()) +} + +/// Tests that no pre message is sent for normal message +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_not_sending_pre_message_no_attachment() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let chat = alice.create_chat(bob).await; + + // send normal text message + let mut msg = Message::new(Viewtype::Text); + msg.set_text("test".to_owned()); + let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 1, "only one message should be sent"); + + let msg = smtp_rows.first().expect("first element exists"); + let mail = mailparse::parse_mail(msg.payload.as_bytes())?; + + assert!( + mail.headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_none(), + "no 'Chat-Is-Post-Message'-header should be present" + ); + assert!( + mail.headers + .get_first_header(HeaderDef::ChatPostMessageId.get_headername()) + .is_none(), + "no 'Chat-Post-Message-ID'-header should be present in clear text headers" + ); + let decrypted_message = bob.parse_msg(msg).await; + assert!( + !decrypted_message.header_exists(HeaderDef::ChatPostMessageId), + "no 'Chat-Post-Message-ID'-header should be present" + ); + + // test that pre message is not send for large large text + let mut msg = Message::new(Viewtype::Text); + let long_text = String::from_utf8(vec![b'a'; 300_000])?; + assert!(long_text.len() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap()); + msg.set_text(long_text); + let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 1, "only one message should be sent"); + + let msg = smtp_rows.first().expect("first element exists"); + let mail = mailparse::parse_mail(msg.payload.as_bytes())?; + + assert!( + mail.headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_none() + ); + assert!( + mail.headers + .get_first_header(HeaderDef::ChatPostMessageId.get_headername()) + .is_none(), + "no 'Chat-Post-Message-ID'-header should be present in clear text headers" + ); + let decrypted_message = bob.parse_msg(msg).await; + assert!( + !decrypted_message.header_exists(HeaderDef::ChatPostMessageId), + "no 'Chat-Post-Message-ID'-header should be present" + ); + Ok(()) +} + +/// Tests that no pre message is sent for attachment smaller than `PRE_MSG_ATTACHMENT_SIZE_THRESHOLD` +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_not_sending_pre_message_for_small_attachment() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let chat = alice.create_chat(bob).await; + + let mut msg = Message::new(Viewtype::File); + msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 100_000], None)?; + msg.set_text("test".to_owned()); + + // assert that test attachment is smaller than limit + assert!(msg.get_filebytes(alice).await?.unwrap() < PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + + let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap(); + let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await; + + // only one message and no "is Post-Message" header should be present + assert_eq!(smtp_rows.len(), 1); + + let msg = smtp_rows.first().expect("first element exists"); + let mail = mailparse::parse_mail(msg.payload.as_bytes())?; + + assert!( + mail.headers + .get_first_header(HeaderDef::ChatIsPostMessage.get_headername()) + .is_none() + ); + assert!( + mail.headers + .get_first_header(HeaderDef::ChatPostMessageId.get_headername()) + .is_none(), + "no 'Chat-Post-Message-ID'-header should be present in clear text headers" + ); + let decrypted_message = bob.parse_msg(msg).await; + assert!( + !decrypted_message.header_exists(HeaderDef::ChatPostMessageId), + "no 'Chat-Post-Message-ID'-header should be present" + ); + + Ok(()) +} + +/// Tests that pre message is not send for large webxdc updates +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_render_webxdc_status_update_object_range() -> Result<()> { + let t = TestContext::new_alice().await; + let chat_id = create_group(&t, "a chat").await?; + + let instance = { + let mut instance = Message::new(Viewtype::File); + instance.set_file_from_bytes( + &t, + "minimal.xdc", + include_bytes!("../../../test-data/webxdc/minimal.xdc"), + None, + )?; + let instance_msg_id = send_msg(&t, chat_id, &mut instance).await?; + assert_eq!(instance.viewtype, Viewtype::Webxdc); + Message::load_from_db(&t, instance_msg_id).await + } + .unwrap(); + + t.pop_sent_msg().await; + assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 0); + + let long_text = String::from_utf8(vec![b'a'; 300_000])?; + assert!(long_text.len() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap()); + t.send_webxdc_status_update(instance.id, &format!("{{\"payload\": \"{long_text}\"}}")) + .await?; + t.flush_status_updates().await?; + + assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 1); + Ok(()) +} diff --git a/src/tests/pre_messages/util.rs b/src/tests/pre_messages/util.rs new file mode 100644 index 0000000000..0d100fed05 --- /dev/null +++ b/src/tests/pre_messages/util.rs @@ -0,0 +1,65 @@ +use anyhow::Result; +use async_zip::tokio::write::ZipFileWriter; +use async_zip::{Compression, ZipEntryBuilder}; +use futures::io::Cursor as FuturesCursor; +use pretty_assertions::assert_eq; +use tokio_util::compat::FuturesAsyncWriteCompatExt; + +use crate::chat::{self, ChatId}; +use crate::download::PRE_MSG_ATTACHMENT_SIZE_THRESHOLD; +use crate::message::{Message, MsgId, Viewtype}; +use crate::test_utils::{SentMessage, TestContext, create_test_image}; + +pub async fn send_large_file_message<'a>( + sender: &'a TestContext, + target_chat: ChatId, + view_type: Viewtype, + content: &[u8], +) -> Result<(SentMessage<'a>, SentMessage<'a>, MsgId)> { + let mut msg = Message::new(view_type); + let file_name = if view_type == Viewtype::Webxdc { + "test.xdc" + } else { + "test.bin" + }; + msg.set_file_from_bytes(sender, file_name, content, None)?; + msg.set_text("test".to_owned()); + + // assert that test attachment is bigger than limit + assert!(msg.get_filebytes(sender).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD); + + let msg_id = chat::send_msg(sender, target_chat, &mut msg).await?; + let smtp_rows = sender.get_smtp_rows_for_msg(msg_id).await; + + assert_eq!(smtp_rows.len(), 2); + let pre_message = smtp_rows.first().expect("Pre-Message exists"); + let post_message = smtp_rows.get(1).expect("Post-Message exists"); + Ok((pre_message.to_owned(), post_message.to_owned(), msg_id)) +} + +pub async fn send_large_webxdc_message<'a>( + sender: &'a TestContext, + target_chat: ChatId, +) -> Result<(SentMessage<'a>, SentMessage<'a>, MsgId)> { + let futures_cursor = FuturesCursor::new(Vec::new()); + let mut buffer = futures_cursor.compat_write(); + let mut writer = ZipFileWriter::with_tokio(&mut buffer); + writer + .write_entry_whole( + ZipEntryBuilder::new("index.html".into(), Compression::Stored), + &[0u8; 1_000_000], + ) + .await?; + writer.close().await?; + let big_webxdc_app = buffer.into_inner().into_inner(); + send_large_file_message(sender, target_chat, Viewtype::Webxdc, &big_webxdc_app).await +} + +pub async fn send_large_image_message<'a>( + sender: &'a TestContext, + target_chat: ChatId, +) -> Result<(SentMessage<'a>, SentMessage<'a>, MsgId)> { + let (width, height) = (1080, 1920); + let test_img = create_test_image(width, height)?; + send_large_file_message(sender, target_chat, Viewtype::Image, &test_img).await +} diff --git a/src/webxdc/webxdc_tests.rs b/src/webxdc/webxdc_tests.rs index 3a03d606f7..792e5d92c3 100644 --- a/src/webxdc/webxdc_tests.rs +++ b/src/webxdc/webxdc_tests.rs @@ -10,9 +10,8 @@ use crate::chat::{ }; use crate::chatlist::Chatlist; use crate::config::Config; -use crate::download::DownloadState; use crate::ephemeral; -use crate::receive_imf::{receive_imf, receive_imf_from_inbox}; +use crate::receive_imf::receive_imf; use crate::test_utils::{E2EE_INFO_MSGS, TestContext, TestContextManager}; use crate::tools::{self, SystemTime}; use crate::{message, sql}; @@ -329,69 +328,6 @@ async fn test_webxdc_contact_request() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_webxdc_update_for_not_downloaded_instance() -> Result<()> { - // Alice sends a larger instance and an update - let alice = TestContext::new_alice().await; - let bob = TestContext::new_bob().await; - let chat = alice.create_chat(&bob).await; - bob.set_config(Config::DownloadLimit, Some("40000")).await?; - let mut alice_instance = create_webxdc_instance( - &alice, - "chess.xdc", - include_bytes!("../../test-data/webxdc/chess.xdc"), - )?; - let sent1 = alice.send_msg(chat.id, &mut alice_instance).await; - let alice_instance = sent1.load_from_db().await; - alice - .send_webxdc_status_update( - alice_instance.id, - r#"{"payload": 7, "summary":"sum", "document":"doc"}"#, - ) - .await?; - alice.flush_status_updates().await?; - let sent2 = alice.pop_sent_msg().await; - - // Bob does not download instance but already receives update - receive_imf_from_inbox( - &bob, - &alice_instance.rfc724_mid, - sent1.payload().as_bytes(), - false, - Some(70790), - ) - .await?; - let bob_instance = bob.get_last_msg().await; - bob_instance.chat_id.accept(&bob).await?; - bob.recv_msg_trash(&sent2).await; - assert_eq!(bob_instance.download_state, DownloadState::Available); - - // Bob downloads instance, updates should be assigned correctly - let received_msg = receive_imf_from_inbox( - &bob, - &alice_instance.rfc724_mid, - sent1.payload().as_bytes(), - false, - None, - ) - .await? - .unwrap(); - assert_eq!(*received_msg.msg_ids.first().unwrap(), bob_instance.id); - let bob_instance = bob.get_last_msg().await; - assert_eq!(bob_instance.viewtype, Viewtype::Webxdc); - assert_eq!(bob_instance.download_state, DownloadState::Done); - assert_eq!( - bob.get_webxdc_status_updates(bob_instance.id, StatusUpdateSerial(0)) - .await?, - r#"[{"payload":7,"document":"doc","summary":"sum","serial":1,"max_serial":1}]"# - ); - let info = bob_instance.get_webxdc_info(&bob).await?; - assert_eq!(info.document, "doc"); - assert_eq!(info.summary, "sum"); - - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_delete_webxdc_instance() -> Result<()> { let t = TestContext::new_alice().await; From eb489bc0166c47f30afa0ad65594fe1712d6d5f4 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sun, 14 Dec 2025 18:03:10 +0000 Subject: [PATCH 02/11] remove "Contact" stock string again, use emoji like in summaries --- deltachat-ffi/deltachat.h | 5 ----- src/stock_str.rs | 10 +--------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 66399f2abd..5379ec6782 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -7764,11 +7764,6 @@ void dc_event_unref(dc_event_t* event); /// Used as the first info messages in newly created classic email threads. #define DC_STR_CHAT_UNENCRYPTED_EXPLANATON 230 -/// "Contact" -/// -/// Used in summaries. -#define DC_STR_CONTACT 231 - /** * @} */ diff --git a/src/stock_str.rs b/src/stock_str.rs index b9388f8f38..68c65c46f0 100644 --- a/src/stock_str.rs +++ b/src/stock_str.rs @@ -415,9 +415,6 @@ https://delta.chat/donate"))] #[strum(props(fallback = "Messages in this chat use classic email and are not encrypted."))] ChatUnencryptedExplanation = 230, - - #[strum(props(fallback = "Contact"))] - Contact = 231, } impl StockMessage { @@ -885,11 +882,6 @@ pub(crate) async fn sticker(context: &Context) -> String { translated(context, StockMessage::Sticker).await } -/// Stock string: `Contact`. -pub(crate) async fn contact(context: &Context) -> String { - translated(context, StockMessage::Contact).await -} - /// Stock string: `Device messages`. pub(crate) async fn device_messages(context: &Context) -> String { translated(context, StockMessage::DeviceMessages).await @@ -1259,7 +1251,7 @@ impl Viewtype { Viewtype::Video => video(context).await, Viewtype::File => file(context).await, Viewtype::Webxdc => "Mini App".to_owned(), - Viewtype::Vcard => contact(context).await, + Viewtype::Vcard => "👤".to_string(), Viewtype::Unknown | Viewtype::Text | Viewtype::Call => self.to_string(), } } From 78f884f1703e5827d7f44f212cbf4f2bfaabc1b1 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Sun, 14 Dec 2025 20:47:56 +0100 Subject: [PATCH 03/11] fix: Add additional_text also to the summary, in order to prevent empty summaries --- src/summary.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/summary.rs b/src/summary.rs index 90ddd1e35e..f47d100bb9 100644 --- a/src/summary.rs +++ b/src/summary.rs @@ -262,7 +262,7 @@ impl Message { } }; - let text = self.text.clone(); + let text = self.text.clone() + &self.additional_text; let summary = if let Some(type_file) = type_file { if append_text && !text.is_empty() { From 411126fd5be90eb6c28637516424a901562c96ef Mon Sep 17 00:00:00 2001 From: Hocuri Date: Sun, 14 Dec 2025 20:58:51 +0100 Subject: [PATCH 04/11] refactor: Rename download_when_normal_starts->download_later --- src/imap.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/imap.rs b/src/imap.rs index 57549584a3..ec7f9142c8 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -616,7 +616,7 @@ impl Imap { let mut uids_fetch = Vec::::with_capacity(msgs.len() + 1); let mut available_post_msgs = Vec::::with_capacity(msgs.len()); - let mut download_when_normal_starts = Vec::::with_capacity(msgs.len()); + let mut download_later = Vec::::with_capacity(msgs.len()); let mut uid_message_ids = BTreeMap::new(); let mut largest_uid_skipped = None; let delete_target = context.get_delete_msgs_target().await?; @@ -728,7 +728,7 @@ impl Imap { // whether it fits download size limit if download_limit.is_none_or(|download_limit| size < download_limit) { - download_when_normal_starts.push(message_id.clone()); + download_later.push(message_id.clone()); } } else { info!(context, "{message_id:?} is not a post-message."); @@ -812,7 +812,7 @@ impl Imap { context, "available_post_msgs: {}, download_when_normal_starts: {}", available_post_msgs.len(), - download_when_normal_starts.len() + download_later.len() ); for rfc724_mid in available_post_msgs { context @@ -820,7 +820,7 @@ impl Imap { .insert("INSERT INTO available_post_msgs VALUES (?)", (rfc724_mid,)) .await?; } - for rfc724_mid in download_when_normal_starts { + for rfc724_mid in download_later { context .sql .insert( From 96fa64d69b8dc6a3b732ba80ece29da485e3215d Mon Sep 17 00:00:00 2001 From: Hocuri Date: Sun, 14 Dec 2025 21:00:16 +0100 Subject: [PATCH 05/11] refactor: use more idiomatic way of creating Vec, and remove premature optimization --- src/imap.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/imap.rs b/src/imap.rs index ec7f9142c8..e7283b8c93 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -614,9 +614,9 @@ impl Imap { .context("prefetch")?; let read_cnt = msgs.len(); - let mut uids_fetch = Vec::::with_capacity(msgs.len() + 1); - let mut available_post_msgs = Vec::::with_capacity(msgs.len()); - let mut download_later = Vec::::with_capacity(msgs.len()); + let mut uids_fetch: Vec = Vec::new(); + let mut available_post_msgs: Vec = Vec::new(); + let mut download_later: Vec = Vec::new(); let mut uid_message_ids = BTreeMap::new(); let mut largest_uid_skipped = None; let delete_target = context.get_delete_msgs_target().await?; From a8f0a6d4228ec0b6fd55133c45795c7241f2ac4c Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 11 Dec 2025 12:00:30 +0000 Subject: [PATCH 06/11] improve error message --- src/download.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/download.rs b/src/download.rs index fd4655381c..0b0318380e 100644 --- a/src/download.rs +++ b/src/download.rs @@ -151,7 +151,7 @@ pub(crate) async fn download_msg( let Some((server_uid, server_folder)) = row else { // No IMAP record found, we don't know the UID and folder. - return Err(anyhow!("Call download_full() again to try over.")); + return Err(anyhow!("IMAP location for {rfc724_mid:?} post-message is unknown")); }; session From 44586638d8d5dcdb4dd8e747f0b7ef5d9df18c47 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 11 Dec 2025 12:35:11 +0000 Subject: [PATCH 07/11] move download_msgs call --- src/scheduler.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/scheduler.rs b/src/scheduler.rs index caf968fb16..089d3821e9 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -501,9 +501,6 @@ async fn inbox_fetch_idle(ctx: &Context, imap: &mut Imap, mut session: Session) } } - download_msgs(ctx, &mut session) - .await - .context("Failed to download messages")?; session .fetch_metadata(ctx) .await @@ -635,6 +632,10 @@ async fn fetch_idle( .log_err(ctx) .ok(); + download_msgs(ctx, &mut session) + .await + .context("Failed to download messages")?; + connection.connectivity.set_idle(ctx); ctx.emit_event(EventType::ImapInboxIdle); From 60b28cb0dbf3f05ed5cff6e1e3af88ec62ca7882 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 11 Dec 2025 12:46:11 +0000 Subject: [PATCH 08/11] log request to download --- src/download.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/download.rs b/src/download.rs index 0b0318380e..5f9bc2664e 100644 --- a/src/download.rs +++ b/src/download.rs @@ -79,6 +79,7 @@ impl MsgId { } self.update_download_state(context, DownloadState::InProgress) .await?; + info!(context, "Requesting full download of {:?}.", msg.rfc724_mid()); context .sql .execute( From 53dc1c70022babffd4b41b4fc6638737eed9f043 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sun, 14 Dec 2025 20:22:14 +0000 Subject: [PATCH 09/11] rustfmt --- src/download.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/download.rs b/src/download.rs index 5f9bc2664e..2aa655005b 100644 --- a/src/download.rs +++ b/src/download.rs @@ -79,7 +79,11 @@ impl MsgId { } self.update_download_state(context, DownloadState::InProgress) .await?; - info!(context, "Requesting full download of {:?}.", msg.rfc724_mid()); + info!( + context, + "Requesting full download of {:?}.", + msg.rfc724_mid() + ); context .sql .execute( @@ -152,7 +156,9 @@ pub(crate) async fn download_msg( let Some((server_uid, server_folder)) = row else { // No IMAP record found, we don't know the UID and folder. - return Err(anyhow!("IMAP location for {rfc724_mid:?} post-message is unknown")); + return Err(anyhow!( + "IMAP location for {rfc724_mid:?} post-message is unknown" + )); }; session From 73a2c9407f604d905e86e013c7e9d1bd7ffcec77 Mon Sep 17 00:00:00 2001 From: link2xt Date: Mon, 15 Dec 2025 02:17:16 +0000 Subject: [PATCH 10/11] test: add pre-message test --- deltachat-rpc-client/tests/test_something.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index 415c135d22..9f9a40e325 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -10,7 +10,7 @@ import pytest from deltachat_rpc_client import EventType, events -from deltachat_rpc_client.const import MessageState +from deltachat_rpc_client.const import MessageState, DownloadState from deltachat_rpc_client.pytestplugin import E2EE_INFO_MSGS from deltachat_rpc_client.rpc import JsonRpcError @@ -963,3 +963,21 @@ def test_synchronize_member_list_on_group_rejoin(acfactory, log): assert chat.num_contacts() == 2 assert msg.get_snapshot().chat.num_contacts() == 2 + + +def test_large_message(acfactory) -> None: + """ + Test sending large message without download limit set, + so it is sent with pre-message but downloaded without user interaction. + """ + alice, bob = acfactory.get_online_accounts(2) + + alice_chat_bob = alice.create_chat(bob) + alice_chat_bob.send_message( + "Hello World, this message is bigger than 5 bytes", + file="../test-data/image/screenshot.jpg", + ) + + msg = bob.wait_for_incoming_msg() + snapshot = msg.get_snapshot() + assert snapshot.text == "Hello World, this message is bigger than 5 bytes" From 2b3d35b621da9b3531d46138193cf86bf4252283 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Thu, 18 Dec 2025 18:28:26 +0100 Subject: [PATCH 11/11] fix: Properly advance uid_next when skipping a post-message Previously, the uid_next wasn't advanced, which didn't create any problems, but it also was inefficient, because another loop was done trying to fetch the message again (and then finally skipping it, because it's already known). --- src/imap.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/imap.rs b/src/imap.rs index e7283b8c93..7e05bfe4af 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -730,6 +730,7 @@ impl Imap { if download_limit.is_none_or(|download_limit| size < download_limit) { download_later.push(message_id.clone()); } + largest_uid_skipped = Some(uid); } else { info!(context, "{message_id:?} is not a post-message.");