feat: message previews

Part of #7367

- Remove partial downloads (remove creation of the stub messages) (#7373)
- Remove "Download maximum available until" and remove stock string `DC_STR_DOWNLOAD_AVAILABILITY` (#7369)
- Send pre-message on messages with large attachments (#7410)
- Pre messages can now get read receipts (#7433)

progress / what's to do:
- [x] send pre-message
- [x] The message's state must be set to MessageState::OutDelivered only
after both messages are sent. If a read receipt is received, the message
can be OutMdnRcvd or OutPending; let's just do whatever is easiest for
now. Take care not to revert from OutMdnReceived to OutDelivered if we
first receive a read receipt and then deliver the full message.
   - this is already the case:
- `OutDelivered` is set when a message is sent out and has no remaining
send jobs in the smtp table for this message id - so already works since
full message and pre message have same msgId in that table.
- `OutMdnRcvd` is a "virtual" state
(#7367 (comment)),
so going back to `OutDelivered` can't happen anymore
- [x] delimit `ChatFullMessageId` with `<` and `>` like the other
message ids
- [x] add tests
	- [x] test that pre message is sent for attachment larger than X
		- test that correct headers are present on both messages
- also test that Autocrypt-gossip and selfavatar should never go into
full-messages
	- [x] test that no pre message is sent for attachment smaller than X
		- no "is full message" header should be present
- [x] test that pre message is not send for large webxdc update or large
text
- [x] fix test `receive_imf::receive_imf_tests::test_dont_reverify_by_self_on_outgoing_msg`

Author: Simon-Laux

deltachat-ffi/deltachat.h

@@ -7263,22 +7263,9 @@ void dc_event_unref(dc_event_t* event);
 /// `%1$s` will be replaced by the percentage used
 #define DC_STR_QUOTA_EXCEEDING_MSG_BODY   98
 
-/// "%1$s message"
-///
-/// Used as the message body when a message
-/// was not yet downloaded completely
-/// (dc_msg_get_download_state() is e.g. @ref DC_DOWNLOAD_AVAILABLE).
-///
-/// `%1$s` will be replaced by human-readable size (e.g. "1.2 MiB").
+/// @deprecated Deprecated 2025-11-12, this string is no longer needed.
 #define DC_STR_PARTIAL_DOWNLOAD_MSG_BODY  99
 
-/// "Download maximum available until %1$s"
-///
-/// Appended after some separator to @ref DC_STR_PARTIAL_DOWNLOAD_MSG_BODY.
-///
-/// `%1$s` will be replaced by human-readable date and time.
-#define DC_STR_DOWNLOAD_AVAILABILITY      100
-
 /// "Multi Device Synchronization"
 ///
 /// Used in subjects of outgoing sync messages.

deltachat-rpc-client/tests/test_chatlist_events.py

@@ -1,7 +1,5 @@
 from __future__ import annotations
 
-import base64
-import os
 from typing import TYPE_CHECKING
 
 from deltachat_rpc_client import Account, EventType, const
@@ -111,40 +109,6 @@ def test_delivery_status_failed(acfactory: ACFactory) -> None:
     assert failing_message.get_snapshot().state == const.MessageState.OUT_FAILED
 
 
-def test_download_on_demand(acfactory: ACFactory) -> None:
-    """
-    Test if download on demand emits chatlist update events.
-    This is only needed for last message in chat, but finding that out is too expensive, so it's always emitted
-    """
-    alice, bob = acfactory.get_online_accounts(2)
-
-    alice_contact_bob = alice.create_contact(bob, "Bob")
-    alice_chat_bob = alice_contact_bob.create_chat()
-    alice_chat_bob.send_text("hi")
-
-    alice.set_config("download_limit", "1")
-
-    msg = bob.wait_for_incoming_msg()
-    chat_id = msg.get_snapshot().chat_id
-    msg.get_snapshot().chat.accept()
-    bob.get_chat_by_id(chat_id).send_message(
-        "Hello World, this message is bigger than 5 bytes",
-        html=base64.b64encode(os.urandom(300000)).decode("utf-8"),
-    )
-
-    message = alice.wait_for_incoming_msg()
-    snapshot = message.get_snapshot()
-    assert snapshot.download_state == const.DownloadState.AVAILABLE
-
-    alice.clear_all_events()
-
-    snapshot = message.get_snapshot()
-    chat_id = snapshot.chat_id
-    alice._rpc.download_full_message(alice.id, message.id)
-
-    wait_for_chatlist_specific_item(alice, chat_id)
-
-
 def get_multi_account_test_setup(acfactory: ACFactory) -> [Account, Account, Account]:
     alice, bob = acfactory.get_online_accounts(2)
 

deltachat-rpc-client/tests/test_something.py

@@ -5,13 +5,12 @@
 import os
 import socket
 import subprocess
-import time
 from unittest.mock import MagicMock
 
 import pytest
 
-from deltachat_rpc_client import Contact, EventType, Message, events
-from deltachat_rpc_client.const import DownloadState, MessageState
+from deltachat_rpc_client import EventType, events
+from deltachat_rpc_client.const import MessageState
 from deltachat_rpc_client.pytestplugin import E2EE_INFO_MSGS
 from deltachat_rpc_client.rpc import JsonRpcError
 
@@ -336,7 +335,7 @@ def test_receive_imf_failure(acfactory) -> None:
     alice_contact_bob = alice.create_contact(bob, "Bob")
     alice_chat_bob = alice_contact_bob.create_chat()
 
-    bob.set_config("fail_on_receiving_full_msg", "1")
+    bob.set_config("simulate_receive_imf_error", "1")
     alice_chat_bob.send_text("Hello!")
     event = bob.wait_for_event(EventType.MSGS_CHANGED)
     assert event.chat_id == bob.get_device_chat().id
@@ -345,17 +344,16 @@ def test_receive_imf_failure(acfactory) -> None:
     snapshot = message.get_snapshot()
     assert (
         snapshot.text == "❌ Failed to receive a message:"
-        " Condition failed: `!context.get_config_bool(Config::FailOnReceivingFullMsg).await?`."
+        " Condition failed: `!context.get_config_bool(Config::SimulateReceiveImfError).await?`."
         " Please report this bug to delta@merlinux.eu or https://support.delta.chat/."
     )
 
     # The failed message doesn't break the IMAP loop.
-    bob.set_config("fail_on_receiving_full_msg", "0")
+    bob.set_config("simulate_receive_imf_error", "0")
     alice_chat_bob.send_text("Hello again!")
     message = bob.wait_for_incoming_msg()
     snapshot = message.get_snapshot()
     assert snapshot.text == "Hello again!"
-    assert snapshot.download_state == DownloadState.DONE
     assert snapshot.error is None
 
 
@@ -591,94 +589,6 @@ def test_mdn_doesnt_break_autocrypt(acfactory) -> None:
     assert snapshot.show_padlock
 
 
-def test_reaction_to_partially_fetched_msg(acfactory, tmp_path):
-    """See https://github.com/deltachat/deltachat-core-rust/issues/3688 "Partially downloaded
-    messages are received out of order".
-
-    If the Inbox contains X small messages followed by Y large messages followed by Z small
-    messages, Delta Chat first downloaded a batch of X+Z messages, and then a batch of Y messages.
-
-    This bug was discovered by @Simon-Laux while testing reactions PR #3644 and can be reproduced
-    with online test as follows:
-    - Bob enables download limit and goes offline.
-    - Alice sends a large message to Bob and reacts to this message with a thumbs-up.
-    - Bob goes online
-    - Bob first processes a reaction message and throws it away because there is no corresponding
-      message, then processes a partially downloaded message.
-    - As a result, Bob does not see a reaction
-    """
-    download_limit = 300000
-    ac1, ac2 = acfactory.get_online_accounts(2)
-    ac1_addr = ac1.get_config("addr")
-    chat = ac1.create_chat(ac2)
-    ac2.set_config("download_limit", str(download_limit))
-    ac2.stop_io()
-
-    logging.info("sending small+large messages from ac1 to ac2")
-    msgs = []
-    msgs.append(chat.send_text("hi"))
-    path = tmp_path / "large"
-    path.write_bytes(os.urandom(download_limit + 1))
-    msgs.append(chat.send_file(str(path)))
-    for m in msgs:
-        m.wait_until_delivered()
-
-    logging.info("sending a reaction to the large message from ac1 to ac2")
-    # TODO: Find the reason of an occasional message reordering on the server (so that the reaction
-    # has a lower UID than the previous message). W/a is to sleep for some time to let the reaction
-    # have a later INTERNALDATE.
-    time.sleep(1.1)
-    react_str = "\N{THUMBS UP SIGN}"
-    msgs.append(msgs[-1].send_reaction(react_str))
-    msgs[-1].wait_until_delivered()
-
-    ac2.start_io()
-
-    logging.info("wait for ac2 to receive a reaction")
-    msg2 = Message(ac2, ac2.wait_for_reactions_changed().msg_id)
-    assert msg2.get_sender_contact().get_snapshot().address == ac1_addr
-    assert msg2.get_snapshot().download_state == DownloadState.AVAILABLE
-    reactions = msg2.get_reactions()
-    contacts = [Contact(ac2, int(i)) for i in reactions.reactions_by_contact]
-    assert len(contacts) == 1
-    assert contacts[0].get_snapshot().address == ac1_addr
-    assert list(reactions.reactions_by_contact.values())[0] == [react_str]
-
-
-@pytest.mark.parametrize("n_accounts", [3, 2])
-def test_download_limit_chat_assignment(acfactory, tmp_path, n_accounts):
-    download_limit = 300000
-
-    alice, *others = acfactory.get_online_accounts(n_accounts)
-    bob = others[0]
-
-    alice_group = alice.create_group("test group")
-    for account in others:
-        chat = account.create_chat(alice)
-        chat.send_text("Hello Alice!")
-        assert alice.wait_for_incoming_msg().get_snapshot().text == "Hello Alice!"
-
-        contact = alice.create_contact(account)
-        alice_group.add_contact(contact)
-
-    bob.set_config("download_limit", str(download_limit))
-
-    alice_group.send_text("hi")
-    snapshot = bob.wait_for_incoming_msg().get_snapshot()
-    assert snapshot.text == "hi"
-    bob_group = snapshot.chat
-
-    path = tmp_path / "large"
-    path.write_bytes(os.urandom(download_limit + 1))
-
-    for i in range(10):
-        logging.info("Sending message %s", i)
-        alice_group.send_file(str(path))
-        snapshot = bob.wait_for_incoming_msg().get_snapshot()
-        assert snapshot.download_state == DownloadState.AVAILABLE
-        assert snapshot.chat == bob_group
-
-
 def test_markseen_contact_request(acfactory):
     """
     Test that seen status is synchronized for contact request messages

python/tests/test_1_online.py

@@ -1,7 +1,6 @@
 import os
 import queue
 import sys
-import base64
 from datetime import datetime, timezone
 
 import pytest
@@ -222,38 +221,6 @@ def test_webxdc_huge_update(acfactory, data, lp):
     assert update["payload"] == payload
 
 
-def test_webxdc_download_on_demand(acfactory, data, lp):
-    ac1, ac2 = acfactory.get_online_accounts(2)
-    acfactory.introduce_each_other([ac1, ac2])
-    chat = acfactory.get_accepted_chat(ac1, ac2)
-
-    msg1 = Message.new_empty(ac1, "webxdc")
-    msg1.set_text("message1")
-    msg1.set_file(data.get_path("webxdc/minimal.xdc"))
-    msg1 = chat.send_msg(msg1)
-    assert msg1.is_webxdc()
-    assert msg1.filename
-
-    msg2 = ac2._evtracker.wait_next_incoming_message()
-    assert msg2.is_webxdc()
-
-    lp.sec("ac2 sets download limit")
-    ac2.set_config("download_limit", "100")
-    assert msg1.send_status_update({"payload": base64.b64encode(os.urandom(300000))}, "some test data")
-    ac2_update = ac2._evtracker.wait_next_incoming_message()
-    assert ac2_update.download_state == dc.const.DC_DOWNLOAD_AVAILABLE
-    assert not msg2.get_status_updates()
-
-    ac2_update.download_full()
-    ac2._evtracker.get_matching("DC_EVENT_WEBXDC_STATUS_UPDATE")
-    assert msg2.get_status_updates()
-
-    # Get a event notifying that the message disappeared from the chat.
-    msgs_changed_event = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-    assert msgs_changed_event.data1 == msg2.chat.id
-    assert msgs_changed_event.data2 == 0
-
-
 def test_enable_mvbox_move(acfactory, lp):
     (ac1,) = acfactory.get_online_accounts(1)
 

src/calls/calls_tests.rs

@@ -2,7 +2,7 @@ use super::*;
 use crate::chat::forward_msgs;
 use crate::config::Config;
 use crate::constants::DC_CHAT_ID_TRASH;
-use crate::receive_imf::{receive_imf, receive_imf_from_inbox};
+use crate::receive_imf::receive_imf;
 use crate::test_utils::{TestContext, TestContextManager};
 
 struct CallSetup {
@@ -610,65 +610,3 @@ async fn test_end_text_call() -> Result<()> {
 
     Ok(())
 }
-
-/// Tests that partially downloaded "call ended"
-/// messages are not processed.
-#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn test_no_partial_calls() -> Result<()> {
-    let mut tcm = TestContextManager::new();
-    let alice = &tcm.alice().await;
-
-    let seen = false;
-
-    // The messages in the test
-    // have no `Date` on purpose,
-    // so they are treated as new.
-    let received_call = receive_imf(
-        alice,
-        b"From: bob@example.net\n\
-        To: alice@example.org\n\
-        Message-ID: <first@example.net>\n\
-        Chat-Version: 1.0\n\
-        Chat-Content: call\n\
-        Chat-Webrtc-Room: YWFhYWFhYWFhCg==\n\
-        \n\
-        Hello, this is a call\n",
-        seen,
-    )
-    .await?
-    .unwrap();
-    assert_eq!(received_call.msg_ids.len(), 1);
-    let call_msg = Message::load_from_db(alice, received_call.msg_ids[0])
-        .await
-        .unwrap();
-    assert_eq!(call_msg.viewtype, Viewtype::Call);
-    assert_eq!(call_state(alice, call_msg.id).await?, CallState::Alerting);
-
-    let imf_raw = b"From: bob@example.net\n\
-        To: alice@example.org\n\
-        Message-ID: <second@example.net>\n\
-        In-Reply-To: <first@example.net>\n\
-        Chat-Version: 1.0\n\
-        Chat-Content: call-ended\n\
-        \n\
-        Call ended\n";
-    receive_imf_from_inbox(
-        alice,
-        "second@example.net",
-        imf_raw,
-        seen,
-        Some(imf_raw.len().try_into().unwrap()),
-    )
-    .await?;
-
-    // The call is still not ended.
-    assert_eq!(call_state(alice, call_msg.id).await?, CallState::Alerting);
-
-    // Fully downloading the message ends the call.
-    receive_imf_from_inbox(alice, "second@example.net", imf_raw, seen, None)
-        .await
-        .context("Failed to fully download end call message")?;
-    assert_eq!(call_state(alice, call_msg.id).await?, CallState::Missed);
-
-    Ok(())
-}