deltachat/
receive_imf.rs

1//! Internet Message Format reception pipeline.
2
3use std::collections::{BTreeMap, HashSet};
4use std::iter;
5use std::sync::LazyLock;
6
7use anyhow::{Context as _, Result, ensure};
8use data_encoding::BASE32_NOPAD;
9use deltachat_contact_tools::{
10    ContactAddress, addr_cmp, addr_normalize, may_be_valid_addr, sanitize_single_line,
11};
12use iroh_gossip::proto::TopicId;
13use mailparse::SingleInfo;
14use num_traits::FromPrimitive;
15use regex::Regex;
16
17use crate::chat::{
18    self, Chat, ChatId, ChatIdBlocked, ProtectionStatus, remove_from_chat_contacts_table,
19};
20use crate::config::Config;
21use crate::constants::{self, Blocked, Chattype, DC_CHAT_ID_TRASH, EDITED_PREFIX, ShowEmails};
22use crate::contact::{Contact, ContactId, Origin, mark_contact_id_as_verified};
23use crate::context::Context;
24use crate::debug_logging::maybe_set_logging_xdc_inner;
25use crate::download::DownloadState;
26use crate::ephemeral::{Timer as EphemeralTimer, stock_ephemeral_timer_changed};
27use crate::events::EventType;
28use crate::headerdef::{HeaderDef, HeaderDefMap};
29use crate::imap::{GENERATED_PREFIX, markseen_on_imap_table};
30use crate::key::self_fingerprint_opt;
31use crate::key::{DcKey, Fingerprint};
32use crate::log::LogExt;
33use crate::log::{info, warn};
34use crate::logged_debug_assert;
35use crate::message::{
36    self, Message, MessageState, MessengerMessage, MsgId, Viewtype, rfc724_mid_exists,
37};
38use crate::mimeparser::{AvatarAction, GossipedKey, MimeMessage, SystemMessage, parse_message_ids};
39use crate::param::{Param, Params};
40use crate::peer_channels::{add_gossip_peer_from_header, insert_topic_stub};
41use crate::reaction::{Reaction, set_msg_reaction};
42use crate::rusqlite::OptionalExtension;
43use crate::securejoin::{self, handle_securejoin_handshake, observe_securejoin_on_other_device};
44use crate::simplify;
45use crate::stock_str;
46use crate::sync::Sync::*;
47use crate::tools::{self, buf_compress, remove_subject_prefix};
48use crate::{chatlist_events, ensure_and_debug_assert, ensure_and_debug_assert_eq, location};
49use crate::{contact, imap};
50
51/// This is the struct that is returned after receiving one email (aka MIME message).
52///
53/// One email with multiple attachments can end up as multiple chat messages, but they
54/// all have the same chat_id, state and sort_timestamp.
55#[derive(Debug)]
56pub struct ReceivedMsg {
57    /// Chat the message is assigned to.
58    pub chat_id: ChatId,
59
60    /// Received message state.
61    pub state: MessageState,
62
63    /// Whether the message is hidden.
64    pub hidden: bool,
65
66    /// Message timestamp for sorting.
67    pub sort_timestamp: i64,
68
69    /// IDs of inserted rows in messages table.
70    pub msg_ids: Vec<MsgId>,
71
72    /// Whether IMAP messages should be immediately deleted.
73    pub needs_delete_job: bool,
74}
75
76/// Decision on which kind of chat the message
77/// should be assigned in.
78///
79/// This is done before looking up contact IDs
80/// so we know in advance whether to lookup
81/// key-contacts or email address contacts.
82///
83/// Once this decision is made,
84/// it should not be changed so we
85/// don't assign the message to an encrypted
86/// group after looking up key-contacts
87/// or vice versa.
88#[derive(Debug)]
89enum ChatAssignment {
90    /// Trash the message.
91    Trash,
92
93    /// Group chat with a Group ID.
94    ///
95    /// Lookup key-contacts and
96    /// assign to encrypted group.
97    GroupChat { grpid: String },
98
99    /// Mailing list or broadcast channel.
100    ///
101    /// Mailing lists don't have members.
102    /// Broadcast channels have members
103    /// on the sender side,
104    /// but their addresses don't go into
105    /// the `To` field.
106    ///
107    /// In any case, the `To`
108    /// field should be ignored
109    /// and no contact IDs should be looked
110    /// up except the `from_id`
111    /// which may be an email address contact
112    /// or a key-contact.
113    MailingListOrBroadcast,
114
115    /// Group chat without a Group ID.
116    ///
117    /// This is not encrypted.
118    AdHocGroup,
119
120    /// Assign the message to existing chat
121    /// with a known `chat_id`.
122    ExistingChat {
123        /// ID of existing chat
124        /// which the message should be assigned to.
125        chat_id: ChatId,
126
127        /// Whether existing chat is blocked.
128        /// This is loaded together with a chat ID
129        /// reduce the number of database calls.
130        ///
131        /// We may want to unblock the chat
132        /// after adding the message there
133        /// if the chat is currently blocked.
134        chat_id_blocked: Blocked,
135    },
136
137    /// 1:1 chat with a single contact.
138    ///
139    /// The chat may be encrypted or not,
140    /// it does not matter.
141    /// It is not possible to mix
142    /// email address contacts
143    /// with key-contacts in a single 1:1 chat anyway.
144    OneOneChat,
145}
146
147/// Emulates reception of a message from the network.
148///
149/// This method returns errors on a failure to parse the mail or extract Message-ID. It's only used
150/// for tests and REPL tool, not actual message reception pipeline.
151#[cfg(any(test, feature = "internals"))]
152pub async fn receive_imf(
153    context: &Context,
154    imf_raw: &[u8],
155    seen: bool,
156) -> Result<Option<ReceivedMsg>> {
157    let mail = mailparse::parse_mail(imf_raw).context("can't parse mail")?;
158    let rfc724_mid =
159        imap::prefetch_get_message_id(&mail.headers).unwrap_or_else(imap::create_message_id);
160    if let Some(download_limit) = context.download_limit().await? {
161        let download_limit: usize = download_limit.try_into()?;
162        if imf_raw.len() > download_limit {
163            let head = std::str::from_utf8(imf_raw)?
164                .split("\r\n\r\n")
165                .next()
166                .context("No empty line in the message")?;
167            return receive_imf_from_inbox(
168                context,
169                &rfc724_mid,
170                head.as_bytes(),
171                seen,
172                Some(imf_raw.len().try_into()?),
173            )
174            .await;
175        }
176    }
177    receive_imf_from_inbox(context, &rfc724_mid, imf_raw, seen, None).await
178}
179
180/// Emulates reception of a message from "INBOX".
181///
182/// Only used for tests and REPL tool, not actual message reception pipeline.
183#[cfg(any(test, feature = "internals"))]
184pub(crate) async fn receive_imf_from_inbox(
185    context: &Context,
186    rfc724_mid: &str,
187    imf_raw: &[u8],
188    seen: bool,
189    is_partial_download: Option<u32>,
190) -> Result<Option<ReceivedMsg>> {
191    receive_imf_inner(
192        context,
193        "INBOX",
194        0,
195        0,
196        rfc724_mid,
197        imf_raw,
198        seen,
199        is_partial_download,
200    )
201    .await
202}
203
204/// Inserts a tombstone into `msgs` table
205/// to prevent downloading the same message in the future.
206///
207/// Returns tombstone database row ID.
208async fn insert_tombstone(context: &Context, rfc724_mid: &str) -> Result<MsgId> {
209    let row_id = context
210        .sql
211        .insert(
212            "INSERT INTO msgs(rfc724_mid, chat_id) VALUES (?,?)",
213            (rfc724_mid, DC_CHAT_ID_TRASH),
214        )
215        .await?;
216    let msg_id = MsgId::new(u32::try_from(row_id)?);
217    Ok(msg_id)
218}
219
220async fn get_to_and_past_contact_ids(
221    context: &Context,
222    mime_parser: &MimeMessage,
223    chat_assignment: &ChatAssignment,
224    is_partial_download: Option<u32>,
225    parent_message: &Option<Message>,
226    incoming_origin: Origin,
227) -> Result<(Vec<Option<ContactId>>, Vec<Option<ContactId>>)> {
228    // `None` means that the chat is encrypted,
229    // but we were not able to convert the address
230    // to key-contact, e.g.
231    // because there was no corresponding
232    // Autocrypt-Gossip header.
233    //
234    // This way we still preserve remaining
235    // number of contacts and their positions
236    // so we can match the contacts to
237    // e.g. Chat-Group-Member-Timestamps
238    // header.
239    let to_ids: Vec<Option<ContactId>>;
240    let past_ids: Vec<Option<ContactId>>;
241
242    // ID of the chat to look up the addresses in.
243    //
244    // Note that this is not necessarily the chat we want to assign the message to.
245    // In case of an outgoing private reply to a group message we may
246    // lookup the address of receipient in the list of addresses used in the group,
247    // but want to assign the message to 1:1 chat.
248    let chat_id = match chat_assignment {
249        ChatAssignment::Trash => None,
250        ChatAssignment::GroupChat { grpid } => {
251            if let Some((chat_id, _protected, _blocked)) =
252                chat::get_chat_id_by_grpid(context, grpid).await?
253            {
254                Some(chat_id)
255            } else {
256                None
257            }
258        }
259        ChatAssignment::AdHocGroup => {
260            // If we are going to assign a message to ad hoc group,
261            // we can just convert the email addresses
262            // to e-mail address contacts and don't need a `ChatId`
263            // to lookup key-contacts.
264            None
265        }
266        ChatAssignment::ExistingChat { chat_id, .. } => Some(*chat_id),
267        ChatAssignment::MailingListOrBroadcast => None,
268        ChatAssignment::OneOneChat => {
269            if is_partial_download.is_none() && !mime_parser.incoming {
270                parent_message.as_ref().map(|m| m.chat_id)
271            } else {
272                None
273            }
274        }
275    };
276
277    let member_fingerprints = mime_parser.chat_group_member_fingerprints();
278    let to_member_fingerprints;
279    let past_member_fingerprints;
280
281    if !member_fingerprints.is_empty() {
282        if member_fingerprints.len() >= mime_parser.recipients.len() {
283            (to_member_fingerprints, past_member_fingerprints) =
284                member_fingerprints.split_at(mime_parser.recipients.len());
285        } else {
286            warn!(
287                context,
288                "Unexpected length of the fingerprint header, expected at least {}, got {}.",
289                mime_parser.recipients.len(),
290                member_fingerprints.len()
291            );
292            to_member_fingerprints = &[];
293            past_member_fingerprints = &[];
294        }
295    } else {
296        to_member_fingerprints = &[];
297        past_member_fingerprints = &[];
298    }
299
300    match chat_assignment {
301        ChatAssignment::GroupChat { .. } => {
302            to_ids = add_or_lookup_key_contacts(
303                context,
304                &mime_parser.recipients,
305                &mime_parser.gossiped_keys,
306                to_member_fingerprints,
307                Origin::Hidden,
308            )
309            .await?;
310
311            if let Some(chat_id) = chat_id {
312                past_ids = lookup_key_contacts_by_address_list(
313                    context,
314                    &mime_parser.past_members,
315                    past_member_fingerprints,
316                    Some(chat_id),
317                )
318                .await?;
319            } else {
320                past_ids = add_or_lookup_key_contacts(
321                    context,
322                    &mime_parser.past_members,
323                    &mime_parser.gossiped_keys,
324                    past_member_fingerprints,
325                    Origin::Hidden,
326                )
327                .await?;
328            }
329        }
330        ChatAssignment::Trash | ChatAssignment::MailingListOrBroadcast => {
331            to_ids = Vec::new();
332            past_ids = Vec::new();
333        }
334        ChatAssignment::ExistingChat { chat_id, .. } => {
335            let chat = Chat::load_from_db(context, *chat_id).await?;
336            if chat.is_encrypted(context).await? {
337                to_ids = add_or_lookup_key_contacts(
338                    context,
339                    &mime_parser.recipients,
340                    &mime_parser.gossiped_keys,
341                    to_member_fingerprints,
342                    Origin::Hidden,
343                )
344                .await?;
345                past_ids = lookup_key_contacts_by_address_list(
346                    context,
347                    &mime_parser.past_members,
348                    past_member_fingerprints,
349                    Some(*chat_id),
350                )
351                .await?;
352            } else {
353                to_ids = add_or_lookup_contacts_by_address_list(
354                    context,
355                    &mime_parser.recipients,
356                    if !mime_parser.incoming {
357                        Origin::OutgoingTo
358                    } else if incoming_origin.is_known() {
359                        Origin::IncomingTo
360                    } else {
361                        Origin::IncomingUnknownTo
362                    },
363                )
364                .await?;
365
366                past_ids = add_or_lookup_contacts_by_address_list(
367                    context,
368                    &mime_parser.past_members,
369                    Origin::Hidden,
370                )
371                .await?;
372            }
373        }
374        ChatAssignment::AdHocGroup => {
375            to_ids = add_or_lookup_contacts_by_address_list(
376                context,
377                &mime_parser.recipients,
378                if !mime_parser.incoming {
379                    Origin::OutgoingTo
380                } else if incoming_origin.is_known() {
381                    Origin::IncomingTo
382                } else {
383                    Origin::IncomingUnknownTo
384                },
385            )
386            .await?;
387
388            past_ids = add_or_lookup_contacts_by_address_list(
389                context,
390                &mime_parser.past_members,
391                Origin::Hidden,
392            )
393            .await?;
394        }
395        ChatAssignment::OneOneChat => {
396            let pgp_to_ids = add_or_lookup_key_contacts(
397                context,
398                &mime_parser.recipients,
399                &mime_parser.gossiped_keys,
400                to_member_fingerprints,
401                Origin::Hidden,
402            )
403            .await?;
404            if pgp_to_ids
405                .first()
406                .is_some_and(|contact_id| contact_id.is_some())
407            {
408                // There is a single recipient and we have
409                // mapped it to a key contact.
410                // This is an encrypted 1:1 chat.
411                to_ids = pgp_to_ids
412            } else if let Some(chat_id) = chat_id {
413                to_ids = match mime_parser.was_encrypted() {
414                    true => {
415                        lookup_key_contacts_by_address_list(
416                            context,
417                            &mime_parser.recipients,
418                            to_member_fingerprints,
419                            Some(chat_id),
420                        )
421                        .await?
422                    }
423                    false => {
424                        add_or_lookup_contacts_by_address_list(
425                            context,
426                            &mime_parser.recipients,
427                            if !mime_parser.incoming {
428                                Origin::OutgoingTo
429                            } else if incoming_origin.is_known() {
430                                Origin::IncomingTo
431                            } else {
432                                Origin::IncomingUnknownTo
433                            },
434                        )
435                        .await?
436                    }
437                }
438            } else {
439                let ids = match mime_parser.was_encrypted() {
440                    true => {
441                        lookup_key_contacts_by_address_list(
442                            context,
443                            &mime_parser.recipients,
444                            to_member_fingerprints,
445                            None,
446                        )
447                        .await?
448                    }
449                    false => vec![],
450                };
451                if mime_parser.was_encrypted() && !ids.contains(&None)
452                // Prefer creating PGP chats if there are any key-contacts. At least this prevents
453                // from replying unencrypted.
454                || ids
455                    .iter()
456                    .any(|&c| c.is_some() && c != Some(ContactId::SELF))
457                {
458                    to_ids = ids;
459                } else {
460                    to_ids = add_or_lookup_contacts_by_address_list(
461                        context,
462                        &mime_parser.recipients,
463                        if !mime_parser.incoming {
464                            Origin::OutgoingTo
465                        } else if incoming_origin.is_known() {
466                            Origin::IncomingTo
467                        } else {
468                            Origin::IncomingUnknownTo
469                        },
470                    )
471                    .await?;
472                }
473            }
474
475            past_ids = add_or_lookup_contacts_by_address_list(
476                context,
477                &mime_parser.past_members,
478                Origin::Hidden,
479            )
480            .await?;
481        }
482    };
483
484    Ok((to_ids, past_ids))
485}
486
487/// Receive a message and add it to the database.
488///
489/// Returns an error on database failure or if the message is broken,
490/// e.g. has nonstandard MIME structure.
491///
492/// If possible, creates a database entry to prevent the message from being
493/// downloaded again, sets `chat_id=DC_CHAT_ID_TRASH` and returns `Ok(Some(…))`.
494/// If the message is so wrong that we didn't even create a database entry,
495/// returns `Ok(None)`.
496///
497/// If `is_partial_download` is set, it contains the full message size in bytes.
498/// Do not confuse that with `replace_msg_id` that will be set when the full message is loaded
499/// later.
500#[expect(clippy::too_many_arguments)]
501pub(crate) async fn receive_imf_inner(
502    context: &Context,
503    folder: &str,
504    uidvalidity: u32,
505    uid: u32,
506    rfc724_mid: &str,
507    imf_raw: &[u8],
508    seen: bool,
509    is_partial_download: Option<u32>,
510) -> Result<Option<ReceivedMsg>> {
511    if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
512        info!(
513            context,
514            "receive_imf: incoming message mime-body:\n{}",
515            String::from_utf8_lossy(imf_raw),
516        );
517    }
518
519    let mut mime_parser = match MimeMessage::from_bytes(context, imf_raw, is_partial_download).await
520    {
521        Err(err) => {
522            warn!(context, "receive_imf: can't parse MIME: {err:#}.");
523            if rfc724_mid.starts_with(GENERATED_PREFIX) {
524                // We don't have an rfc724_mid, there's no point in adding a trash entry
525                return Ok(None);
526            }
527
528            let msg_ids = vec![insert_tombstone(context, rfc724_mid).await?];
529
530            return Ok(Some(ReceivedMsg {
531                chat_id: DC_CHAT_ID_TRASH,
532                state: MessageState::Undefined,
533                hidden: false,
534                sort_timestamp: 0,
535                msg_ids,
536                needs_delete_job: false,
537            }));
538        }
539        Ok(mime_parser) => mime_parser,
540    };
541
542    let rfc724_mid_orig = &mime_parser
543        .get_rfc724_mid()
544        .unwrap_or(rfc724_mid.to_string());
545    info!(
546        context,
547        "Receiving message {rfc724_mid_orig:?}, seen={seen}...",
548    );
549
550    // check, if the mail is already in our database.
551    // make sure, this check is done eg. before securejoin-processing.
552    let (replace_msg_id, replace_chat_id);
553    if let Some((old_msg_id, _)) = message::rfc724_mid_exists(context, rfc724_mid).await? {
554        if is_partial_download.is_some() {
555            // Should never happen, see imap::prefetch_should_download(), but still.
556            info!(
557                context,
558                "Got a partial download and message is already in DB."
559            );
560            return Ok(None);
561        }
562        let msg = Message::load_from_db(context, old_msg_id).await?;
563        replace_msg_id = Some(old_msg_id);
564        replace_chat_id = if msg.download_state() != DownloadState::Done {
565            // the message was partially downloaded before and is fully downloaded now.
566            info!(
567                context,
568                "Message already partly in DB, replacing by full message."
569            );
570            Some(msg.chat_id)
571        } else {
572            None
573        };
574    } else {
575        replace_msg_id = if rfc724_mid_orig == rfc724_mid {
576            None
577        } else if let Some((old_msg_id, old_ts_sent)) =
578            message::rfc724_mid_exists(context, rfc724_mid_orig).await?
579        {
580            if imap::is_dup_msg(
581                mime_parser.has_chat_version(),
582                mime_parser.timestamp_sent,
583                old_ts_sent,
584            ) {
585                info!(context, "Deleting duplicate message {rfc724_mid_orig}.");
586                let target = context.get_delete_msgs_target().await?;
587                context
588                    .sql
589                    .execute(
590                        "UPDATE imap SET target=? WHERE folder=? AND uidvalidity=? AND uid=?",
591                        (target, folder, uidvalidity, uid),
592                    )
593                    .await?;
594            }
595            Some(old_msg_id)
596        } else {
597            None
598        };
599        replace_chat_id = None;
600    }
601
602    if replace_chat_id.is_some() {
603        // Need to update chat id in the db.
604    } else if let Some(msg_id) = replace_msg_id {
605        info!(context, "Message is already downloaded.");
606        if mime_parser.incoming {
607            return Ok(None);
608        }
609        // For the case if we missed a successful SMTP response. Be optimistic that the message is
610        // delivered also.
611        let self_addr = context.get_primary_self_addr().await?;
612        context
613            .sql
614            .execute(
615                "DELETE FROM smtp \
616                WHERE rfc724_mid=?1 AND (recipients LIKE ?2 OR recipients LIKE ('% ' || ?2))",
617                (rfc724_mid_orig, &self_addr),
618            )
619            .await?;
620        if !context
621            .sql
622            .exists(
623                "SELECT COUNT(*) FROM smtp WHERE rfc724_mid=?",
624                (rfc724_mid_orig,),
625            )
626            .await?
627        {
628            msg_id.set_delivered(context).await?;
629        }
630        return Ok(None);
631    };
632
633    let prevent_rename = should_prevent_rename(&mime_parser);
634
635    // get From: (it can be an address list!) and check if it is known (for known From:'s we add
636    // the other To:/Cc: in the 3rd pass)
637    // or if From: is equal to SELF (in this case, it is any outgoing messages,
638    // we do not check Return-Path any more as this is unreliable, see
639    // <https://github.com/deltachat/deltachat-core/issues/150>)
640    //
641    // If this is a mailing list email (i.e. list_id_header is some), don't change the displayname because in
642    // a mailing list the sender displayname sometimes does not belong to the sender email address.
643    // For example, GitHub sends messages from `notifications@github.com`,
644    // but uses display name of the user whose action generated the notification
645    // as the display name.
646    let fingerprint = mime_parser.signatures.iter().next();
647    let (from_id, _from_id_blocked, incoming_origin) = match from_field_to_contact_id(
648        context,
649        &mime_parser.from,
650        fingerprint,
651        prevent_rename,
652        is_partial_download.is_some()
653            && mime_parser
654                .get_header(HeaderDef::ContentType)
655                .unwrap_or_default()
656                .starts_with("multipart/encrypted"),
657    )
658    .await?
659    {
660        Some(contact_id_res) => contact_id_res,
661        None => {
662            warn!(
663                context,
664                "receive_imf: From field does not contain an acceptable address."
665            );
666            return Ok(None);
667        }
668    };
669
670    // Lookup parent message.
671    //
672    // This may be useful to assign the message to
673    // group chats without Chat-Group-ID
674    // when a message is sent by Thunderbird.
675    //
676    // This can be also used to lookup
677    // key-contact by email address
678    // when receiving a private 1:1 reply
679    // to a group chat message.
680    let parent_message = get_parent_message(
681        context,
682        mime_parser.get_header(HeaderDef::References),
683        mime_parser.get_header(HeaderDef::InReplyTo),
684    )
685    .await?
686    .filter(|p| Some(p.id) != replace_msg_id);
687
688    let chat_assignment = decide_chat_assignment(
689        context,
690        &mime_parser,
691        &parent_message,
692        rfc724_mid,
693        from_id,
694        &is_partial_download,
695    )
696    .await?;
697    info!(context, "Chat assignment is {chat_assignment:?}.");
698
699    let (to_ids, past_ids) = get_to_and_past_contact_ids(
700        context,
701        &mime_parser,
702        &chat_assignment,
703        is_partial_download,
704        &parent_message,
705        incoming_origin,
706    )
707    .await?;
708
709    let received_msg;
710    if mime_parser.get_header(HeaderDef::SecureJoin).is_some() {
711        let res = if mime_parser.incoming {
712            handle_securejoin_handshake(context, &mut mime_parser, from_id)
713                .await
714                .context("error in Secure-Join message handling")?
715        } else {
716            let to_id = to_ids.first().copied().flatten().unwrap_or(ContactId::SELF);
717            // handshake may mark contacts as verified and must be processed before chats are created
718            observe_securejoin_on_other_device(context, &mime_parser, to_id)
719                .await
720                .context("error in Secure-Join watching")?
721        };
722
723        match res {
724            securejoin::HandshakeMessage::Done | securejoin::HandshakeMessage::Ignore => {
725                let msg_id = insert_tombstone(context, rfc724_mid).await?;
726                received_msg = Some(ReceivedMsg {
727                    chat_id: DC_CHAT_ID_TRASH,
728                    state: MessageState::InSeen,
729                    hidden: false,
730                    sort_timestamp: mime_parser.timestamp_sent,
731                    msg_ids: vec![msg_id],
732                    needs_delete_job: res == securejoin::HandshakeMessage::Done,
733                });
734            }
735            securejoin::HandshakeMessage::Propagate => {
736                received_msg = None;
737            }
738        }
739    } else {
740        received_msg = None;
741    }
742
743    let verified_encryption = has_verified_encryption(context, &mime_parser, from_id).await?;
744
745    if verified_encryption == VerifiedEncryption::Verified {
746        mark_recipients_as_verified(context, from_id, &to_ids, &mime_parser).await?;
747    }
748
749    let received_msg = if let Some(received_msg) = received_msg {
750        received_msg
751    } else {
752        let is_dc_message = if mime_parser.has_chat_version() {
753            MessengerMessage::Yes
754        } else if let Some(parent_message) = &parent_message {
755            match parent_message.is_dc_message {
756                MessengerMessage::No => MessengerMessage::No,
757                MessengerMessage::Yes | MessengerMessage::Reply => MessengerMessage::Reply,
758            }
759        } else {
760            MessengerMessage::No
761        };
762
763        let show_emails = ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await?)
764            .unwrap_or_default();
765
766        let allow_creation = if mime_parser.decrypting_failed {
767            false
768        } else if mime_parser.is_system_message != SystemMessage::AutocryptSetupMessage
769            && is_dc_message == MessengerMessage::No
770            && !context.get_config_bool(Config::IsChatmail).await?
771        {
772            // the message is a classic email in a classic profile
773            // (in chatmail profiles, we always show all messages, because shared dc-mua usage is not supported)
774            match show_emails {
775                ShowEmails::Off | ShowEmails::AcceptedContacts => false,
776                ShowEmails::All => true,
777            }
778        } else {
779            !mime_parser.parts.iter().all(|part| part.is_reaction)
780        };
781
782        let to_id = if mime_parser.incoming {
783            ContactId::SELF
784        } else {
785            to_ids.first().copied().flatten().unwrap_or(ContactId::SELF)
786        };
787
788        let (chat_id, chat_id_blocked) = do_chat_assignment(
789            context,
790            chat_assignment,
791            from_id,
792            &to_ids,
793            &past_ids,
794            to_id,
795            allow_creation,
796            &mut mime_parser,
797            is_partial_download,
798            &verified_encryption,
799            parent_message,
800        )
801        .await?;
802
803        // Add parts
804        add_parts(
805            context,
806            &mut mime_parser,
807            imf_raw,
808            &to_ids,
809            &past_ids,
810            rfc724_mid_orig,
811            from_id,
812            seen,
813            is_partial_download,
814            replace_msg_id,
815            prevent_rename,
816            verified_encryption,
817            chat_id,
818            chat_id_blocked,
819            is_dc_message,
820        )
821        .await
822        .context("add_parts error")?
823    };
824
825    if !from_id.is_special() {
826        contact::update_last_seen(context, from_id, mime_parser.timestamp_sent).await?;
827    }
828
829    // Update gossiped timestamp for the chat if someone else or our other device sent
830    // Autocrypt-Gossip header to avoid sending Autocrypt-Gossip ourselves
831    // and waste traffic.
832    let chat_id = received_msg.chat_id;
833    if !chat_id.is_special() {
834        for gossiped_key in mime_parser.gossiped_keys.values() {
835            context
836                .sql
837                .transaction(move |transaction| {
838                    let fingerprint = gossiped_key.public_key.dc_fingerprint().hex();
839                    transaction.execute(
840                        "INSERT INTO gossip_timestamp (chat_id, fingerprint, timestamp)
841                         VALUES                       (?, ?, ?)
842                         ON CONFLICT                  (chat_id, fingerprint)
843                         DO UPDATE SET timestamp=MAX(timestamp, excluded.timestamp)",
844                        (chat_id, &fingerprint, mime_parser.timestamp_sent),
845                    )?;
846
847                    Ok(())
848                })
849                .await?;
850        }
851    }
852
853    let insert_msg_id = if let Some(msg_id) = received_msg.msg_ids.last() {
854        *msg_id
855    } else {
856        MsgId::new_unset()
857    };
858
859    save_locations(context, &mime_parser, chat_id, from_id, insert_msg_id).await?;
860
861    if let Some(ref sync_items) = mime_parser.sync_items {
862        if from_id == ContactId::SELF {
863            if mime_parser.was_encrypted() {
864                context.execute_sync_items(sync_items).await;
865            } else {
866                warn!(context, "Sync items are not encrypted.");
867            }
868        } else {
869            warn!(context, "Sync items not sent by self.");
870        }
871    }
872
873    if let Some(ref status_update) = mime_parser.webxdc_status_update {
874        let can_info_msg;
875        let instance = if mime_parser
876            .parts
877            .first()
878            .filter(|part| part.typ == Viewtype::Webxdc)
879            .is_some()
880        {
881            can_info_msg = false;
882            Some(Message::load_from_db(context, insert_msg_id).await?)
883        } else if let Some(field) = mime_parser.get_header(HeaderDef::InReplyTo) {
884            if let Some(instance) =
885                message::get_by_rfc724_mids(context, &parse_message_ids(field)).await?
886            {
887                can_info_msg = instance.download_state() == DownloadState::Done;
888                Some(instance)
889            } else {
890                can_info_msg = false;
891                None
892            }
893        } else {
894            can_info_msg = false;
895            None
896        };
897
898        if let Some(instance) = instance {
899            if let Err(err) = context
900                .receive_status_update(
901                    from_id,
902                    &instance,
903                    received_msg.sort_timestamp,
904                    can_info_msg,
905                    status_update,
906                )
907                .await
908            {
909                warn!(context, "receive_imf cannot update status: {err:#}.");
910            }
911        } else {
912            warn!(
913                context,
914                "Received webxdc update, but cannot assign it to message."
915            );
916        }
917    }
918
919    if let Some(avatar_action) = &mime_parser.user_avatar {
920        if from_id != ContactId::UNDEFINED
921            && context
922                .update_contacts_timestamp(
923                    from_id,
924                    Param::AvatarTimestamp,
925                    mime_parser.timestamp_sent,
926                )
927                .await?
928        {
929            if let Err(err) = contact::set_profile_image(
930                context,
931                from_id,
932                avatar_action,
933                mime_parser.was_encrypted(),
934            )
935            .await
936            {
937                warn!(context, "receive_imf cannot update profile image: {err:#}.");
938            };
939        }
940    }
941
942    // Ignore footers from mailinglists as they are often created or modified by the mailinglist software.
943    if let Some(footer) = &mime_parser.footer {
944        if !mime_parser.is_mailinglist_message()
945            && from_id != ContactId::UNDEFINED
946            && context
947                .update_contacts_timestamp(
948                    from_id,
949                    Param::StatusTimestamp,
950                    mime_parser.timestamp_sent,
951                )
952                .await?
953        {
954            if let Err(err) = contact::set_status(
955                context,
956                from_id,
957                footer.to_string(),
958                mime_parser.was_encrypted(),
959                mime_parser.has_chat_version(),
960            )
961            .await
962            {
963                warn!(context, "Cannot update contact status: {err:#}.");
964            }
965        }
966    }
967
968    // Get user-configured server deletion
969    let delete_server_after = context.get_config_delete_server_after().await?;
970
971    if !received_msg.msg_ids.is_empty() {
972        let target = if received_msg.needs_delete_job
973            || (delete_server_after == Some(0) && is_partial_download.is_none())
974        {
975            Some(context.get_delete_msgs_target().await?)
976        } else {
977            None
978        };
979        if target.is_some() || rfc724_mid_orig != rfc724_mid {
980            let target_subst = match &target {
981                Some(_) => "target=?1,",
982                None => "",
983            };
984            context
985                .sql
986                .execute(
987                    &format!("UPDATE imap SET {target_subst} rfc724_mid=?2 WHERE rfc724_mid=?3"),
988                    (
989                        target.as_deref().unwrap_or_default(),
990                        rfc724_mid_orig,
991                        rfc724_mid,
992                    ),
993                )
994                .await?;
995        }
996        if target.is_none() && !mime_parser.mdn_reports.is_empty() && mime_parser.has_chat_version()
997        {
998            // This is a Delta Chat MDN. Mark as read.
999            markseen_on_imap_table(context, rfc724_mid_orig).await?;
1000        }
1001    }
1002
1003    if mime_parser.is_call() {
1004        context
1005            .handle_call_msg(insert_msg_id, &mime_parser, from_id)
1006            .await?;
1007    } else if received_msg.hidden {
1008        // No need to emit an event about the changed message
1009    } else if let Some(replace_chat_id) = replace_chat_id {
1010        context.emit_msgs_changed_without_msg_id(replace_chat_id);
1011    } else if !chat_id.is_trash() {
1012        let fresh = received_msg.state == MessageState::InFresh
1013            && mime_parser.is_system_message != SystemMessage::CallAccepted
1014            && mime_parser.is_system_message != SystemMessage::CallEnded;
1015        for msg_id in &received_msg.msg_ids {
1016            chat_id.emit_msg_event(context, *msg_id, mime_parser.incoming && fresh);
1017        }
1018    }
1019    context.new_msgs_notify.notify_one();
1020
1021    mime_parser
1022        .handle_reports(context, from_id, &mime_parser.parts)
1023        .await;
1024
1025    if let Some(is_bot) = mime_parser.is_bot {
1026        // If the message is auto-generated and was generated by Delta Chat,
1027        // mark the contact as a bot.
1028        if mime_parser.get_header(HeaderDef::ChatVersion).is_some() {
1029            from_id.mark_bot(context, is_bot).await?;
1030        }
1031    }
1032
1033    Ok(Some(received_msg))
1034}
1035
1036/// Converts "From" field to contact id.
1037///
1038/// Also returns whether it is blocked or not and its origin.
1039///
1040/// * `prevent_rename`: if true, the display_name of this contact will not be changed. Useful for
1041///   mailing lists: In some mailing lists, many users write from the same address but with different
1042///   display names. We don't want the display name to change every time the user gets a new email from
1043///   a mailing list.
1044///
1045/// * `find_key_contact_by_addr`: if true, we only know the e-mail address
1046///   of the contact, but not the fingerprint,
1047///   yet want to assign the message to some key-contact.
1048///   This can happen during prefetch or when the message is partially downloaded.
1049///   If we get it wrong, the message will be placed into the correct
1050///   chat after downloading.
1051///
1052/// Returns `None` if From field does not contain a valid contact address.
1053pub async fn from_field_to_contact_id(
1054    context: &Context,
1055    from: &SingleInfo,
1056    fingerprint: Option<&Fingerprint>,
1057    prevent_rename: bool,
1058    find_key_contact_by_addr: bool,
1059) -> Result<Option<(ContactId, bool, Origin)>> {
1060    let fingerprint = fingerprint.as_ref().map(|fp| fp.hex()).unwrap_or_default();
1061    let display_name = if prevent_rename {
1062        Some("")
1063    } else {
1064        from.display_name.as_deref()
1065    };
1066    let from_addr = match ContactAddress::new(&from.addr) {
1067        Ok(from_addr) => from_addr,
1068        Err(err) => {
1069            warn!(
1070                context,
1071                "Cannot create a contact for the given From field: {err:#}."
1072            );
1073            return Ok(None);
1074        }
1075    };
1076
1077    if fingerprint.is_empty() && find_key_contact_by_addr {
1078        let addr_normalized = addr_normalize(&from_addr);
1079
1080        // Try to assign to some key-contact.
1081        if let Some((from_id, origin)) = context
1082            .sql
1083            .query_row_optional(
1084                "SELECT id, origin FROM contacts
1085                 WHERE addr=?1 COLLATE NOCASE
1086                 AND fingerprint<>'' -- Only key-contacts
1087                 AND id>?2 AND origin>=?3 AND blocked=?4
1088                 ORDER BY last_seen DESC
1089                 LIMIT 1",
1090                (
1091                    &addr_normalized,
1092                    ContactId::LAST_SPECIAL,
1093                    Origin::IncomingUnknownFrom,
1094                    Blocked::Not,
1095                ),
1096                |row| {
1097                    let id: ContactId = row.get(0)?;
1098                    let origin: Origin = row.get(1)?;
1099                    Ok((id, origin))
1100                },
1101            )
1102            .await?
1103        {
1104            return Ok(Some((from_id, false, origin)));
1105        }
1106    }
1107
1108    let (from_id, _) = Contact::add_or_lookup_ex(
1109        context,
1110        display_name.unwrap_or_default(),
1111        &from_addr,
1112        &fingerprint,
1113        Origin::IncomingUnknownFrom,
1114    )
1115    .await?;
1116
1117    if from_id == ContactId::SELF {
1118        Ok(Some((ContactId::SELF, false, Origin::OutgoingBcc)))
1119    } else {
1120        let contact = Contact::get_by_id(context, from_id).await?;
1121        let from_id_blocked = contact.blocked;
1122        let incoming_origin = contact.origin;
1123
1124        context
1125            .sql
1126            .execute(
1127                "UPDATE contacts SET addr=? WHERE id=?",
1128                (from_addr, from_id),
1129            )
1130            .await?;
1131
1132        Ok(Some((from_id, from_id_blocked, incoming_origin)))
1133    }
1134}
1135
1136async fn decide_chat_assignment(
1137    context: &Context,
1138    mime_parser: &MimeMessage,
1139    parent_message: &Option<Message>,
1140    rfc724_mid: &str,
1141    from_id: ContactId,
1142    is_partial_download: &Option<u32>,
1143) -> Result<ChatAssignment> {
1144    let should_trash = if !mime_parser.mdn_reports.is_empty() {
1145        info!(context, "Message is an MDN (TRASH).");
1146        true
1147    } else if mime_parser.delivery_report.is_some() {
1148        info!(context, "Message is a DSN (TRASH).");
1149        markseen_on_imap_table(context, rfc724_mid).await.ok();
1150        true
1151    } else if mime_parser.get_header(HeaderDef::ChatEdit).is_some()
1152        || mime_parser.get_header(HeaderDef::ChatDelete).is_some()
1153        || mime_parser.get_header(HeaderDef::IrohNodeAddr).is_some()
1154        || mime_parser.sync_items.is_some()
1155    {
1156        info!(context, "Chat edit/delete/iroh/sync message (TRASH).");
1157        true
1158    } else if mime_parser.is_system_message == SystemMessage::CallAccepted
1159        || mime_parser.is_system_message == SystemMessage::CallEnded
1160    {
1161        info!(context, "Call state changed (TRASH).");
1162        true
1163    } else if mime_parser.decrypting_failed && !mime_parser.incoming {
1164        // Outgoing undecryptable message.
1165        let last_time = context
1166            .get_config_i64(Config::LastCantDecryptOutgoingMsgs)
1167            .await?;
1168        let now = tools::time();
1169        let update_config = if last_time.saturating_add(24 * 60 * 60) <= now {
1170            let mut msg = Message::new_text(stock_str::cant_decrypt_outgoing_msgs(context).await);
1171            chat::add_device_msg(context, None, Some(&mut msg))
1172                .await
1173                .log_err(context)
1174                .ok();
1175            true
1176        } else {
1177            last_time > now
1178        };
1179        if update_config {
1180            context
1181                .set_config_internal(Config::LastCantDecryptOutgoingMsgs, Some(&now.to_string()))
1182                .await?;
1183        }
1184        info!(context, "Outgoing undecryptable message (TRASH).");
1185        true
1186    } else if mime_parser.is_system_message != SystemMessage::AutocryptSetupMessage
1187        && !mime_parser.has_chat_version()
1188        && parent_message
1189            .as_ref()
1190            .is_none_or(|p| p.is_dc_message == MessengerMessage::No)
1191        && !context.get_config_bool(Config::IsChatmail).await?
1192        && ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await?)
1193            .unwrap_or_default()
1194            == ShowEmails::Off
1195    {
1196        info!(context, "Classical email not shown (TRASH).");
1197        // the message is a classic email in a classic profile
1198        // (in chatmail profiles, we always show all messages, because shared dc-mua usage is not supported)
1199        true
1200    } else if mime_parser
1201        .get_header(HeaderDef::XMozillaDraftInfo)
1202        .is_some()
1203    {
1204        // Mozilla Thunderbird does not set \Draft flag on "Templates", but sets
1205        // X-Mozilla-Draft-Info header, which can be used to detect both drafts and templates
1206        // created by Thunderbird.
1207
1208        // Most mailboxes have a "Drafts" folder where constantly new emails appear but we don't actually want to show them
1209        info!(context, "Email is probably just a draft (TRASH).");
1210        true
1211    } else if mime_parser.webxdc_status_update.is_some() && mime_parser.parts.len() == 1 {
1212        if let Some(part) = mime_parser.parts.first() {
1213            if part.typ == Viewtype::Text && part.msg.is_empty() {
1214                info!(context, "Message is a status update only (TRASH).");
1215                markseen_on_imap_table(context, rfc724_mid).await.ok();
1216                true
1217            } else {
1218                false
1219            }
1220        } else {
1221            false
1222        }
1223    } else {
1224        false
1225    };
1226
1227    // Decide on the type of chat we assign the message to.
1228    //
1229    // The chat may not exist yet, i.e. there may be
1230    // no database row and ChatId yet.
1231    let mut num_recipients = 0;
1232    let mut has_self_addr = false;
1233    for recipient in &mime_parser.recipients {
1234        if addr_cmp(&recipient.addr, &mime_parser.from.addr) {
1235            continue;
1236        }
1237
1238        if context.is_self_addr(&recipient.addr).await? {
1239            has_self_addr = true;
1240        }
1241
1242        num_recipients += 1;
1243    }
1244    if from_id != ContactId::SELF && !has_self_addr {
1245        num_recipients += 1;
1246    }
1247
1248    let chat_assignment = if should_trash {
1249        ChatAssignment::Trash
1250    } else if mime_parser.get_mailinglist_header().is_some() {
1251        ChatAssignment::MailingListOrBroadcast
1252    } else if let Some(grpid) = mime_parser.get_chat_group_id() {
1253        if mime_parser.was_encrypted() {
1254            ChatAssignment::GroupChat {
1255                grpid: grpid.to_string(),
1256            }
1257        } else if let Some(parent) = &parent_message {
1258            if let Some((chat_id, chat_id_blocked)) =
1259                lookup_chat_by_reply(context, mime_parser, parent, is_partial_download).await?
1260            {
1261                // Try to assign to a chat based on In-Reply-To/References.
1262                ChatAssignment::ExistingChat {
1263                    chat_id,
1264                    chat_id_blocked,
1265                }
1266            } else {
1267                ChatAssignment::AdHocGroup
1268            }
1269        } else {
1270            // Could be a message from old version
1271            // with opportunistic encryption.
1272            //
1273            // We still want to assign this to a group
1274            // even if it had only two members.
1275            //
1276            // Group ID is ignored, however.
1277            ChatAssignment::AdHocGroup
1278        }
1279    } else if let Some(parent) = &parent_message {
1280        if let Some((chat_id, chat_id_blocked)) =
1281            lookup_chat_by_reply(context, mime_parser, parent, is_partial_download).await?
1282        {
1283            // Try to assign to a chat based on In-Reply-To/References.
1284            ChatAssignment::ExistingChat {
1285                chat_id,
1286                chat_id_blocked,
1287            }
1288        } else if mime_parser.get_header(HeaderDef::ChatGroupName).is_some() {
1289            ChatAssignment::AdHocGroup
1290        } else if num_recipients <= 1 {
1291            ChatAssignment::OneOneChat
1292        } else {
1293            ChatAssignment::AdHocGroup
1294        }
1295    } else if mime_parser.get_header(HeaderDef::ChatGroupName).is_some() {
1296        ChatAssignment::AdHocGroup
1297    } else if num_recipients <= 1 {
1298        ChatAssignment::OneOneChat
1299    } else {
1300        ChatAssignment::AdHocGroup
1301    };
1302    Ok(chat_assignment)
1303}
1304
1305/// Assigns the message to a chat.
1306///
1307/// Creates a new chat if necessary.
1308#[expect(clippy::too_many_arguments)]
1309async fn do_chat_assignment(
1310    context: &Context,
1311    chat_assignment: ChatAssignment,
1312    from_id: ContactId,
1313    to_ids: &[Option<ContactId>],
1314    past_ids: &[Option<ContactId>],
1315    to_id: ContactId,
1316    allow_creation: bool,
1317    mime_parser: &mut MimeMessage,
1318    is_partial_download: Option<u32>,
1319    verified_encryption: &VerifiedEncryption,
1320    parent_message: Option<Message>,
1321) -> Result<(ChatId, Blocked)> {
1322    let is_bot = context.get_config_bool(Config::Bot).await?;
1323
1324    let mut chat_id = None;
1325    let mut chat_id_blocked = Blocked::Not;
1326
1327    if mime_parser.incoming {
1328        let test_normal_chat = ChatIdBlocked::lookup_by_contact(context, from_id).await?;
1329
1330        let create_blocked_default = if is_bot {
1331            Blocked::Not
1332        } else {
1333            Blocked::Request
1334        };
1335        let create_blocked = if let Some(ChatIdBlocked { id: _, blocked }) = test_normal_chat {
1336            match blocked {
1337                Blocked::Request => create_blocked_default,
1338                Blocked::Not => Blocked::Not,
1339                Blocked::Yes => {
1340                    if Contact::is_blocked_load(context, from_id).await? {
1341                        // User has blocked the contact.
1342                        // Block the group contact created as well.
1343                        Blocked::Yes
1344                    } else {
1345                        // 1:1 chat is blocked, but the contact is not.
1346                        // This happens when 1:1 chat is hidden
1347                        // during scanning of a group invitation code.
1348                        create_blocked_default
1349                    }
1350                }
1351            }
1352        } else {
1353            create_blocked_default
1354        };
1355
1356        match &chat_assignment {
1357            ChatAssignment::Trash => {
1358                chat_id = Some(DC_CHAT_ID_TRASH);
1359            }
1360            ChatAssignment::GroupChat { grpid } => {
1361                // Try to assign to a chat based on Chat-Group-ID.
1362                if let Some((id, _protected, blocked)) =
1363                    chat::get_chat_id_by_grpid(context, grpid).await?
1364                {
1365                    chat_id = Some(id);
1366                    chat_id_blocked = blocked;
1367                } else if allow_creation || test_normal_chat.is_some() {
1368                    if let Some((new_chat_id, new_chat_id_blocked)) = create_group(
1369                        context,
1370                        mime_parser,
1371                        is_partial_download.is_some(),
1372                        create_blocked,
1373                        from_id,
1374                        to_ids,
1375                        past_ids,
1376                        verified_encryption,
1377                        grpid,
1378                    )
1379                    .await?
1380                    {
1381                        chat_id = Some(new_chat_id);
1382                        chat_id_blocked = new_chat_id_blocked;
1383                    }
1384                }
1385            }
1386            ChatAssignment::MailingListOrBroadcast => {
1387                if let Some(mailinglist_header) = mime_parser.get_mailinglist_header() {
1388                    if let Some((new_chat_id, new_chat_id_blocked)) =
1389                        create_or_lookup_mailinglist_or_broadcast(
1390                            context,
1391                            allow_creation,
1392                            mailinglist_header,
1393                            from_id,
1394                            mime_parser,
1395                        )
1396                        .await?
1397                    {
1398                        chat_id = Some(new_chat_id);
1399                        chat_id_blocked = new_chat_id_blocked;
1400
1401                        apply_mailinglist_changes(context, mime_parser, new_chat_id).await?;
1402                    }
1403                }
1404            }
1405            ChatAssignment::ExistingChat {
1406                chat_id: new_chat_id,
1407                chat_id_blocked: new_chat_id_blocked,
1408            } => {
1409                chat_id = Some(*new_chat_id);
1410                chat_id_blocked = *new_chat_id_blocked;
1411            }
1412            ChatAssignment::AdHocGroup => {
1413                if let Some((new_chat_id, new_chat_id_blocked)) = lookup_or_create_adhoc_group(
1414                    context,
1415                    mime_parser,
1416                    to_ids,
1417                    allow_creation || test_normal_chat.is_some(),
1418                    create_blocked,
1419                    is_partial_download.is_some(),
1420                )
1421                .await?
1422                {
1423                    chat_id = Some(new_chat_id);
1424                    chat_id_blocked = new_chat_id_blocked;
1425                }
1426            }
1427            ChatAssignment::OneOneChat => {}
1428        }
1429
1430        // if the chat is somehow blocked but we want to create a non-blocked chat,
1431        // unblock the chat
1432        if chat_id_blocked != Blocked::Not
1433            && create_blocked != Blocked::Yes
1434            && !matches!(chat_assignment, ChatAssignment::MailingListOrBroadcast)
1435        {
1436            if let Some(chat_id) = chat_id {
1437                chat_id.set_blocked(context, create_blocked).await?;
1438                chat_id_blocked = create_blocked;
1439            }
1440        }
1441
1442        if chat_id.is_none() {
1443            // Try to create a 1:1 chat.
1444            let contact = Contact::get_by_id(context, from_id).await?;
1445            let create_blocked = match contact.is_blocked() {
1446                true => Blocked::Yes,
1447                false if is_bot => Blocked::Not,
1448                false => Blocked::Request,
1449            };
1450
1451            if let Some(chat) = test_normal_chat {
1452                chat_id = Some(chat.id);
1453                chat_id_blocked = chat.blocked;
1454            } else if allow_creation {
1455                let chat = ChatIdBlocked::get_for_contact(context, from_id, create_blocked)
1456                    .await
1457                    .context("Failed to get (new) chat for contact")?;
1458                chat_id = Some(chat.id);
1459                chat_id_blocked = chat.blocked;
1460            }
1461
1462            if let Some(chat_id) = chat_id {
1463                if chat_id_blocked != Blocked::Not {
1464                    if chat_id_blocked != create_blocked {
1465                        chat_id.set_blocked(context, create_blocked).await?;
1466                    }
1467                    if create_blocked == Blocked::Request && parent_message.is_some() {
1468                        // we do not want any chat to be created implicitly.  Because of the origin-scale-up,
1469                        // the contact requests will pop up and this should be just fine.
1470                        ContactId::scaleup_origin(context, &[from_id], Origin::IncomingReplyTo)
1471                            .await?;
1472                        info!(
1473                            context,
1474                            "Message is a reply to a known message, mark sender as known.",
1475                        );
1476                    }
1477                }
1478
1479                // Check if the message was sent with verified encryption and set the protection of
1480                // the 1:1 chat accordingly.
1481                let chat = match is_partial_download.is_none()
1482                    && mime_parser.get_header(HeaderDef::SecureJoin).is_none()
1483                {
1484                    true => Some(Chat::load_from_db(context, chat_id).await?)
1485                        .filter(|chat| chat.typ == Chattype::Single),
1486                    false => None,
1487                };
1488                if let Some(chat) = chat {
1489                    ensure_and_debug_assert!(
1490                        chat.typ == Chattype::Single,
1491                        "Chat {chat_id} is not Single",
1492                    );
1493                    let new_protection = match verified_encryption {
1494                        VerifiedEncryption::Verified => ProtectionStatus::Protected,
1495                        VerifiedEncryption::NotVerified(_) => ProtectionStatus::Unprotected,
1496                    };
1497
1498                    ensure_and_debug_assert!(
1499                        chat.protected == ProtectionStatus::Unprotected
1500                            || new_protection == ProtectionStatus::Protected,
1501                        "Chat {chat_id} can't downgrade to Unprotected",
1502                    );
1503                    if chat.protected != new_protection {
1504                        // The message itself will be sorted under the device message since the device
1505                        // message is `MessageState::InNoticed`, which means that all following
1506                        // messages are sorted under it.
1507                        chat_id
1508                            .set_protection(
1509                                context,
1510                                new_protection,
1511                                mime_parser.timestamp_sent,
1512                                Some(from_id),
1513                            )
1514                            .await?;
1515                    }
1516                }
1517            }
1518        }
1519    } else {
1520        // Outgoing
1521
1522        // Older Delta Chat versions with core <=1.152.2 only accepted
1523        // self-sent messages in Saved Messages with own address in the `To` field.
1524        // New Delta Chat versions may use empty `To` field
1525        // with only a single `hidden-recipients` group in this case.
1526        let self_sent = to_ids.len() <= 1 && to_id == ContactId::SELF;
1527
1528        match &chat_assignment {
1529            ChatAssignment::Trash => {
1530                chat_id = Some(DC_CHAT_ID_TRASH);
1531            }
1532            ChatAssignment::GroupChat { grpid } => {
1533                if let Some((id, _protected, blocked)) =
1534                    chat::get_chat_id_by_grpid(context, grpid).await?
1535                {
1536                    chat_id = Some(id);
1537                    chat_id_blocked = blocked;
1538                } else if allow_creation {
1539                    if let Some((new_chat_id, new_chat_id_blocked)) = create_group(
1540                        context,
1541                        mime_parser,
1542                        is_partial_download.is_some(),
1543                        Blocked::Not,
1544                        from_id,
1545                        to_ids,
1546                        past_ids,
1547                        verified_encryption,
1548                        grpid,
1549                    )
1550                    .await?
1551                    {
1552                        chat_id = Some(new_chat_id);
1553                        chat_id_blocked = new_chat_id_blocked;
1554                    }
1555                }
1556            }
1557            ChatAssignment::ExistingChat {
1558                chat_id: new_chat_id,
1559                chat_id_blocked: new_chat_id_blocked,
1560            } => {
1561                chat_id = Some(*new_chat_id);
1562                chat_id_blocked = *new_chat_id_blocked;
1563            }
1564            ChatAssignment::MailingListOrBroadcast => {
1565                // Check if the message belongs to a broadcast channel
1566                // (it can't be a mailing list, since it's outgoing)
1567                if let Some(mailinglist_header) = mime_parser.get_mailinglist_header() {
1568                    let listid = mailinglist_header_listid(mailinglist_header)?;
1569                    chat_id = Some(
1570                        if let Some((id, ..)) = chat::get_chat_id_by_grpid(context, &listid).await?
1571                        {
1572                            id
1573                        } else {
1574                            let name =
1575                                compute_mailinglist_name(mailinglist_header, &listid, mime_parser);
1576                            chat::create_broadcast_ex(context, Nosync, listid, name).await?
1577                        },
1578                    );
1579                }
1580            }
1581            ChatAssignment::AdHocGroup => {
1582                if let Some((new_chat_id, new_chat_id_blocked)) = lookup_or_create_adhoc_group(
1583                    context,
1584                    mime_parser,
1585                    to_ids,
1586                    allow_creation,
1587                    Blocked::Not,
1588                    is_partial_download.is_some(),
1589                )
1590                .await?
1591                {
1592                    chat_id = Some(new_chat_id);
1593                    chat_id_blocked = new_chat_id_blocked;
1594                }
1595            }
1596            ChatAssignment::OneOneChat => {}
1597        }
1598
1599        if !to_ids.is_empty() {
1600            if chat_id.is_none() && allow_creation {
1601                let to_contact = Contact::get_by_id(context, to_id).await?;
1602                if let Some(list_id) = to_contact.param.get(Param::ListId) {
1603                    if let Some((id, _, blocked)) =
1604                        chat::get_chat_id_by_grpid(context, list_id).await?
1605                    {
1606                        chat_id = Some(id);
1607                        chat_id_blocked = blocked;
1608                    }
1609                } else {
1610                    let chat = ChatIdBlocked::get_for_contact(context, to_id, Blocked::Not).await?;
1611                    chat_id = Some(chat.id);
1612                    chat_id_blocked = chat.blocked;
1613                }
1614            }
1615            if chat_id.is_none() && mime_parser.has_chat_version() {
1616                if let Some(chat) = ChatIdBlocked::lookup_by_contact(context, to_id).await? {
1617                    chat_id = Some(chat.id);
1618                    chat_id_blocked = chat.blocked;
1619                }
1620            }
1621        }
1622
1623        if chat_id.is_none() && self_sent {
1624            // from_id==to_id==ContactId::SELF - this is a self-sent messages,
1625            // maybe an Autocrypt Setup Message
1626            let chat = ChatIdBlocked::get_for_contact(context, ContactId::SELF, Blocked::Not)
1627                .await
1628                .context("Failed to get (new) chat for contact")?;
1629
1630            chat_id = Some(chat.id);
1631            chat_id_blocked = chat.blocked;
1632
1633            if Blocked::Not != chat.blocked {
1634                chat.id.unblock_ex(context, Nosync).await?;
1635            }
1636        }
1637
1638        // automatically unblock chat when the user sends a message
1639        if chat_id_blocked != Blocked::Not {
1640            if let Some(chat_id) = chat_id {
1641                chat_id.unblock_ex(context, Nosync).await?;
1642                chat_id_blocked = Blocked::Not;
1643            }
1644        }
1645    }
1646    let chat_id = chat_id.unwrap_or_else(|| {
1647        info!(context, "No chat id for message (TRASH).");
1648        DC_CHAT_ID_TRASH
1649    });
1650    Ok((chat_id, chat_id_blocked))
1651}
1652
1653/// Creates a `ReceivedMsg` from given parts which might consist of
1654/// multiple messages (if there are multiple attachments).
1655/// Every entry in `mime_parser.parts` produces a new row in the `msgs` table.
1656#[expect(clippy::too_many_arguments)]
1657async fn add_parts(
1658    context: &Context,
1659    mime_parser: &mut MimeMessage,
1660    imf_raw: &[u8],
1661    to_ids: &[Option<ContactId>],
1662    past_ids: &[Option<ContactId>],
1663    rfc724_mid: &str,
1664    from_id: ContactId,
1665    seen: bool,
1666    is_partial_download: Option<u32>,
1667    mut replace_msg_id: Option<MsgId>,
1668    prevent_rename: bool,
1669    verified_encryption: VerifiedEncryption,
1670    chat_id: ChatId,
1671    chat_id_blocked: Blocked,
1672    is_dc_message: MessengerMessage,
1673) -> Result<ReceivedMsg> {
1674    let to_id = if mime_parser.incoming {
1675        ContactId::SELF
1676    } else {
1677        to_ids.first().copied().flatten().unwrap_or(ContactId::SELF)
1678    };
1679
1680    // if contact renaming is prevented (for mailinglists and bots),
1681    // we use name from From:-header as override name
1682    if prevent_rename {
1683        if let Some(name) = &mime_parser.from.display_name {
1684            for part in &mut mime_parser.parts {
1685                part.param.set(Param::OverrideSenderDisplayname, name);
1686            }
1687        }
1688    }
1689
1690    let mut chat = Chat::load_from_db(context, chat_id).await?;
1691
1692    if mime_parser.incoming && !chat_id.is_trash() {
1693        // It can happen that the message is put into a chat
1694        // but the From-address is not a member of this chat.
1695        if !chat::is_contact_in_chat(context, chat_id, from_id).await? {
1696            // Mark the sender as overridden.
1697            // The UI will prepend `~` to the sender's name,
1698            // indicating that the sender is not part of the group.
1699            let from = &mime_parser.from;
1700            let name: &str = from.display_name.as_ref().unwrap_or(&from.addr);
1701            for part in &mut mime_parser.parts {
1702                part.param.set(Param::OverrideSenderDisplayname, name);
1703            }
1704        }
1705    }
1706
1707    let is_location_kml = mime_parser.location_kml.is_some();
1708    let is_mdn = !mime_parser.mdn_reports.is_empty();
1709
1710    let mut group_changes = match chat.typ {
1711        _ if chat.id.is_special() => GroupChangesInfo::default(),
1712        Chattype::Single => GroupChangesInfo::default(),
1713        Chattype::Mailinglist => GroupChangesInfo::default(),
1714        Chattype::OutBroadcast => {
1715            apply_out_broadcast_changes(context, mime_parser, &mut chat, from_id).await?
1716        }
1717        Chattype::Group => {
1718            apply_group_changes(
1719                context,
1720                mime_parser,
1721                &mut chat,
1722                from_id,
1723                to_ids,
1724                past_ids,
1725                &verified_encryption,
1726            )
1727            .await?
1728        }
1729        Chattype::InBroadcast => {
1730            apply_in_broadcast_changes(context, mime_parser, &mut chat, from_id).await?
1731        }
1732    };
1733
1734    let rfc724_mid_orig = &mime_parser
1735        .get_rfc724_mid()
1736        .unwrap_or(rfc724_mid.to_string());
1737
1738    // Extract ephemeral timer from the message or use the existing timer if the message is not fully downloaded.
1739    let mut ephemeral_timer = if is_partial_download.is_some() {
1740        chat_id.get_ephemeral_timer(context).await?
1741    } else if let Some(value) = mime_parser.get_header(HeaderDef::EphemeralTimer) {
1742        match value.parse::<EphemeralTimer>() {
1743            Ok(timer) => timer,
1744            Err(err) => {
1745                warn!(context, "Can't parse ephemeral timer \"{value}\": {err:#}.");
1746                EphemeralTimer::Disabled
1747            }
1748        }
1749    } else {
1750        EphemeralTimer::Disabled
1751    };
1752
1753    let state = if !mime_parser.incoming {
1754        MessageState::OutDelivered
1755    } else if seen || is_mdn || chat_id_blocked == Blocked::Yes || group_changes.silent
1756    // No check for `hidden` because only reactions are such and they should be `InFresh`.
1757    {
1758        MessageState::InSeen
1759    } else {
1760        MessageState::InFresh
1761    };
1762    let in_fresh = state == MessageState::InFresh;
1763
1764    let sort_to_bottom = false;
1765    let received = true;
1766    let sort_timestamp = chat_id
1767        .calc_sort_timestamp(
1768            context,
1769            mime_parser.timestamp_sent,
1770            sort_to_bottom,
1771            received,
1772            mime_parser.incoming,
1773        )
1774        .await?;
1775
1776    // Apply ephemeral timer changes to the chat.
1777    //
1778    // Only apply the timer when there are visible parts (e.g., the message does not consist only
1779    // of `location.kml` attachment).  Timer changes without visible received messages may be
1780    // confusing to the user.
1781    if !chat_id.is_special()
1782        && !mime_parser.parts.is_empty()
1783        && chat_id.get_ephemeral_timer(context).await? != ephemeral_timer
1784    {
1785        let chat_contacts =
1786            HashSet::<ContactId>::from_iter(chat::get_chat_contacts(context, chat_id).await?);
1787        let is_from_in_chat =
1788            !chat_contacts.contains(&ContactId::SELF) || chat_contacts.contains(&from_id);
1789
1790        info!(
1791            context,
1792            "Received new ephemeral timer value {ephemeral_timer:?} for chat {chat_id}, checking if it should be applied."
1793        );
1794        if !is_from_in_chat {
1795            warn!(
1796                context,
1797                "Ignoring ephemeral timer change to {ephemeral_timer:?} for chat {chat_id} because sender {from_id} is not a member.",
1798            );
1799        } else if is_dc_message == MessengerMessage::Yes
1800            && get_previous_message(context, mime_parser)
1801                .await?
1802                .map(|p| p.ephemeral_timer)
1803                == Some(ephemeral_timer)
1804            && mime_parser.is_system_message != SystemMessage::EphemeralTimerChanged
1805        {
1806            // The message is a Delta Chat message, so we know that previous message according to
1807            // References header is the last message in the chat as seen by the sender. The timer
1808            // is the same in both the received message and the last message, so we know that the
1809            // sender has not seen any change of the timer between these messages. As our timer
1810            // value is different, it means the sender has not received some timer update that we
1811            // have seen or sent ourselves, so we ignore incoming timer to prevent a rollback.
1812            warn!(
1813                context,
1814                "Ignoring ephemeral timer change to {ephemeral_timer:?} for chat {chat_id} to avoid rollback.",
1815            );
1816        } else if chat_id
1817            .update_timestamp(
1818                context,
1819                Param::EphemeralSettingsTimestamp,
1820                mime_parser.timestamp_sent,
1821            )
1822            .await?
1823        {
1824            if let Err(err) = chat_id
1825                .inner_set_ephemeral_timer(context, ephemeral_timer)
1826                .await
1827            {
1828                warn!(
1829                    context,
1830                    "Failed to modify timer for chat {chat_id}: {err:#}."
1831                );
1832            } else {
1833                info!(
1834                    context,
1835                    "Updated ephemeral timer to {ephemeral_timer:?} for chat {chat_id}."
1836                );
1837                if mime_parser.is_system_message != SystemMessage::EphemeralTimerChanged {
1838                    chat::add_info_msg(
1839                        context,
1840                        chat_id,
1841                        &stock_ephemeral_timer_changed(context, ephemeral_timer, from_id).await,
1842                        sort_timestamp,
1843                    )
1844                    .await?;
1845                }
1846            }
1847        } else {
1848            warn!(
1849                context,
1850                "Ignoring ephemeral timer change to {ephemeral_timer:?} because it is outdated."
1851            );
1852        }
1853    }
1854
1855    let mut better_msg = if mime_parser.is_system_message == SystemMessage::LocationStreamingEnabled
1856    {
1857        Some(stock_str::msg_location_enabled_by(context, from_id).await)
1858    } else if mime_parser.is_system_message == SystemMessage::EphemeralTimerChanged {
1859        let better_msg = stock_ephemeral_timer_changed(context, ephemeral_timer, from_id).await;
1860
1861        // Do not delete the system message itself.
1862        //
1863        // This prevents confusion when timer is changed
1864        // to 1 week, and then changed to 1 hour: after 1
1865        // hour, only the message about the change to 1
1866        // week is left.
1867        ephemeral_timer = EphemeralTimer::Disabled;
1868
1869        Some(better_msg)
1870    } else {
1871        None
1872    };
1873
1874    drop(chat); // Avoid using stale `chat` object.
1875
1876    let sort_timestamp = tweak_sort_timestamp(
1877        context,
1878        mime_parser,
1879        group_changes.silent,
1880        chat_id,
1881        sort_timestamp,
1882    )
1883    .await?;
1884
1885    let mime_in_reply_to = mime_parser
1886        .get_header(HeaderDef::InReplyTo)
1887        .unwrap_or_default();
1888    let mime_references = mime_parser
1889        .get_header(HeaderDef::References)
1890        .unwrap_or_default();
1891
1892    // fine, so far.  now, split the message into simple parts usable as "short messages"
1893    // and add them to the database (mails sent by other messenger clients should result
1894    // into only one message; mails sent by other clients may result in several messages
1895    // (eg. one per attachment))
1896    let icnt = mime_parser.parts.len();
1897
1898    let subject = mime_parser.get_subject().unwrap_or_default();
1899
1900    let is_system_message = mime_parser.is_system_message;
1901
1902    // if indicated by the parser,
1903    // we save the full mime-message and add a flag
1904    // that the ui should show button to display the full message.
1905
1906    // We add "Show Full Message" button to the last message bubble (part) if this flag evaluates to
1907    // `true` finally.
1908    let mut save_mime_modified = false;
1909
1910    let mime_headers = if mime_parser.is_mime_modified {
1911        let headers = if !mime_parser.decoded_data.is_empty() {
1912            mime_parser.decoded_data.clone()
1913        } else {
1914            imf_raw.to_vec()
1915        };
1916        tokio::task::block_in_place(move || buf_compress(&headers))?
1917    } else {
1918        Vec::new()
1919    };
1920
1921    let mut created_db_entries = Vec::with_capacity(mime_parser.parts.len());
1922
1923    if let Some(m) = group_changes.better_msg {
1924        match &better_msg {
1925            None => better_msg = Some(m),
1926            Some(_) => {
1927                if !m.is_empty() {
1928                    group_changes.extra_msgs.push((m, is_system_message, None))
1929                }
1930            }
1931        }
1932    }
1933
1934    let chat_id = if better_msg
1935        .as_ref()
1936        .is_some_and(|better_msg| better_msg.is_empty())
1937        && is_partial_download.is_none()
1938    {
1939        DC_CHAT_ID_TRASH
1940    } else {
1941        chat_id
1942    };
1943
1944    for (group_changes_msg, cmd, added_removed_id) in group_changes.extra_msgs {
1945        chat::add_info_msg_with_cmd(
1946            context,
1947            chat_id,
1948            &group_changes_msg,
1949            cmd,
1950            sort_timestamp,
1951            None,
1952            None,
1953            None,
1954            added_removed_id,
1955        )
1956        .await?;
1957    }
1958
1959    if let Some(node_addr) = mime_parser.get_header(HeaderDef::IrohNodeAddr) {
1960        match mime_parser.get_header(HeaderDef::InReplyTo) {
1961            Some(in_reply_to) => match rfc724_mid_exists(context, in_reply_to).await? {
1962                Some((instance_id, _ts_sent)) => {
1963                    if let Err(err) =
1964                        add_gossip_peer_from_header(context, instance_id, node_addr).await
1965                    {
1966                        warn!(context, "Failed to add iroh peer from header: {err:#}.");
1967                    }
1968                }
1969                None => {
1970                    warn!(
1971                        context,
1972                        "Cannot add iroh peer because WebXDC instance does not exist."
1973                    );
1974                }
1975            },
1976            None => {
1977                warn!(
1978                    context,
1979                    "Cannot add iroh peer because the message has no In-Reply-To."
1980                );
1981            }
1982        }
1983    }
1984
1985    handle_edit_delete(context, mime_parser, from_id).await?;
1986
1987    if mime_parser.is_system_message == SystemMessage::CallAccepted
1988        || mime_parser.is_system_message == SystemMessage::CallEnded
1989    {
1990        if let Some(field) = mime_parser.get_header(HeaderDef::InReplyTo) {
1991            if let Some(call) =
1992                message::get_by_rfc724_mids(context, &parse_message_ids(field)).await?
1993            {
1994                context
1995                    .handle_call_msg(call.get_id(), mime_parser, from_id)
1996                    .await?;
1997            } else {
1998                warn!(context, "Call: Cannot load parent.")
1999            }
2000        } else {
2001            warn!(context, "Call: Not a reply.")
2002        }
2003    }
2004
2005    let hidden = mime_parser.parts.iter().all(|part| part.is_reaction);
2006    let mut parts = mime_parser.parts.iter().peekable();
2007    while let Some(part) = parts.next() {
2008        let hidden = part.is_reaction;
2009        if part.is_reaction {
2010            let reaction_str = simplify::remove_footers(part.msg.as_str());
2011            let is_incoming_fresh = mime_parser.incoming && !seen;
2012            set_msg_reaction(
2013                context,
2014                mime_in_reply_to,
2015                chat_id,
2016                from_id,
2017                sort_timestamp,
2018                Reaction::from(reaction_str.as_str()),
2019                is_incoming_fresh,
2020            )
2021            .await?;
2022        }
2023
2024        let mut param = part.param.clone();
2025        if is_system_message != SystemMessage::Unknown {
2026            param.set_int(Param::Cmd, is_system_message as i32);
2027        }
2028
2029        if let Some(replace_msg_id) = replace_msg_id {
2030            let placeholder = Message::load_from_db(context, replace_msg_id).await?;
2031            for key in [
2032                Param::WebxdcSummary,
2033                Param::WebxdcSummaryTimestamp,
2034                Param::WebxdcDocument,
2035                Param::WebxdcDocumentTimestamp,
2036            ] {
2037                if let Some(value) = placeholder.param.get(key) {
2038                    param.set(key, value);
2039                }
2040            }
2041        }
2042
2043        let (msg, typ): (&str, Viewtype) = if let Some(better_msg) = &better_msg {
2044            (better_msg, Viewtype::Text)
2045        } else {
2046            (&part.msg, part.typ)
2047        };
2048        let part_is_empty =
2049            typ == Viewtype::Text && msg.is_empty() && part.param.get(Param::Quote).is_none();
2050
2051        if let Some(contact_id) = group_changes.added_removed_id {
2052            param.set(Param::ContactAddedRemoved, contact_id.to_u32().to_string());
2053        }
2054
2055        save_mime_modified |= mime_parser.is_mime_modified && !part_is_empty && !hidden;
2056        let save_mime_modified = save_mime_modified && parts.peek().is_none();
2057
2058        let ephemeral_timestamp = if in_fresh {
2059            0
2060        } else {
2061            match ephemeral_timer {
2062                EphemeralTimer::Disabled => 0,
2063                EphemeralTimer::Enabled { duration } => {
2064                    mime_parser.timestamp_rcvd.saturating_add(duration.into())
2065                }
2066            }
2067        };
2068
2069        // If you change which information is skipped if the message is trashed,
2070        // also change `MsgId::trash()` and `delete_expired_messages()`
2071        let trash = chat_id.is_trash() || (is_location_kml && part_is_empty && !save_mime_modified);
2072
2073        let row_id = context
2074            .sql
2075            .call_write(|conn| {
2076                let mut stmt = conn.prepare_cached(
2077            r#"
2078INSERT INTO msgs
2079  (
2080    id,
2081    rfc724_mid, chat_id,
2082    from_id, to_id, timestamp, timestamp_sent, 
2083    timestamp_rcvd, type, state, msgrmsg, 
2084    txt, txt_normalized, subject, param, hidden,
2085    bytes, mime_headers, mime_compressed, mime_in_reply_to,
2086    mime_references, mime_modified, error, ephemeral_timer,
2087    ephemeral_timestamp, download_state, hop_info
2088  )
2089  VALUES (
2090    ?,
2091    ?, ?, ?, ?,
2092    ?, ?, ?, ?,
2093    ?, ?, ?, ?,
2094    ?, ?, ?, ?, ?, 1,
2095    ?, ?, ?, ?,
2096    ?, ?, ?, ?
2097  )
2098ON CONFLICT (id) DO UPDATE
2099SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id,
2100    from_id=excluded.from_id, to_id=excluded.to_id, timestamp_sent=excluded.timestamp_sent,
2101    type=excluded.type, state=max(state,excluded.state), msgrmsg=excluded.msgrmsg,
2102    txt=excluded.txt, txt_normalized=excluded.txt_normalized, subject=excluded.subject,
2103    param=excluded.param,
2104    hidden=excluded.hidden,bytes=excluded.bytes, mime_headers=excluded.mime_headers,
2105    mime_compressed=excluded.mime_compressed, mime_in_reply_to=excluded.mime_in_reply_to,
2106    mime_references=excluded.mime_references, mime_modified=excluded.mime_modified, error=excluded.error, ephemeral_timer=excluded.ephemeral_timer,
2107    ephemeral_timestamp=excluded.ephemeral_timestamp, download_state=excluded.download_state, hop_info=excluded.hop_info
2108RETURNING id
2109"#)?;
2110                let row_id: MsgId = stmt.query_row(params![
2111                    replace_msg_id,
2112                    rfc724_mid_orig,
2113                    if trash { DC_CHAT_ID_TRASH } else { chat_id },
2114                    if trash { ContactId::UNDEFINED } else { from_id },
2115                    if trash { ContactId::UNDEFINED } else { to_id },
2116                    sort_timestamp,
2117                    if trash { 0 } else { mime_parser.timestamp_sent },
2118                    if trash { 0 } else { mime_parser.timestamp_rcvd },
2119                    if trash { Viewtype::Unknown } else { typ },
2120                    if trash { MessageState::Undefined } else { state },
2121                    if trash { MessengerMessage::No } else { is_dc_message },
2122                    if trash || hidden { "" } else { msg },
2123                    if trash || hidden { None } else { message::normalize_text(msg) },
2124                    if trash || hidden { "" } else { &subject },
2125                    if trash {
2126                        "".to_string()
2127                    } else {
2128                        param.to_string()
2129                    },
2130                    !trash && hidden,
2131                    if trash { 0 } else { part.bytes as isize },
2132                    if save_mime_modified && !(trash || hidden) {
2133                        mime_headers.clone()
2134                    } else {
2135                        Vec::new()
2136                    },
2137                    if trash { "" } else { mime_in_reply_to },
2138                    if trash { "" } else { mime_references },
2139                    !trash && save_mime_modified,
2140                    if trash { "" } else { part.error.as_deref().unwrap_or_default() },
2141                    if trash { 0 } else { ephemeral_timer.to_u32() },
2142                    if trash { 0 } else { ephemeral_timestamp },
2143                    if trash {
2144                        DownloadState::Done
2145                    } else if is_partial_download.is_some() {
2146                        DownloadState::Available
2147                    } else if mime_parser.decrypting_failed {
2148                        DownloadState::Undecipherable
2149                    } else {
2150                        DownloadState::Done
2151                    },
2152                    if trash { "" } else { &mime_parser.hop_info },
2153                ],
2154                |row| {
2155                    let msg_id: MsgId = row.get(0)?;
2156                    Ok(msg_id)
2157                }
2158                )?;
2159                Ok(row_id)
2160            })
2161            .await?;
2162
2163        // We only replace placeholder with a first part,
2164        // afterwards insert additional parts.
2165        replace_msg_id = None;
2166
2167        ensure_and_debug_assert!(!row_id.is_special(), "Rowid {row_id} is special");
2168        created_db_entries.push(row_id);
2169    }
2170
2171    // Maybe set logging xdc and add gossip topics for webxdcs.
2172    for (part, msg_id) in mime_parser.parts.iter().zip(&created_db_entries) {
2173        // check if any part contains a webxdc topic id
2174        if part.typ == Viewtype::Webxdc {
2175            if let Some(topic) = mime_parser.get_header(HeaderDef::IrohGossipTopic) {
2176                // default encoding of topic ids is `hex`.
2177                let mut topic_raw = [0u8; 32];
2178                BASE32_NOPAD
2179                    .decode_mut(topic.to_ascii_uppercase().as_bytes(), &mut topic_raw)
2180                    .map_err(|e| e.error)
2181                    .context("Wrong gossip topic header")?;
2182
2183                let topic = TopicId::from_bytes(topic_raw);
2184                insert_topic_stub(context, *msg_id, topic).await?;
2185            } else {
2186                warn!(context, "webxdc doesn't have a gossip topic")
2187            }
2188        }
2189
2190        maybe_set_logging_xdc_inner(
2191            context,
2192            part.typ,
2193            chat_id,
2194            part.param.get(Param::Filename),
2195            *msg_id,
2196        )
2197        .await?;
2198    }
2199
2200    if let Some(replace_msg_id) = replace_msg_id {
2201        // Trash the "replace" placeholder with a message that has no parts. If it has the original
2202        // "Message-ID", mark the placeholder for server-side deletion so as if the user deletes the
2203        // fully downloaded message later, the server-side deletion is issued.
2204        let on_server = rfc724_mid == rfc724_mid_orig;
2205        replace_msg_id.trash(context, on_server).await?;
2206    }
2207
2208    let unarchive = match mime_parser.get_header(HeaderDef::ChatGroupMemberRemoved) {
2209        Some(addr) => context.is_self_addr(addr).await?,
2210        None => true,
2211    };
2212    if unarchive {
2213        chat_id.unarchive_if_not_muted(context, state).await?;
2214    }
2215
2216    info!(
2217        context,
2218        "Message has {icnt} parts and is assigned to chat #{chat_id}."
2219    );
2220
2221    if !chat_id.is_trash() && !hidden {
2222        let mut chat = Chat::load_from_db(context, chat_id).await?;
2223
2224        // In contrast to most other update-timestamps,
2225        // use `sort_timestamp` instead of `sent_timestamp` for the subject-timestamp comparison.
2226        // This way, `LastSubject` actually refers to the most recent message _shown_ in the chat.
2227        if chat
2228            .param
2229            .update_timestamp(Param::SubjectTimestamp, sort_timestamp)?
2230        {
2231            // write the last subject even if empty -
2232            // otherwise a reply may get an outdated subject.
2233            let subject = mime_parser.get_subject().unwrap_or_default();
2234
2235            chat.param.set(Param::LastSubject, subject);
2236            chat.update_param(context).await?;
2237        }
2238    }
2239
2240    // Normally outgoing MDNs sent by us never appear in mailboxes, but Gmail saves all
2241    // outgoing messages, including MDNs, to the Sent folder. If we detect such saved MDN,
2242    // delete it.
2243    let needs_delete_job =
2244        !mime_parser.incoming && is_mdn && is_dc_message == MessengerMessage::Yes;
2245
2246    Ok(ReceivedMsg {
2247        chat_id,
2248        state,
2249        hidden,
2250        sort_timestamp,
2251        msg_ids: created_db_entries,
2252        needs_delete_job,
2253    })
2254}
2255
2256/// Checks for "Chat-Edit" and "Chat-Delete" headers,
2257/// and edits/deletes existing messages accordingly.
2258///
2259/// Returns `true` if this message is an edit/deletion request.
2260async fn handle_edit_delete(
2261    context: &Context,
2262    mime_parser: &MimeMessage,
2263    from_id: ContactId,
2264) -> Result<()> {
2265    if let Some(rfc724_mid) = mime_parser.get_header(HeaderDef::ChatEdit) {
2266        if let Some((original_msg_id, _)) = rfc724_mid_exists(context, rfc724_mid).await? {
2267            if let Some(mut original_msg) =
2268                Message::load_from_db_optional(context, original_msg_id).await?
2269            {
2270                if original_msg.from_id == from_id {
2271                    if let Some(part) = mime_parser.parts.first() {
2272                        let edit_msg_showpadlock = part
2273                            .param
2274                            .get_bool(Param::GuaranteeE2ee)
2275                            .unwrap_or_default();
2276                        if edit_msg_showpadlock || !original_msg.get_showpadlock() {
2277                            let new_text =
2278                                part.msg.strip_prefix(EDITED_PREFIX).unwrap_or(&part.msg);
2279                            chat::save_text_edit_to_db(context, &mut original_msg, new_text)
2280                                .await?;
2281                        } else {
2282                            warn!(context, "Edit message: Not encrypted.");
2283                        }
2284                    }
2285                } else {
2286                    warn!(context, "Edit message: Bad sender.");
2287                }
2288            } else {
2289                warn!(context, "Edit message: Database entry does not exist.");
2290            }
2291        } else {
2292            warn!(
2293                context,
2294                "Edit message: rfc724_mid {rfc724_mid:?} not found."
2295            );
2296        }
2297    } else if let Some(rfc724_mid_list) = mime_parser.get_header(HeaderDef::ChatDelete) {
2298        if let Some(part) = mime_parser.parts.first() {
2299            // See `message::delete_msgs_ex()`, unlike edit requests, DC doesn't send unencrypted
2300            // deletion requests, so there's no need to support them.
2301            if part.param.get_bool(Param::GuaranteeE2ee).unwrap_or(false) {
2302                let mut modified_chat_ids = HashSet::new();
2303                let mut msg_ids = Vec::new();
2304
2305                let rfc724_mid_vec: Vec<&str> = rfc724_mid_list.split_whitespace().collect();
2306                for rfc724_mid in rfc724_mid_vec {
2307                    if let Some((msg_id, _)) =
2308                        message::rfc724_mid_exists(context, rfc724_mid).await?
2309                    {
2310                        if let Some(msg) = Message::load_from_db_optional(context, msg_id).await? {
2311                            if msg.from_id == from_id {
2312                                message::delete_msg_locally(context, &msg).await?;
2313                                msg_ids.push(msg.id);
2314                                modified_chat_ids.insert(msg.chat_id);
2315                            } else {
2316                                warn!(context, "Delete message: Bad sender.");
2317                            }
2318                        } else {
2319                            warn!(context, "Delete message: Database entry does not exist.");
2320                        }
2321                    } else {
2322                        warn!(context, "Delete message: {rfc724_mid:?} not found.");
2323                    }
2324                }
2325                message::delete_msgs_locally_done(context, &msg_ids, modified_chat_ids).await?;
2326            } else {
2327                warn!(context, "Delete message: Not encrypted.");
2328            }
2329        }
2330    }
2331    Ok(())
2332}
2333
2334async fn tweak_sort_timestamp(
2335    context: &Context,
2336    mime_parser: &mut MimeMessage,
2337    silent: bool,
2338    chat_id: ChatId,
2339    sort_timestamp: i64,
2340) -> Result<i64> {
2341    // Ensure replies to messages are sorted after the parent message.
2342    //
2343    // This is useful in a case where sender clocks are not
2344    // synchronized and parent message has a Date: header with a
2345    // timestamp higher than reply timestamp.
2346    //
2347    // This does not help if parent message arrives later than the
2348    // reply.
2349    let parent_timestamp = mime_parser.get_parent_timestamp(context).await?;
2350    let mut sort_timestamp = parent_timestamp.map_or(sort_timestamp, |parent_timestamp| {
2351        std::cmp::max(sort_timestamp, parent_timestamp)
2352    });
2353
2354    // If the message should be silent,
2355    // set the timestamp to be no more than the same as last message
2356    // so that the chat is not sorted to the top of the chatlist.
2357    if silent {
2358        let last_msg_timestamp = if let Some(t) = chat_id.get_timestamp(context).await? {
2359            t
2360        } else {
2361            chat_id.created_timestamp(context).await?
2362        };
2363        sort_timestamp = std::cmp::min(sort_timestamp, last_msg_timestamp);
2364    }
2365    Ok(sort_timestamp)
2366}
2367
2368/// Saves attached locations to the database.
2369///
2370/// Emits an event if at least one new location was added.
2371async fn save_locations(
2372    context: &Context,
2373    mime_parser: &MimeMessage,
2374    chat_id: ChatId,
2375    from_id: ContactId,
2376    msg_id: MsgId,
2377) -> Result<()> {
2378    if chat_id.is_special() {
2379        // Do not save locations for trashed messages.
2380        return Ok(());
2381    }
2382
2383    let mut send_event = false;
2384
2385    if let Some(message_kml) = &mime_parser.message_kml {
2386        if let Some(newest_location_id) =
2387            location::save(context, chat_id, from_id, &message_kml.locations, true).await?
2388        {
2389            location::set_msg_location_id(context, msg_id, newest_location_id).await?;
2390            send_event = true;
2391        }
2392    }
2393
2394    if let Some(location_kml) = &mime_parser.location_kml {
2395        if let Some(addr) = &location_kml.addr {
2396            let contact = Contact::get_by_id(context, from_id).await?;
2397            if contact.get_addr().to_lowercase() == addr.to_lowercase() {
2398                if location::save(context, chat_id, from_id, &location_kml.locations, false)
2399                    .await?
2400                    .is_some()
2401                {
2402                    send_event = true;
2403                }
2404            } else {
2405                warn!(
2406                    context,
2407                    "Address in location.kml {:?} is not the same as the sender address {:?}.",
2408                    addr,
2409                    contact.get_addr()
2410                );
2411            }
2412        }
2413    }
2414    if send_event {
2415        context.emit_location_changed(Some(from_id)).await?;
2416    }
2417    Ok(())
2418}
2419
2420async fn lookup_chat_by_reply(
2421    context: &Context,
2422    mime_parser: &MimeMessage,
2423    parent: &Message,
2424    is_partial_download: &Option<u32>,
2425) -> Result<Option<(ChatId, Blocked)>> {
2426    // If the message is encrypted and has group ID,
2427    // lookup by reply should never be needed
2428    // as we can directly assign the message to the chat
2429    // by its group ID.
2430    ensure_and_debug_assert!(
2431        mime_parser.get_chat_group_id().is_none() || !mime_parser.was_encrypted(),
2432        "Encrypted message has group ID {}",
2433        mime_parser.get_chat_group_id().unwrap_or_default(),
2434    );
2435
2436    // Try to assign message to the same chat as the parent message.
2437    let Some(parent_chat_id) = ChatId::lookup_by_message(parent) else {
2438        return Ok(None);
2439    };
2440
2441    // If this was a private message just to self, it was probably a private reply.
2442    // It should not go into the group then, but into the private chat.
2443    if is_probably_private_reply(context, mime_parser, parent_chat_id).await? {
2444        return Ok(None);
2445    }
2446
2447    // If the parent chat is a 1:1 chat, and the sender added
2448    // a new person to TO/CC, then the message should not go to the 1:1 chat, but to a
2449    // newly created ad-hoc group.
2450    let parent_chat = Chat::load_from_db(context, parent_chat_id).await?;
2451    if parent_chat.typ == Chattype::Single && mime_parser.recipients.len() > 1 {
2452        return Ok(None);
2453    }
2454
2455    // Do not assign unencrypted messages to encrypted chats.
2456    if is_partial_download.is_none()
2457        && parent_chat.is_encrypted(context).await?
2458        && !mime_parser.was_encrypted()
2459    {
2460        return Ok(None);
2461    }
2462
2463    info!(
2464        context,
2465        "Assigning message to {parent_chat_id} as it's a reply to {}.", parent.rfc724_mid
2466    );
2467    Ok(Some((parent_chat.id, parent_chat.blocked)))
2468}
2469
2470async fn lookup_or_create_adhoc_group(
2471    context: &Context,
2472    mime_parser: &MimeMessage,
2473    to_ids: &[Option<ContactId>],
2474    allow_creation: bool,
2475    create_blocked: Blocked,
2476    is_partial_download: bool,
2477) -> Result<Option<(ChatId, Blocked)>> {
2478    // Partial download may be an encrypted message with protected Subject header. We do not want to
2479    // create a group with "..." or "Encrypted message" as a subject. The same is for undecipherable
2480    // messages. Instead, assign the message to 1:1 chat with the sender.
2481    if is_partial_download {
2482        info!(
2483            context,
2484            "Ad-hoc group cannot be created from partial download."
2485        );
2486        return Ok(None);
2487    }
2488    if mime_parser.decrypting_failed {
2489        warn!(
2490            context,
2491            "Not creating ad-hoc group for message that cannot be decrypted."
2492        );
2493        return Ok(None);
2494    }
2495
2496    // Lookup address-contact by the From address.
2497    let fingerprint = None;
2498    let find_key_contact_by_addr = false;
2499    let prevent_rename = should_prevent_rename(mime_parser);
2500    let (from_id, _from_id_blocked, _incoming_origin) = from_field_to_contact_id(
2501        context,
2502        &mime_parser.from,
2503        fingerprint,
2504        prevent_rename,
2505        find_key_contact_by_addr,
2506    )
2507    .await?
2508    .context("Cannot lookup address-contact by the From field")?;
2509
2510    let grpname = mime_parser
2511        .get_header(HeaderDef::ChatGroupName)
2512        .map(|s| s.to_string())
2513        .unwrap_or_else(|| {
2514            mime_parser
2515                .get_subject()
2516                .map(|s| remove_subject_prefix(&s))
2517                .unwrap_or_else(|| "👥📧".to_string())
2518        });
2519    let to_ids: Vec<ContactId> = to_ids.iter().filter_map(|x| *x).collect();
2520    let mut contact_ids = Vec::with_capacity(to_ids.len() + 1);
2521    contact_ids.extend(&to_ids);
2522    if !contact_ids.contains(&from_id) {
2523        contact_ids.push(from_id);
2524    }
2525    let trans_fn = |t: &mut rusqlite::Transaction| {
2526        t.pragma_update(None, "query_only", "0")?;
2527        t.execute(
2528            "CREATE TEMP TABLE temp.contacts (
2529                id INTEGER PRIMARY KEY
2530            ) STRICT",
2531            (),
2532        )?;
2533        let mut stmt = t.prepare("INSERT INTO temp.contacts(id) VALUES (?)")?;
2534        for &id in &contact_ids {
2535            stmt.execute((id,))?;
2536        }
2537        let val = t
2538            .query_row(
2539                "SELECT c.id, c.blocked
2540                FROM chats c INNER JOIN msgs m ON c.id=m.chat_id
2541                WHERE m.hidden=0 AND c.grpid='' AND c.name=?
2542                AND (SELECT COUNT(*) FROM chats_contacts
2543                     WHERE chat_id=c.id
2544                     AND add_timestamp >= remove_timestamp)=?
2545                AND (SELECT COUNT(*) FROM chats_contacts
2546                     WHERE chat_id=c.id
2547                     AND contact_id NOT IN (SELECT id FROM temp.contacts)
2548                     AND add_timestamp >= remove_timestamp)=0
2549                ORDER BY m.timestamp DESC",
2550                (&grpname, contact_ids.len()),
2551                |row| {
2552                    let id: ChatId = row.get(0)?;
2553                    let blocked: Blocked = row.get(1)?;
2554                    Ok((id, blocked))
2555                },
2556            )
2557            .optional()?;
2558        t.execute("DROP TABLE temp.contacts", ())?;
2559        Ok(val)
2560    };
2561    let query_only = true;
2562    if let Some((chat_id, blocked)) = context.sql.transaction_ex(query_only, trans_fn).await? {
2563        info!(
2564            context,
2565            "Assigning message to ad-hoc group {chat_id} with matching name and members."
2566        );
2567        return Ok(Some((chat_id, blocked)));
2568    }
2569    if !allow_creation {
2570        return Ok(None);
2571    }
2572    create_adhoc_group(
2573        context,
2574        mime_parser,
2575        create_blocked,
2576        from_id,
2577        &to_ids,
2578        &grpname,
2579    )
2580    .await
2581    .context("Could not create ad hoc group")
2582}
2583
2584/// If this method returns true, the message shall be assigned to the 1:1 chat with the sender.
2585/// If it returns false, it shall be assigned to the parent chat.
2586async fn is_probably_private_reply(
2587    context: &Context,
2588    mime_parser: &MimeMessage,
2589    parent_chat_id: ChatId,
2590) -> Result<bool> {
2591    // Message cannot be a private reply if it has an explicit Chat-Group-ID header.
2592    if mime_parser.get_chat_group_id().is_some() {
2593        return Ok(false);
2594    }
2595
2596    // Usually we don't want to show private replies in the parent chat, but in the
2597    // 1:1 chat with the sender.
2598    //
2599    // There is one exception: Classical MUA replies to two-member groups
2600    // should be assigned to the group chat. We restrict this exception to classical emails, as chat-group-messages
2601    // contain a Chat-Group-Id header and can be sorted into the correct chat this way.
2602
2603    if mime_parser.recipients.len() != 1 {
2604        return Ok(false);
2605    }
2606
2607    if !mime_parser.has_chat_version() {
2608        let chat_contacts = chat::get_chat_contacts(context, parent_chat_id).await?;
2609        if chat_contacts.len() == 2 && chat_contacts.contains(&ContactId::SELF) {
2610            return Ok(false);
2611        }
2612    }
2613
2614    Ok(true)
2615}
2616
2617/// This function tries to extract the group-id from the message and create a new group
2618/// chat with this ID. If there is no group-id and there are more
2619/// than two members, a new ad hoc group is created.
2620///
2621/// On success the function returns the created (chat_id, chat_blocked) tuple.
2622#[expect(clippy::too_many_arguments)]
2623async fn create_group(
2624    context: &Context,
2625    mime_parser: &mut MimeMessage,
2626    is_partial_download: bool,
2627    create_blocked: Blocked,
2628    from_id: ContactId,
2629    to_ids: &[Option<ContactId>],
2630    past_ids: &[Option<ContactId>],
2631    verified_encryption: &VerifiedEncryption,
2632    grpid: &str,
2633) -> Result<Option<(ChatId, Blocked)>> {
2634    let to_ids_flat: Vec<ContactId> = to_ids.iter().filter_map(|x| *x).collect();
2635    let mut chat_id = None;
2636    let mut chat_id_blocked = Default::default();
2637
2638    let create_protected = if mime_parser.get_header(HeaderDef::ChatVerified).is_some() {
2639        if let VerifiedEncryption::NotVerified(err) = verified_encryption {
2640            warn!(
2641                context,
2642                "Creating unprotected group because of the verification problem: {err:#}."
2643            );
2644            ProtectionStatus::Unprotected
2645        } else {
2646            ProtectionStatus::Protected
2647        }
2648    } else {
2649        ProtectionStatus::Unprotected
2650    };
2651
2652    async fn self_explicitly_added(
2653        context: &Context,
2654        mime_parser: &&mut MimeMessage,
2655    ) -> Result<bool> {
2656        let ret = match mime_parser.get_header(HeaderDef::ChatGroupMemberAdded) {
2657            Some(member_addr) => context.is_self_addr(member_addr).await?,
2658            None => false,
2659        };
2660        Ok(ret)
2661    }
2662
2663    if chat_id.is_none()
2664            && !mime_parser.is_mailinglist_message()
2665            && !grpid.is_empty()
2666            && mime_parser.get_header(HeaderDef::ChatGroupName).is_some()
2667            // otherwise, a pending "quit" message may pop up
2668            && mime_parser.get_header(HeaderDef::ChatGroupMemberRemoved).is_none()
2669            // re-create explicitly left groups only if ourself is re-added
2670            && (!chat::is_group_explicitly_left(context, grpid).await?
2671                || self_explicitly_added(context, &mime_parser).await?)
2672    {
2673        // Group does not exist but should be created.
2674        let grpname = mime_parser
2675            .get_header(HeaderDef::ChatGroupName)
2676            .context("Chat-Group-Name vanished")?
2677            // Workaround for the "Space added before long group names after MIME
2678            // serialization/deserialization #3650" issue. DC itself never creates group names with
2679            // leading/trailing whitespace.
2680            .trim();
2681        let new_chat_id = ChatId::create_multiuser_record(
2682            context,
2683            Chattype::Group,
2684            grpid,
2685            grpname,
2686            create_blocked,
2687            create_protected,
2688            None,
2689            mime_parser.timestamp_sent,
2690        )
2691        .await
2692        .with_context(|| format!("Failed to create group '{grpname}' for grpid={grpid}"))?;
2693
2694        chat_id = Some(new_chat_id);
2695        chat_id_blocked = create_blocked;
2696
2697        // Create initial member list.
2698        if let Some(mut chat_group_member_timestamps) = mime_parser.chat_group_member_timestamps() {
2699            let mut new_to_ids = to_ids.to_vec();
2700            if !new_to_ids.contains(&Some(from_id)) {
2701                new_to_ids.insert(0, Some(from_id));
2702                chat_group_member_timestamps.insert(0, mime_parser.timestamp_sent);
2703            }
2704
2705            update_chats_contacts_timestamps(
2706                context,
2707                new_chat_id,
2708                None,
2709                &new_to_ids,
2710                past_ids,
2711                &chat_group_member_timestamps,
2712            )
2713            .await?;
2714        } else {
2715            let mut members = vec![ContactId::SELF];
2716            if !from_id.is_special() {
2717                members.push(from_id);
2718            }
2719            members.extend(to_ids_flat);
2720
2721            // Add all members with 0 timestamp
2722            // because we don't know the real timestamp of their addition.
2723            // This will allow other senders who support
2724            // `Chat-Group-Member-Timestamps` to overwrite
2725            // timestamps later.
2726            let timestamp = 0;
2727
2728            chat::add_to_chat_contacts_table(context, timestamp, new_chat_id, &members).await?;
2729        }
2730
2731        context.emit_event(EventType::ChatModified(new_chat_id));
2732        chatlist_events::emit_chatlist_changed(context);
2733        chatlist_events::emit_chatlist_item_changed(context, new_chat_id);
2734    }
2735
2736    if let Some(chat_id) = chat_id {
2737        Ok(Some((chat_id, chat_id_blocked)))
2738    } else if is_partial_download || mime_parser.decrypting_failed {
2739        // It is possible that the message was sent to a valid,
2740        // yet unknown group, which was rejected because
2741        // Chat-Group-Name, which is in the encrypted part, was
2742        // not found. We can't create a properly named group in
2743        // this case, so assign error message to 1:1 chat with the
2744        // sender instead.
2745        Ok(None)
2746    } else {
2747        // The message was decrypted successfully, but contains a late "quit" or otherwise
2748        // unwanted message.
2749        info!(context, "Message belongs to unwanted group (TRASH).");
2750        Ok(Some((DC_CHAT_ID_TRASH, Blocked::Not)))
2751    }
2752}
2753
2754async fn update_chats_contacts_timestamps(
2755    context: &Context,
2756    chat_id: ChatId,
2757    ignored_id: Option<ContactId>,
2758    to_ids: &[Option<ContactId>],
2759    past_ids: &[Option<ContactId>],
2760    chat_group_member_timestamps: &[i64],
2761) -> Result<bool> {
2762    let expected_timestamps_count = to_ids.len() + past_ids.len();
2763
2764    if chat_group_member_timestamps.len() != expected_timestamps_count {
2765        warn!(
2766            context,
2767            "Chat-Group-Member-Timestamps has wrong number of timestamps, got {}, expected {}.",
2768            chat_group_member_timestamps.len(),
2769            expected_timestamps_count
2770        );
2771        return Ok(false);
2772    }
2773
2774    let mut modified = false;
2775
2776    context
2777        .sql
2778        .transaction(|transaction| {
2779            let mut add_statement = transaction.prepare(
2780                "INSERT INTO chats_contacts (chat_id, contact_id, add_timestamp)
2781                 VALUES                     (?1,      ?2,         ?3)
2782                 ON CONFLICT (chat_id, contact_id)
2783                 DO
2784                   UPDATE SET add_timestamp=?3
2785                   WHERE ?3>add_timestamp AND ?3>=remove_timestamp",
2786            )?;
2787
2788            for (contact_id, ts) in iter::zip(
2789                to_ids.iter(),
2790                chat_group_member_timestamps.iter().take(to_ids.len()),
2791            ) {
2792                if let Some(contact_id) = contact_id {
2793                    if Some(*contact_id) != ignored_id {
2794                        // It could be that member was already added,
2795                        // but updated addition timestamp
2796                        // is also a modification worth notifying about.
2797                        modified |= add_statement.execute((chat_id, contact_id, ts))? > 0;
2798                    }
2799                }
2800            }
2801
2802            let mut remove_statement = transaction.prepare(
2803                "INSERT INTO chats_contacts (chat_id, contact_id, remove_timestamp)
2804                 VALUES                     (?1,      ?2,         ?3)
2805                 ON CONFLICT (chat_id, contact_id)
2806                 DO
2807                   UPDATE SET remove_timestamp=?3
2808                   WHERE ?3>remove_timestamp AND ?3>add_timestamp",
2809            )?;
2810
2811            for (contact_id, ts) in iter::zip(
2812                past_ids.iter(),
2813                chat_group_member_timestamps.iter().skip(to_ids.len()),
2814            ) {
2815                if let Some(contact_id) = contact_id {
2816                    // It could be that member was already removed,
2817                    // but updated removal timestamp
2818                    // is also a modification worth notifying about.
2819                    modified |= remove_statement.execute((chat_id, contact_id, ts))? > 0;
2820                }
2821            }
2822
2823            Ok(())
2824        })
2825        .await?;
2826
2827    Ok(modified)
2828}
2829
2830/// The return type of [apply_group_changes].
2831/// Contains information on which system messages
2832/// should be shown in the chat.
2833#[derive(Default)]
2834struct GroupChangesInfo {
2835    /// Optional: A better message that should replace the original system message.
2836    /// If this is an empty string, the original system message should be trashed.
2837    better_msg: Option<String>,
2838    /// Added/removed contact `better_msg` refers to.
2839    added_removed_id: Option<ContactId>,
2840    /// If true, the user should not be notified about the group change.
2841    silent: bool,
2842    /// A list of additional group changes messages that should be shown in the chat.
2843    extra_msgs: Vec<(String, SystemMessage, Option<ContactId>)>,
2844}
2845
2846/// Apply group member list, name, avatar and protection status changes from the MIME message.
2847///
2848/// Returns [GroupChangesInfo].
2849///
2850/// * `to_ids` - contents of the `To` and `Cc` headers.
2851/// * `past_ids` - contents of the `Chat-Group-Past-Members` header.
2852async fn apply_group_changes(
2853    context: &Context,
2854    mime_parser: &mut MimeMessage,
2855    chat: &mut Chat,
2856    from_id: ContactId,
2857    to_ids: &[Option<ContactId>],
2858    past_ids: &[Option<ContactId>],
2859    verified_encryption: &VerifiedEncryption,
2860) -> Result<GroupChangesInfo> {
2861    let to_ids_flat: Vec<ContactId> = to_ids.iter().filter_map(|x| *x).collect();
2862    ensure!(chat.typ == Chattype::Group);
2863    ensure!(!chat.id.is_special());
2864
2865    let mut send_event_chat_modified = false;
2866    let (mut removed_id, mut added_id) = (None, None);
2867    let mut better_msg = None;
2868    let mut silent = false;
2869
2870    // True if a Delta Chat client has explicitly added our current primary address.
2871    let self_added =
2872        if let Some(added_addr) = mime_parser.get_header(HeaderDef::ChatGroupMemberAdded) {
2873            addr_cmp(&context.get_primary_self_addr().await?, added_addr)
2874        } else {
2875            false
2876        };
2877
2878    let chat_contacts =
2879        HashSet::<ContactId>::from_iter(chat::get_chat_contacts(context, chat.id).await?);
2880    let is_from_in_chat =
2881        !chat_contacts.contains(&ContactId::SELF) || chat_contacts.contains(&from_id);
2882
2883    if mime_parser.get_header(HeaderDef::ChatVerified).is_some() && !chat.is_protected() {
2884        if let VerifiedEncryption::NotVerified(err) = verified_encryption {
2885            warn!(
2886                context,
2887                "Not marking chat {} as protected due to verification problem: {err:#}.", chat.id,
2888            );
2889        } else {
2890            chat.id
2891                .set_protection(
2892                    context,
2893                    ProtectionStatus::Protected,
2894                    mime_parser.timestamp_sent,
2895                    Some(from_id),
2896                )
2897                .await?;
2898        }
2899    }
2900
2901    if let Some(removed_addr) = mime_parser.get_header(HeaderDef::ChatGroupMemberRemoved) {
2902        // TODO: if address "alice@example.org" is a member of the group twice,
2903        // with old and new key,
2904        // and someone (maybe Alice's new contact) just removed Alice's old contact,
2905        // we may lookup the wrong contact because we only look up by the address.
2906        // The result is that info message may contain the new Alice's display name
2907        // rather than old display name.
2908        // This could be fixed by looking up the contact with the highest
2909        // `remove_timestamp` after applying Chat-Group-Member-Timestamps.
2910        removed_id = lookup_key_contact_by_address(context, removed_addr, Some(chat.id)).await?;
2911        if let Some(id) = removed_id {
2912            better_msg = if id == from_id {
2913                silent = true;
2914                Some(stock_str::msg_group_left_local(context, from_id).await)
2915            } else {
2916                Some(stock_str::msg_del_member_local(context, id, from_id).await)
2917            };
2918        } else {
2919            warn!(context, "Removed {removed_addr:?} has no contact id.")
2920        }
2921    } else if let Some(added_addr) = mime_parser.get_header(HeaderDef::ChatGroupMemberAdded) {
2922        if let Some(key) = mime_parser.gossiped_keys.get(added_addr) {
2923            // TODO: if gossiped keys contain the same address multiple times,
2924            // we may lookup the wrong contact.
2925            // This could be fixed by looking up the contact with
2926            // highest `add_timestamp` to disambiguate.
2927            // The result of the error is that info message
2928            // may contain display name of the wrong contact.
2929            let fingerprint = key.public_key.dc_fingerprint().hex();
2930            if let Some(contact_id) =
2931                lookup_key_contact_by_fingerprint(context, &fingerprint).await?
2932            {
2933                added_id = Some(contact_id);
2934                better_msg =
2935                    Some(stock_str::msg_add_member_local(context, contact_id, from_id).await);
2936            } else {
2937                warn!(context, "Added {added_addr:?} has no contact id.");
2938            }
2939        } else {
2940            warn!(context, "Added {added_addr:?} has no gossiped key.");
2941        }
2942    }
2943
2944    if is_from_in_chat {
2945        apply_chat_name_and_avatar_changes(
2946            context,
2947            mime_parser,
2948            from_id,
2949            chat,
2950            &mut send_event_chat_modified,
2951            &mut better_msg,
2952        )
2953        .await?;
2954
2955        if chat.member_list_is_stale(context).await? {
2956            info!(context, "Member list is stale.");
2957            let mut new_members: HashSet<ContactId> =
2958                HashSet::from_iter(to_ids_flat.iter().copied());
2959            new_members.insert(ContactId::SELF);
2960            if !from_id.is_special() {
2961                new_members.insert(from_id);
2962            }
2963
2964            context
2965                .sql
2966                .transaction(|transaction| {
2967                    // Remove all contacts and tombstones.
2968                    transaction.execute(
2969                        "DELETE FROM chats_contacts
2970                         WHERE chat_id=?",
2971                        (chat.id,),
2972                    )?;
2973
2974                    // Insert contacts with default timestamps of 0.
2975                    let mut statement = transaction.prepare(
2976                        "INSERT INTO chats_contacts (chat_id, contact_id)
2977                         VALUES                     (?,       ?)",
2978                    )?;
2979                    for contact_id in &new_members {
2980                        statement.execute((chat.id, contact_id))?;
2981                    }
2982
2983                    Ok(())
2984                })
2985                .await?;
2986            send_event_chat_modified = true;
2987        } else if let Some(ref chat_group_member_timestamps) =
2988            mime_parser.chat_group_member_timestamps()
2989        {
2990            send_event_chat_modified |= update_chats_contacts_timestamps(
2991                context,
2992                chat.id,
2993                Some(from_id),
2994                to_ids,
2995                past_ids,
2996                chat_group_member_timestamps,
2997            )
2998            .await?;
2999        } else {
3000            let mut new_members: HashSet<ContactId>;
3001            if self_added {
3002                new_members = HashSet::from_iter(to_ids_flat.iter().copied());
3003                new_members.insert(ContactId::SELF);
3004                if !from_id.is_special() {
3005                    new_members.insert(from_id);
3006                }
3007            } else {
3008                new_members = chat_contacts.clone();
3009            }
3010
3011            // Allow non-Delta Chat MUAs to add members.
3012            if mime_parser.get_header(HeaderDef::ChatVersion).is_none() {
3013                // Don't delete any members locally, but instead add absent ones to provide group
3014                // membership consistency for all members:
3015                new_members.extend(to_ids_flat.iter());
3016            }
3017
3018            // Apply explicit addition if any.
3019            if let Some(added_id) = added_id {
3020                new_members.insert(added_id);
3021            }
3022
3023            // Apply explicit removal if any.
3024            if let Some(removed_id) = removed_id {
3025                new_members.remove(&removed_id);
3026            }
3027
3028            if new_members != chat_contacts {
3029                chat::update_chat_contacts_table(
3030                    context,
3031                    mime_parser.timestamp_sent,
3032                    chat.id,
3033                    &new_members,
3034                )
3035                .await?;
3036                send_event_chat_modified = true;
3037            }
3038        }
3039
3040        chat.id
3041            .update_timestamp(
3042                context,
3043                Param::MemberListTimestamp,
3044                mime_parser.timestamp_sent,
3045            )
3046            .await?;
3047    }
3048
3049    let new_chat_contacts = HashSet::<ContactId>::from_iter(
3050        chat::get_chat_contacts(context, chat.id)
3051            .await?
3052            .iter()
3053            .copied(),
3054    );
3055
3056    // These are for adding info messages about implicit membership changes.
3057    let mut added_ids: HashSet<ContactId> = new_chat_contacts
3058        .difference(&chat_contacts)
3059        .copied()
3060        .collect();
3061    let mut removed_ids: HashSet<ContactId> = chat_contacts
3062        .difference(&new_chat_contacts)
3063        .copied()
3064        .collect();
3065
3066    if let Some(added_id) = added_id {
3067        if !added_ids.remove(&added_id) && !self_added {
3068            // No-op "Member added" message.
3069            //
3070            // Trash it.
3071            better_msg = Some(String::new());
3072        }
3073    }
3074    if let Some(removed_id) = removed_id {
3075        removed_ids.remove(&removed_id);
3076    }
3077    let group_changes_msgs = if self_added {
3078        Vec::new()
3079    } else {
3080        group_changes_msgs(context, &added_ids, &removed_ids, chat.id).await?
3081    };
3082
3083    if send_event_chat_modified {
3084        context.emit_event(EventType::ChatModified(chat.id));
3085        chatlist_events::emit_chatlist_item_changed(context, chat.id);
3086    }
3087    Ok(GroupChangesInfo {
3088        better_msg,
3089        added_removed_id: if added_id.is_some() {
3090            added_id
3091        } else {
3092            removed_id
3093        },
3094        silent,
3095        extra_msgs: group_changes_msgs,
3096    })
3097}
3098
3099/// Applies incoming changes to the group's or broadcast channel's name and avatar.
3100///
3101/// - `send_event_chat_modified` is set to `true` if ChatModified event should be sent
3102/// - `better_msg` is filled with an info message about name change, if necessary
3103async fn apply_chat_name_and_avatar_changes(
3104    context: &Context,
3105    mime_parser: &MimeMessage,
3106    from_id: ContactId,
3107    chat: &mut Chat,
3108    send_event_chat_modified: &mut bool,
3109    better_msg: &mut Option<String>,
3110) -> Result<()> {
3111    // ========== Apply chat name changes ==========
3112
3113    let group_name_timestamp = mime_parser
3114        .get_header(HeaderDef::ChatGroupNameTimestamp)
3115        .and_then(|s| s.parse::<i64>().ok());
3116
3117    if let Some(old_name) = mime_parser
3118        .get_header(HeaderDef::ChatGroupNameChanged)
3119        .map(|s| s.trim())
3120        .or(match group_name_timestamp {
3121            Some(0) => None,
3122            Some(_) => Some(chat.name.as_str()),
3123            None => None,
3124        })
3125    {
3126        if let Some(grpname) = mime_parser
3127            .get_header(HeaderDef::ChatGroupName)
3128            .map(|grpname| grpname.trim())
3129            .filter(|grpname| grpname.len() < 200)
3130        {
3131            let grpname = &sanitize_single_line(grpname);
3132
3133            let chat_group_name_timestamp =
3134                chat.param.get_i64(Param::GroupNameTimestamp).unwrap_or(0);
3135            let group_name_timestamp = group_name_timestamp.unwrap_or(mime_parser.timestamp_sent);
3136            // To provide group name consistency, compare names if timestamps are equal.
3137            if (chat_group_name_timestamp, grpname) < (group_name_timestamp, &chat.name)
3138                && chat
3139                    .id
3140                    .update_timestamp(context, Param::GroupNameTimestamp, group_name_timestamp)
3141                    .await?
3142                && grpname != &chat.name
3143            {
3144                info!(context, "Updating grpname for chat {}.", chat.id);
3145                context
3146                    .sql
3147                    .execute("UPDATE chats SET name=? WHERE id=?;", (grpname, chat.id))
3148                    .await?;
3149                *send_event_chat_modified = true;
3150            }
3151            if mime_parser
3152                .get_header(HeaderDef::ChatGroupNameChanged)
3153                .is_some()
3154            {
3155                let old_name = &sanitize_single_line(old_name);
3156                better_msg.get_or_insert(
3157                    stock_str::msg_grp_name(context, old_name, grpname, from_id).await,
3158                );
3159            }
3160        }
3161    }
3162
3163    // ========== Apply chat avatar changes ==========
3164
3165    if let (Some(value), None) = (mime_parser.get_header(HeaderDef::ChatContent), &better_msg) {
3166        if value == "group-avatar-changed" {
3167            if let Some(avatar_action) = &mime_parser.group_avatar {
3168                // this is just an explicit message containing the group-avatar,
3169                // apart from that, the group-avatar is send along with various other messages
3170                better_msg.get_or_insert(match avatar_action {
3171                    AvatarAction::Delete => stock_str::msg_grp_img_deleted(context, from_id).await,
3172                    AvatarAction::Change(_) => {
3173                        stock_str::msg_grp_img_changed(context, from_id).await
3174                    }
3175                });
3176            }
3177        }
3178    }
3179
3180    if let Some(avatar_action) = &mime_parser.group_avatar {
3181        info!(context, "Group-avatar change for {}.", chat.id);
3182        if chat
3183            .param
3184            .update_timestamp(Param::AvatarTimestamp, mime_parser.timestamp_sent)?
3185        {
3186            match avatar_action {
3187                AvatarAction::Change(profile_image) => {
3188                    chat.param.set(Param::ProfileImage, profile_image);
3189                }
3190                AvatarAction::Delete => {
3191                    chat.param.remove(Param::ProfileImage);
3192                }
3193            };
3194            chat.update_param(context).await?;
3195            *send_event_chat_modified = true;
3196        }
3197    }
3198
3199    Ok(())
3200}
3201
3202/// Returns a list of strings that should be shown as info messages, informing about group membership changes.
3203async fn group_changes_msgs(
3204    context: &Context,
3205    added_ids: &HashSet<ContactId>,
3206    removed_ids: &HashSet<ContactId>,
3207    chat_id: ChatId,
3208) -> Result<Vec<(String, SystemMessage, Option<ContactId>)>> {
3209    let mut group_changes_msgs: Vec<(String, SystemMessage, Option<ContactId>)> = Vec::new();
3210    if !added_ids.is_empty() {
3211        warn!(
3212            context,
3213            "Implicit addition of {added_ids:?} to chat {chat_id}."
3214        );
3215    }
3216    if !removed_ids.is_empty() {
3217        warn!(
3218            context,
3219            "Implicit removal of {removed_ids:?} from chat {chat_id}."
3220        );
3221    }
3222    group_changes_msgs.reserve(added_ids.len() + removed_ids.len());
3223    for contact_id in added_ids {
3224        group_changes_msgs.push((
3225            stock_str::msg_add_member_local(context, *contact_id, ContactId::UNDEFINED).await,
3226            SystemMessage::MemberAddedToGroup,
3227            Some(*contact_id),
3228        ));
3229    }
3230    for contact_id in removed_ids {
3231        group_changes_msgs.push((
3232            stock_str::msg_del_member_local(context, *contact_id, ContactId::UNDEFINED).await,
3233            SystemMessage::MemberRemovedFromGroup,
3234            Some(*contact_id),
3235        ));
3236    }
3237
3238    Ok(group_changes_msgs)
3239}
3240
3241static LIST_ID_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^(.+)<(.+)>$").unwrap());
3242
3243fn mailinglist_header_listid(list_id_header: &str) -> Result<String> {
3244    Ok(match LIST_ID_REGEX.captures(list_id_header) {
3245        Some(cap) => cap.get(2).context("no match??")?.as_str().trim(),
3246        None => list_id_header
3247            .trim()
3248            .trim_start_matches('<')
3249            .trim_end_matches('>'),
3250    }
3251    .to_string())
3252}
3253
3254/// Create or lookup a mailing list or incoming broadcast channel chat.
3255///
3256/// `list_id_header` contains the Id that must be used for the mailing list
3257/// and has the form `Name <Id>`, `<Id>` or just `Id`.
3258/// Depending on the mailing list type, `list_id_header`
3259/// was picked from `ListId:`-header or the `Sender:`-header.
3260///
3261/// `mime_parser` is the corresponding message
3262/// and is used to figure out the mailing list name from different header fields.
3263async fn create_or_lookup_mailinglist_or_broadcast(
3264    context: &Context,
3265    allow_creation: bool,
3266    list_id_header: &str,
3267    from_id: ContactId,
3268    mime_parser: &MimeMessage,
3269) -> Result<Option<(ChatId, Blocked)>> {
3270    let listid = mailinglist_header_listid(list_id_header)?;
3271
3272    if let Some((chat_id, _, blocked)) = chat::get_chat_id_by_grpid(context, &listid).await? {
3273        return Ok(Some((chat_id, blocked)));
3274    }
3275
3276    let chattype = if mime_parser.was_encrypted() {
3277        Chattype::InBroadcast
3278    } else {
3279        Chattype::Mailinglist
3280    };
3281
3282    let name = if chattype == Chattype::InBroadcast {
3283        mime_parser
3284            .get_header(HeaderDef::ChatGroupName)
3285            .unwrap_or("Broadcast Channel")
3286    } else {
3287        &compute_mailinglist_name(list_id_header, &listid, mime_parser)
3288    };
3289
3290    if allow_creation {
3291        // list does not exist but should be created
3292        let param = mime_parser.list_post.as_ref().map(|list_post| {
3293            let mut p = Params::new();
3294            p.set(Param::ListPost, list_post);
3295            p.to_string()
3296        });
3297
3298        let is_bot = context.get_config_bool(Config::Bot).await?;
3299        let blocked = if is_bot {
3300            Blocked::Not
3301        } else {
3302            Blocked::Request
3303        };
3304        let chat_id = ChatId::create_multiuser_record(
3305            context,
3306            chattype,
3307            &listid,
3308            name,
3309            blocked,
3310            ProtectionStatus::Unprotected,
3311            param,
3312            mime_parser.timestamp_sent,
3313        )
3314        .await
3315        .with_context(|| {
3316            format!(
3317                "failed to create mailinglist '{}' for grpid={}",
3318                &name, &listid
3319            )
3320        })?;
3321
3322        chat::add_to_chat_contacts_table(
3323            context,
3324            mime_parser.timestamp_sent,
3325            chat_id,
3326            &[ContactId::SELF],
3327        )
3328        .await?;
3329        if chattype == Chattype::InBroadcast {
3330            chat::add_to_chat_contacts_table(
3331                context,
3332                mime_parser.timestamp_sent,
3333                chat_id,
3334                &[from_id],
3335            )
3336            .await?;
3337        }
3338        Ok(Some((chat_id, blocked)))
3339    } else {
3340        info!(context, "Creating list forbidden by caller.");
3341        Ok(None)
3342    }
3343}
3344
3345fn compute_mailinglist_name(
3346    list_id_header: &str,
3347    listid: &str,
3348    mime_parser: &MimeMessage,
3349) -> String {
3350    let mut name = match LIST_ID_REGEX
3351        .captures(list_id_header)
3352        .and_then(|caps| caps.get(1))
3353    {
3354        Some(cap) => cap.as_str().trim().to_string(),
3355        None => "".to_string(),
3356    };
3357
3358    // for mailchimp lists, the name in `ListId` is just a long number.
3359    // a usable name for these lists is in the `From` header
3360    // and we can detect these lists by a unique `ListId`-suffix.
3361    if listid.ends_with(".list-id.mcsv.net") {
3362        if let Some(display_name) = &mime_parser.from.display_name {
3363            name.clone_from(display_name);
3364        }
3365    }
3366
3367    // additional names in square brackets in the subject are preferred
3368    // (as that part is much more visible, we assume, that names is shorter and comes more to the point,
3369    // than the sometimes longer part from ListId)
3370    let subject = mime_parser.get_subject().unwrap_or_default();
3371    static SUBJECT: LazyLock<Regex> =
3372        LazyLock::new(|| Regex::new(r"^.{0,5}\[(.+?)\](\s*\[.+\])?").unwrap()); // remove square brackets around first name
3373    if let Some(cap) = SUBJECT.captures(&subject) {
3374        name = cap[1].to_string() + cap.get(2).map_or("", |m| m.as_str());
3375    }
3376
3377    // if we do not have a name yet and `From` indicates, that this is a notification list,
3378    // a usable name is often in the `From` header (seen for several parcel service notifications).
3379    // same, if we do not have a name yet and `List-Id` has a known suffix (`.xt.local`)
3380    //
3381    // this pattern is similar to mailchimp above, however,
3382    // with weaker conditions and does not overwrite existing names.
3383    if name.is_empty()
3384        && (mime_parser.from.addr.contains("noreply")
3385            || mime_parser.from.addr.contains("no-reply")
3386            || mime_parser.from.addr.starts_with("notifications@")
3387            || mime_parser.from.addr.starts_with("newsletter@")
3388            || listid.ends_with(".xt.local"))
3389    {
3390        if let Some(display_name) = &mime_parser.from.display_name {
3391            name.clone_from(display_name);
3392        }
3393    }
3394
3395    // as a last resort, use the ListId as the name
3396    // but strip some known, long hash prefixes
3397    if name.is_empty() {
3398        // 51231231231231231231231232869f58.xing.com -> xing.com
3399        static PREFIX_32_CHARS_HEX: LazyLock<Regex> =
3400            LazyLock::new(|| Regex::new(r"([0-9a-fA-F]{32})\.(.{6,})").unwrap());
3401        if let Some(cap) = PREFIX_32_CHARS_HEX
3402            .captures(listid)
3403            .and_then(|caps| caps.get(2))
3404        {
3405            name = cap.as_str().to_string();
3406        } else {
3407            name = listid.to_string();
3408        }
3409    }
3410
3411    sanitize_single_line(&name)
3412}
3413
3414/// Set ListId param on the contact and ListPost param the chat.
3415/// Only called for incoming messages since outgoing messages never have a
3416/// List-Post header, anyway.
3417async fn apply_mailinglist_changes(
3418    context: &Context,
3419    mime_parser: &MimeMessage,
3420    chat_id: ChatId,
3421) -> Result<()> {
3422    let Some(mailinglist_header) = mime_parser.get_mailinglist_header() else {
3423        return Ok(());
3424    };
3425
3426    let mut chat = Chat::load_from_db(context, chat_id).await?;
3427    if chat.typ != Chattype::Mailinglist {
3428        return Ok(());
3429    }
3430    let listid = &chat.grpid;
3431
3432    let new_name = compute_mailinglist_name(mailinglist_header, listid, mime_parser);
3433    if chat.name != new_name
3434        && chat_id
3435            .update_timestamp(
3436                context,
3437                Param::GroupNameTimestamp,
3438                mime_parser.timestamp_sent,
3439            )
3440            .await?
3441    {
3442        info!(context, "Updating listname for chat {chat_id}.");
3443        context
3444            .sql
3445            .execute("UPDATE chats SET name=? WHERE id=?;", (new_name, chat_id))
3446            .await?;
3447        context.emit_event(EventType::ChatModified(chat_id));
3448    }
3449
3450    let Some(list_post) = &mime_parser.list_post else {
3451        return Ok(());
3452    };
3453
3454    let list_post = match ContactAddress::new(list_post) {
3455        Ok(list_post) => list_post,
3456        Err(err) => {
3457            warn!(context, "Invalid List-Post: {:#}.", err);
3458            return Ok(());
3459        }
3460    };
3461    let (contact_id, _) = Contact::add_or_lookup(context, "", &list_post, Origin::Hidden).await?;
3462    let mut contact = Contact::get_by_id(context, contact_id).await?;
3463    if contact.param.get(Param::ListId) != Some(listid) {
3464        contact.param.set(Param::ListId, listid);
3465        contact.update_param(context).await?;
3466    }
3467
3468    if let Some(old_list_post) = chat.param.get(Param::ListPost) {
3469        if list_post.as_ref() != old_list_post {
3470            // Apparently the mailing list is using a different List-Post header in each message.
3471            // Make the mailing list read-only because we wouldn't know which message the user wants to reply to.
3472            chat.param.remove(Param::ListPost);
3473            chat.update_param(context).await?;
3474        }
3475    } else {
3476        chat.param.set(Param::ListPost, list_post);
3477        chat.update_param(context).await?;
3478    }
3479
3480    Ok(())
3481}
3482
3483async fn apply_out_broadcast_changes(
3484    context: &Context,
3485    mime_parser: &MimeMessage,
3486    chat: &mut Chat,
3487    from_id: ContactId,
3488) -> Result<GroupChangesInfo> {
3489    ensure!(chat.typ == Chattype::OutBroadcast);
3490
3491    if let Some(_removed_addr) = mime_parser.get_header(HeaderDef::ChatGroupMemberRemoved) {
3492        // The sender of the message left the broadcast channel
3493        remove_from_chat_contacts_table(context, chat.id, from_id).await?;
3494
3495        return Ok(GroupChangesInfo {
3496            better_msg: Some("".to_string()),
3497            added_removed_id: None,
3498            silent: true,
3499            extra_msgs: vec![],
3500        });
3501    }
3502
3503    Ok(GroupChangesInfo::default())
3504}
3505
3506async fn apply_in_broadcast_changes(
3507    context: &Context,
3508    mime_parser: &MimeMessage,
3509    chat: &mut Chat,
3510    from_id: ContactId,
3511) -> Result<GroupChangesInfo> {
3512    ensure!(chat.typ == Chattype::InBroadcast);
3513
3514    let mut send_event_chat_modified = false;
3515    let mut better_msg = None;
3516
3517    apply_chat_name_and_avatar_changes(
3518        context,
3519        mime_parser,
3520        from_id,
3521        chat,
3522        &mut send_event_chat_modified,
3523        &mut better_msg,
3524    )
3525    .await?;
3526
3527    if let Some(_removed_addr) = mime_parser.get_header(HeaderDef::ChatGroupMemberRemoved) {
3528        // The only member added/removed message that is ever sent is "I left.",
3529        // so, this is the only case we need to handle here
3530        if from_id == ContactId::SELF {
3531            better_msg
3532                .get_or_insert(stock_str::msg_group_left_local(context, ContactId::SELF).await);
3533        }
3534    }
3535
3536    if send_event_chat_modified {
3537        context.emit_event(EventType::ChatModified(chat.id));
3538        chatlist_events::emit_chatlist_item_changed(context, chat.id);
3539    }
3540    Ok(GroupChangesInfo {
3541        better_msg,
3542        added_removed_id: None,
3543        silent: false,
3544        extra_msgs: vec![],
3545    })
3546}
3547
3548/// Creates ad-hoc group and returns chat ID on success.
3549async fn create_adhoc_group(
3550    context: &Context,
3551    mime_parser: &MimeMessage,
3552    create_blocked: Blocked,
3553    from_id: ContactId,
3554    to_ids: &[ContactId],
3555    grpname: &str,
3556) -> Result<Option<(ChatId, Blocked)>> {
3557    let mut member_ids: Vec<ContactId> = to_ids.to_vec();
3558    if !member_ids.contains(&(from_id)) {
3559        member_ids.push(from_id);
3560    }
3561    if !member_ids.contains(&(ContactId::SELF)) {
3562        member_ids.push(ContactId::SELF);
3563    }
3564
3565    if mime_parser.is_mailinglist_message() {
3566        return Ok(None);
3567    }
3568    if mime_parser
3569        .get_header(HeaderDef::ChatGroupMemberRemoved)
3570        .is_some()
3571    {
3572        info!(
3573            context,
3574            "Message removes member from unknown ad-hoc group (TRASH)."
3575        );
3576        return Ok(Some((DC_CHAT_ID_TRASH, Blocked::Not)));
3577    }
3578    if member_ids.len() < 2 {
3579        info!(
3580            context,
3581            "Not creating ad hoc group with less than 2 members."
3582        );
3583        return Ok(None);
3584    }
3585
3586    let new_chat_id: ChatId = ChatId::create_multiuser_record(
3587        context,
3588        Chattype::Group,
3589        "", // Ad hoc groups have no ID.
3590        grpname,
3591        create_blocked,
3592        ProtectionStatus::Unprotected,
3593        None,
3594        mime_parser.timestamp_sent,
3595    )
3596    .await?;
3597
3598    info!(
3599        context,
3600        "Created ad-hoc group id={new_chat_id}, name={grpname:?}."
3601    );
3602    chat::add_to_chat_contacts_table(
3603        context,
3604        mime_parser.timestamp_sent,
3605        new_chat_id,
3606        &member_ids,
3607    )
3608    .await?;
3609
3610    context.emit_event(EventType::ChatModified(new_chat_id));
3611    chatlist_events::emit_chatlist_changed(context);
3612    chatlist_events::emit_chatlist_item_changed(context, new_chat_id);
3613
3614    Ok(Some((new_chat_id, create_blocked)))
3615}
3616
3617#[derive(Debug, PartialEq, Eq)]
3618enum VerifiedEncryption {
3619    Verified,
3620    NotVerified(String), // The string contains the reason why it's not verified
3621}
3622
3623/// Checks whether the message is allowed to appear in a protected chat.
3624///
3625/// This means that it is encrypted and signed with a verified key.
3626async fn has_verified_encryption(
3627    context: &Context,
3628    mimeparser: &MimeMessage,
3629    from_id: ContactId,
3630) -> Result<VerifiedEncryption> {
3631    use VerifiedEncryption::*;
3632
3633    if !mimeparser.was_encrypted() {
3634        return Ok(NotVerified("This message is not encrypted".to_string()));
3635    };
3636
3637    if from_id == ContactId::SELF {
3638        return Ok(Verified);
3639    }
3640
3641    let from_contact = Contact::get_by_id(context, from_id).await?;
3642
3643    let Some(fingerprint) = from_contact.fingerprint() else {
3644        return Ok(NotVerified(
3645            "The message was sent without encryption".to_string(),
3646        ));
3647    };
3648
3649    if from_contact.get_verifier_id(context).await?.is_none() {
3650        return Ok(NotVerified(
3651            "The message was sent by non-verified contact".to_string(),
3652        ));
3653    }
3654
3655    let signed_with_verified_key = mimeparser.signatures.contains(&fingerprint);
3656    if signed_with_verified_key {
3657        Ok(Verified)
3658    } else {
3659        Ok(NotVerified(
3660            "The message was sent with non-verified encryption".to_string(),
3661        ))
3662    }
3663}
3664
3665async fn mark_recipients_as_verified(
3666    context: &Context,
3667    from_id: ContactId,
3668    to_ids: &[Option<ContactId>],
3669    mimeparser: &MimeMessage,
3670) -> Result<()> {
3671    let verifier_id = Some(from_id).filter(|&id| id != ContactId::SELF);
3672    for gossiped_key in mimeparser
3673        .gossiped_keys
3674        .values()
3675        .filter(|gossiped_key| gossiped_key.verified)
3676    {
3677        let fingerprint = gossiped_key.public_key.dc_fingerprint().hex();
3678        let Some(to_id) = lookup_key_contact_by_fingerprint(context, &fingerprint).await? else {
3679            continue;
3680        };
3681
3682        if to_id == ContactId::SELF || to_id == from_id {
3683            continue;
3684        }
3685
3686        mark_contact_id_as_verified(context, to_id, verifier_id).await?;
3687        ChatId::set_protection_for_contact(context, to_id, mimeparser.timestamp_sent).await?;
3688    }
3689
3690    if mimeparser.get_header(HeaderDef::ChatVerified).is_none() {
3691        return Ok(());
3692    }
3693    for to_id in to_ids.iter().filter_map(|&x| x) {
3694        if to_id == ContactId::SELF || to_id == from_id {
3695            continue;
3696        }
3697
3698        mark_contact_id_as_verified(context, to_id, verifier_id).await?;
3699        ChatId::set_protection_for_contact(context, to_id, mimeparser.timestamp_sent).await?;
3700    }
3701
3702    Ok(())
3703}
3704
3705/// Returns the last message referenced from `References` header if it is in the database.
3706///
3707/// For Delta Chat messages it is the last message in the chat of the sender.
3708async fn get_previous_message(
3709    context: &Context,
3710    mime_parser: &MimeMessage,
3711) -> Result<Option<Message>> {
3712    if let Some(field) = mime_parser.get_header(HeaderDef::References) {
3713        if let Some(rfc724mid) = parse_message_ids(field).last() {
3714            if let Some((msg_id, _)) = rfc724_mid_exists(context, rfc724mid).await? {
3715                return Message::load_from_db_optional(context, msg_id).await;
3716            }
3717        }
3718    }
3719    Ok(None)
3720}
3721
3722/// Returns the last message referenced from References: header found in the database.
3723///
3724/// If none found, tries In-Reply-To: as a fallback for classic MUAs that don't set the
3725/// References: header.
3726async fn get_parent_message(
3727    context: &Context,
3728    references: Option<&str>,
3729    in_reply_to: Option<&str>,
3730) -> Result<Option<Message>> {
3731    let mut mids = Vec::new();
3732    if let Some(field) = in_reply_to {
3733        mids = parse_message_ids(field);
3734    }
3735    if let Some(field) = references {
3736        mids.append(&mut parse_message_ids(field));
3737    }
3738    message::get_by_rfc724_mids(context, &mids).await
3739}
3740
3741pub(crate) async fn get_prefetch_parent_message(
3742    context: &Context,
3743    headers: &[mailparse::MailHeader<'_>],
3744) -> Result<Option<Message>> {
3745    get_parent_message(
3746        context,
3747        headers.get_header_value(HeaderDef::References).as_deref(),
3748        headers.get_header_value(HeaderDef::InReplyTo).as_deref(),
3749    )
3750    .await
3751}
3752
3753/// Looks up contact IDs from the database given the list of recipients.
3754async fn add_or_lookup_contacts_by_address_list(
3755    context: &Context,
3756    address_list: &[SingleInfo],
3757    origin: Origin,
3758) -> Result<Vec<Option<ContactId>>> {
3759    let mut contact_ids = Vec::new();
3760    for info in address_list {
3761        let addr = &info.addr;
3762        if !may_be_valid_addr(addr) {
3763            contact_ids.push(None);
3764            continue;
3765        }
3766        let display_name = info.display_name.as_deref();
3767        if let Ok(addr) = ContactAddress::new(addr) {
3768            let (contact_id, _) =
3769                Contact::add_or_lookup(context, display_name.unwrap_or_default(), &addr, origin)
3770                    .await?;
3771            contact_ids.push(Some(contact_id));
3772        } else {
3773            warn!(context, "Contact with address {:?} cannot exist.", addr);
3774            contact_ids.push(None);
3775        }
3776    }
3777
3778    Ok(contact_ids)
3779}
3780
3781/// Looks up contact IDs from the database given the list of recipients.
3782async fn add_or_lookup_key_contacts(
3783    context: &Context,
3784    address_list: &[SingleInfo],
3785    gossiped_keys: &BTreeMap<String, GossipedKey>,
3786    fingerprints: &[Fingerprint],
3787    origin: Origin,
3788) -> Result<Vec<Option<ContactId>>> {
3789    let mut contact_ids = Vec::new();
3790    let mut fingerprint_iter = fingerprints.iter();
3791    for info in address_list {
3792        let addr = &info.addr;
3793        if !may_be_valid_addr(addr) {
3794            contact_ids.push(None);
3795            continue;
3796        }
3797        let fingerprint: String = if let Some(fp) = fingerprint_iter.next() {
3798            // Iterator has not ran out of fingerprints yet.
3799            fp.hex()
3800        } else if let Some(key) = gossiped_keys.get(addr) {
3801            key.public_key.dc_fingerprint().hex()
3802        } else if context.is_self_addr(addr).await? {
3803            contact_ids.push(Some(ContactId::SELF));
3804            continue;
3805        } else {
3806            contact_ids.push(None);
3807            continue;
3808        };
3809        let display_name = info.display_name.as_deref();
3810        if let Ok(addr) = ContactAddress::new(addr) {
3811            let (contact_id, _) = Contact::add_or_lookup_ex(
3812                context,
3813                display_name.unwrap_or_default(),
3814                &addr,
3815                &fingerprint,
3816                origin,
3817            )
3818            .await?;
3819            contact_ids.push(Some(contact_id));
3820        } else {
3821            warn!(context, "Contact with address {:?} cannot exist.", addr);
3822            contact_ids.push(None);
3823        }
3824    }
3825
3826    ensure_and_debug_assert_eq!(contact_ids.len(), address_list.len(),);
3827    Ok(contact_ids)
3828}
3829
3830/// Looks up a key-contact by email address.
3831///
3832/// If provided, `chat_id` must be an encrypted chat ID that has key-contacts inside.
3833/// Otherwise the function searches in all contacts, preferring accepted and most recently seen ones.
3834async fn lookup_key_contact_by_address(
3835    context: &Context,
3836    addr: &str,
3837    chat_id: Option<ChatId>,
3838) -> Result<Option<ContactId>> {
3839    if context.is_self_addr(addr).await? {
3840        if chat_id.is_none() {
3841            return Ok(Some(ContactId::SELF));
3842        }
3843        let is_self_in_chat = context
3844            .sql
3845            .exists(
3846                "SELECT COUNT(*) FROM chats_contacts WHERE chat_id=? AND contact_id=1",
3847                (chat_id,),
3848            )
3849            .await?;
3850        if is_self_in_chat {
3851            return Ok(Some(ContactId::SELF));
3852        }
3853    }
3854    let contact_id: Option<ContactId> = match chat_id {
3855        Some(chat_id) => {
3856            context
3857                .sql
3858                .query_row_optional(
3859                    "SELECT id FROM contacts
3860                     WHERE contacts.addr=?
3861                     AND EXISTS (SELECT 1 FROM chats_contacts
3862                                 WHERE contact_id=contacts.id
3863                                 AND chat_id=?)
3864                     AND fingerprint<>'' -- Should always be true
3865                     ",
3866                    (addr, chat_id),
3867                    |row| {
3868                        let contact_id: ContactId = row.get(0)?;
3869                        Ok(contact_id)
3870                    },
3871                )
3872                .await?
3873        }
3874        None => {
3875            context
3876                .sql
3877                .query_row_optional(
3878                    "SELECT id FROM contacts
3879                     WHERE addr=?
3880                     AND fingerprint<>''
3881                     ORDER BY
3882                         (
3883                             SELECT COUNT(*) FROM chats c
3884                             INNER JOIN chats_contacts cc
3885                             ON c.id=cc.chat_id
3886                             WHERE c.type=?
3887                                 AND c.id>?
3888                                 AND c.blocked=?
3889                                 AND cc.contact_id=contacts.id
3890                         ) DESC,
3891                         last_seen DESC, id DESC
3892                     ",
3893                    (
3894                        addr,
3895                        Chattype::Single,
3896                        constants::DC_CHAT_ID_LAST_SPECIAL,
3897                        Blocked::Not,
3898                    ),
3899                    |row| {
3900                        let contact_id: ContactId = row.get(0)?;
3901                        Ok(contact_id)
3902                    },
3903                )
3904                .await?
3905        }
3906    };
3907    Ok(contact_id)
3908}
3909
3910async fn lookup_key_contact_by_fingerprint(
3911    context: &Context,
3912    fingerprint: &str,
3913) -> Result<Option<ContactId>> {
3914    logged_debug_assert!(
3915        context,
3916        !fingerprint.is_empty(),
3917        "lookup_key_contact_by_fingerprint: fingerprint is empty."
3918    );
3919    if fingerprint.is_empty() {
3920        // Avoid accidentally looking up a non-key-contact.
3921        return Ok(None);
3922    }
3923    if let Some(contact_id) = context
3924        .sql
3925        .query_row_optional(
3926            "SELECT id FROM contacts
3927             WHERE fingerprint=? AND fingerprint!=''",
3928            (fingerprint,),
3929            |row| {
3930                let contact_id: ContactId = row.get(0)?;
3931                Ok(contact_id)
3932            },
3933        )
3934        .await?
3935    {
3936        Ok(Some(contact_id))
3937    } else if let Some(self_fp) = self_fingerprint_opt(context).await? {
3938        if self_fp == fingerprint {
3939            Ok(Some(ContactId::SELF))
3940        } else {
3941            Ok(None)
3942        }
3943    } else {
3944        Ok(None)
3945    }
3946}
3947
3948/// Looks up key-contacts by email addresses.
3949///
3950/// `fingerprints` may be empty.
3951/// This is used as a fallback when email addresses are available,
3952/// but not the fingerprints, e.g. when core 1.157.3
3953/// client sends the `To` and `Chat-Group-Past-Members` header
3954/// but not the corresponding fingerprint list.
3955///
3956/// Lookup is restricted to the chat ID.
3957///
3958/// If contact cannot be found, `None` is returned.
3959/// This ensures that the length of the result vector
3960/// is the same as the number of addresses in the header
3961/// and it is possible to find corresponding
3962/// `Chat-Group-Member-Timestamps` items.
3963async fn lookup_key_contacts_by_address_list(
3964    context: &Context,
3965    address_list: &[SingleInfo],
3966    fingerprints: &[Fingerprint],
3967    chat_id: Option<ChatId>,
3968) -> Result<Vec<Option<ContactId>>> {
3969    let mut contact_ids = Vec::new();
3970    let mut fingerprint_iter = fingerprints.iter();
3971    for info in address_list {
3972        let addr = &info.addr;
3973        if !may_be_valid_addr(addr) {
3974            contact_ids.push(None);
3975            continue;
3976        }
3977
3978        if let Some(fp) = fingerprint_iter.next() {
3979            // Iterator has not ran out of fingerprints yet.
3980            let display_name = info.display_name.as_deref();
3981            let fingerprint: String = fp.hex();
3982
3983            if let Ok(addr) = ContactAddress::new(addr) {
3984                let (contact_id, _) = Contact::add_or_lookup_ex(
3985                    context,
3986                    display_name.unwrap_or_default(),
3987                    &addr,
3988                    &fingerprint,
3989                    Origin::Hidden,
3990                )
3991                .await?;
3992                contact_ids.push(Some(contact_id));
3993            } else {
3994                warn!(context, "Contact with address {:?} cannot exist.", addr);
3995                contact_ids.push(None);
3996            }
3997        } else {
3998            let contact_id = lookup_key_contact_by_address(context, addr, chat_id).await?;
3999            contact_ids.push(contact_id);
4000        }
4001    }
4002    ensure_and_debug_assert_eq!(address_list.len(), contact_ids.len(),);
4003    Ok(contact_ids)
4004}
4005
4006/// Returns true if the message should not result in renaming
4007/// of the sender contact.
4008fn should_prevent_rename(mime_parser: &MimeMessage) -> bool {
4009    (mime_parser.is_mailinglist_message() && !mime_parser.was_encrypted())
4010        || mime_parser.get_header(HeaderDef::Sender).is_some()
4011}
4012
4013#[cfg(test)]
4014mod receive_imf_tests;