deltachat/
receive_imf.rs

1//! Internet Message Format reception pipeline.
2
3use std::collections::{HashMap, HashSet};
4use std::iter;
5use std::sync::LazyLock;
6
7use anyhow::{Context as _, Result, ensure};
8use data_encoding::BASE32_NOPAD;
9use deltachat_contact_tools::{
10    ContactAddress, addr_cmp, addr_normalize, may_be_valid_addr, sanitize_single_line,
11};
12use iroh_gossip::proto::TopicId;
13use mailparse::SingleInfo;
14use num_traits::FromPrimitive;
15use regex::Regex;
16
17use crate::chat::{
18    self, Chat, ChatId, ChatIdBlocked, ProtectionStatus, remove_from_chat_contacts_table,
19};
20use crate::config::Config;
21use crate::constants::{self, Blocked, Chattype, DC_CHAT_ID_TRASH, EDITED_PREFIX, ShowEmails};
22use crate::contact::{Contact, ContactId, Origin, mark_contact_id_as_verified};
23use crate::context::Context;
24use crate::debug_logging::maybe_set_logging_xdc_inner;
25use crate::download::DownloadState;
26use crate::ephemeral::{Timer as EphemeralTimer, stock_ephemeral_timer_changed};
27use crate::events::EventType;
28use crate::headerdef::{HeaderDef, HeaderDefMap};
29use crate::imap::{GENERATED_PREFIX, markseen_on_imap_table};
30use crate::key::self_fingerprint_opt;
31use crate::key::{DcKey, Fingerprint, SignedPublicKey};
32use crate::log::LogExt;
33use crate::log::{info, warn};
34use crate::logged_debug_assert;
35use crate::message::{
36    self, Message, MessageState, MessengerMessage, MsgId, Viewtype, rfc724_mid_exists,
37};
38use crate::mimeparser::{AvatarAction, MimeMessage, SystemMessage, parse_message_ids};
39use crate::param::{Param, Params};
40use crate::peer_channels::{add_gossip_peer_from_header, insert_topic_stub};
41use crate::reaction::{Reaction, set_msg_reaction};
42use crate::rusqlite::OptionalExtension;
43use crate::securejoin::{self, handle_securejoin_handshake, observe_securejoin_on_other_device};
44use crate::simplify;
45use crate::stock_str;
46use crate::sync::Sync::*;
47use crate::tools::{self, buf_compress, remove_subject_prefix};
48use crate::{chatlist_events, ensure_and_debug_assert, ensure_and_debug_assert_eq, location};
49use crate::{contact, imap};
50
51/// This is the struct that is returned after receiving one email (aka MIME message).
52///
53/// One email with multiple attachments can end up as multiple chat messages, but they
54/// all have the same chat_id, state and sort_timestamp.
55#[derive(Debug)]
56pub struct ReceivedMsg {
57    /// Chat the message is assigned to.
58    pub chat_id: ChatId,
59
60    /// Received message state.
61    pub state: MessageState,
62
63    /// Whether the message is hidden.
64    pub hidden: bool,
65
66    /// Message timestamp for sorting.
67    pub sort_timestamp: i64,
68
69    /// IDs of inserted rows in messages table.
70    pub msg_ids: Vec<MsgId>,
71
72    /// Whether IMAP messages should be immediately deleted.
73    pub needs_delete_job: bool,
74}
75
76/// Decision on which kind of chat the message
77/// should be assigned in.
78///
79/// This is done before looking up contact IDs
80/// so we know in advance whether to lookup
81/// key-contacts or email address contacts.
82///
83/// Once this decision is made,
84/// it should not be changed so we
85/// don't assign the message to an encrypted
86/// group after looking up key-contacts
87/// or vice versa.
88#[derive(Debug)]
89enum ChatAssignment {
90    /// Trash the message.
91    Trash,
92
93    /// Group chat with a Group ID.
94    ///
95    /// Lookup key-contacts and
96    /// assign to encrypted group.
97    GroupChat { grpid: String },
98
99    /// Mailing list or broadcast channel.
100    ///
101    /// Mailing lists don't have members.
102    /// Broadcast channels have members
103    /// on the sender side,
104    /// but their addresses don't go into
105    /// the `To` field.
106    ///
107    /// In any case, the `To`
108    /// field should be ignored
109    /// and no contact IDs should be looked
110    /// up except the `from_id`
111    /// which may be an email address contact
112    /// or a key-contact.
113    MailingListOrBroadcast,
114
115    /// Group chat without a Group ID.
116    ///
117    /// This is not encrypted.
118    AdHocGroup,
119
120    /// Assign the message to existing chat
121    /// with a known `chat_id`.
122    ExistingChat {
123        /// ID of existing chat
124        /// which the message should be assigned to.
125        chat_id: ChatId,
126
127        /// Whether existing chat is blocked.
128        /// This is loaded together with a chat ID
129        /// reduce the number of database calls.
130        ///
131        /// We may want to unblock the chat
132        /// after adding the message there
133        /// if the chat is currently blocked.
134        chat_id_blocked: Blocked,
135    },
136
137    /// 1:1 chat with a single contact.
138    ///
139    /// The chat may be encrypted or not,
140    /// it does not matter.
141    /// It is not possible to mix
142    /// email address contacts
143    /// with key-contacts in a single 1:1 chat anyway.
144    OneOneChat,
145}
146
147/// Emulates reception of a message from the network.
148///
149/// This method returns errors on a failure to parse the mail or extract Message-ID. It's only used
150/// for tests and REPL tool, not actual message reception pipeline.
151#[cfg(any(test, feature = "internals"))]
152pub async fn receive_imf(
153    context: &Context,
154    imf_raw: &[u8],
155    seen: bool,
156) -> Result<Option<ReceivedMsg>> {
157    let mail = mailparse::parse_mail(imf_raw).context("can't parse mail")?;
158    let rfc724_mid =
159        imap::prefetch_get_message_id(&mail.headers).unwrap_or_else(imap::create_message_id);
160    if let Some(download_limit) = context.download_limit().await? {
161        let download_limit: usize = download_limit.try_into()?;
162        if imf_raw.len() > download_limit {
163            let head = std::str::from_utf8(imf_raw)?
164                .split("\r\n\r\n")
165                .next()
166                .context("No empty line in the message")?;
167            return receive_imf_from_inbox(
168                context,
169                &rfc724_mid,
170                head.as_bytes(),
171                seen,
172                Some(imf_raw.len().try_into()?),
173            )
174            .await;
175        }
176    }
177    receive_imf_from_inbox(context, &rfc724_mid, imf_raw, seen, None).await
178}
179
180/// Emulates reception of a message from "INBOX".
181///
182/// Only used for tests and REPL tool, not actual message reception pipeline.
183#[cfg(any(test, feature = "internals"))]
184pub(crate) async fn receive_imf_from_inbox(
185    context: &Context,
186    rfc724_mid: &str,
187    imf_raw: &[u8],
188    seen: bool,
189    is_partial_download: Option<u32>,
190) -> Result<Option<ReceivedMsg>> {
191    receive_imf_inner(
192        context,
193        "INBOX",
194        0,
195        0,
196        rfc724_mid,
197        imf_raw,
198        seen,
199        is_partial_download,
200    )
201    .await
202}
203
204/// Inserts a tombstone into `msgs` table
205/// to prevent downloading the same message in the future.
206///
207/// Returns tombstone database row ID.
208async fn insert_tombstone(context: &Context, rfc724_mid: &str) -> Result<MsgId> {
209    let row_id = context
210        .sql
211        .insert(
212            "INSERT INTO msgs(rfc724_mid, chat_id) VALUES (?,?)",
213            (rfc724_mid, DC_CHAT_ID_TRASH),
214        )
215        .await?;
216    let msg_id = MsgId::new(u32::try_from(row_id)?);
217    Ok(msg_id)
218}
219
220async fn get_to_and_past_contact_ids(
221    context: &Context,
222    mime_parser: &MimeMessage,
223    chat_assignment: &ChatAssignment,
224    is_partial_download: Option<u32>,
225    parent_message: &Option<Message>,
226    incoming_origin: Origin,
227) -> Result<(Vec<Option<ContactId>>, Vec<Option<ContactId>>)> {
228    // `None` means that the chat is encrypted,
229    // but we were not able to convert the address
230    // to key-contact, e.g.
231    // because there was no corresponding
232    // Autocrypt-Gossip header.
233    //
234    // This way we still preserve remaining
235    // number of contacts and their positions
236    // so we can match the contacts to
237    // e.g. Chat-Group-Member-Timestamps
238    // header.
239    let to_ids: Vec<Option<ContactId>>;
240    let past_ids: Vec<Option<ContactId>>;
241
242    // ID of the chat to look up the addresses in.
243    //
244    // Note that this is not necessarily the chat we want to assign the message to.
245    // In case of an outgoing private reply to a group message we may
246    // lookup the address of receipient in the list of addresses used in the group,
247    // but want to assign the message to 1:1 chat.
248    let chat_id = match chat_assignment {
249        ChatAssignment::Trash => None,
250        ChatAssignment::GroupChat { grpid } => {
251            if let Some((chat_id, _protected, _blocked)) =
252                chat::get_chat_id_by_grpid(context, grpid).await?
253            {
254                Some(chat_id)
255            } else {
256                None
257            }
258        }
259        ChatAssignment::AdHocGroup => {
260            // If we are going to assign a message to ad hoc group,
261            // we can just convert the email addresses
262            // to e-mail address contacts and don't need a `ChatId`
263            // to lookup key-contacts.
264            None
265        }
266        ChatAssignment::ExistingChat { chat_id, .. } => Some(*chat_id),
267        ChatAssignment::MailingListOrBroadcast => None,
268        ChatAssignment::OneOneChat => {
269            if is_partial_download.is_none() && !mime_parser.incoming {
270                parent_message.as_ref().map(|m| m.chat_id)
271            } else {
272                None
273            }
274        }
275    };
276
277    let member_fingerprints = mime_parser.chat_group_member_fingerprints();
278    let to_member_fingerprints;
279    let past_member_fingerprints;
280
281    if !member_fingerprints.is_empty() {
282        if member_fingerprints.len() >= mime_parser.recipients.len() {
283            (to_member_fingerprints, past_member_fingerprints) =
284                member_fingerprints.split_at(mime_parser.recipients.len());
285        } else {
286            warn!(
287                context,
288                "Unexpected length of the fingerprint header, expected at least {}, got {}.",
289                mime_parser.recipients.len(),
290                member_fingerprints.len()
291            );
292            to_member_fingerprints = &[];
293            past_member_fingerprints = &[];
294        }
295    } else {
296        to_member_fingerprints = &[];
297        past_member_fingerprints = &[];
298    }
299
300    match chat_assignment {
301        ChatAssignment::GroupChat { .. } => {
302            to_ids = add_or_lookup_key_contacts(
303                context,
304                &mime_parser.recipients,
305                &mime_parser.gossiped_keys,
306                to_member_fingerprints,
307                Origin::Hidden,
308            )
309            .await?;
310
311            if let Some(chat_id) = chat_id {
312                past_ids = lookup_key_contacts_by_address_list(
313                    context,
314                    &mime_parser.past_members,
315                    past_member_fingerprints,
316                    Some(chat_id),
317                )
318                .await?;
319            } else {
320                past_ids = add_or_lookup_key_contacts(
321                    context,
322                    &mime_parser.past_members,
323                    &mime_parser.gossiped_keys,
324                    past_member_fingerprints,
325                    Origin::Hidden,
326                )
327                .await?;
328            }
329        }
330        ChatAssignment::Trash | ChatAssignment::MailingListOrBroadcast => {
331            to_ids = Vec::new();
332            past_ids = Vec::new();
333        }
334        ChatAssignment::ExistingChat { chat_id, .. } => {
335            let chat = Chat::load_from_db(context, *chat_id).await?;
336            if chat.is_encrypted(context).await? {
337                to_ids = add_or_lookup_key_contacts(
338                    context,
339                    &mime_parser.recipients,
340                    &mime_parser.gossiped_keys,
341                    to_member_fingerprints,
342                    Origin::Hidden,
343                )
344                .await?;
345                past_ids = lookup_key_contacts_by_address_list(
346                    context,
347                    &mime_parser.past_members,
348                    past_member_fingerprints,
349                    Some(*chat_id),
350                )
351                .await?;
352            } else {
353                to_ids = add_or_lookup_contacts_by_address_list(
354                    context,
355                    &mime_parser.recipients,
356                    if !mime_parser.incoming {
357                        Origin::OutgoingTo
358                    } else if incoming_origin.is_known() {
359                        Origin::IncomingTo
360                    } else {
361                        Origin::IncomingUnknownTo
362                    },
363                )
364                .await?;
365
366                past_ids = add_or_lookup_contacts_by_address_list(
367                    context,
368                    &mime_parser.past_members,
369                    Origin::Hidden,
370                )
371                .await?;
372            }
373        }
374        ChatAssignment::AdHocGroup => {
375            to_ids = add_or_lookup_contacts_by_address_list(
376                context,
377                &mime_parser.recipients,
378                if !mime_parser.incoming {
379                    Origin::OutgoingTo
380                } else if incoming_origin.is_known() {
381                    Origin::IncomingTo
382                } else {
383                    Origin::IncomingUnknownTo
384                },
385            )
386            .await?;
387
388            past_ids = add_or_lookup_contacts_by_address_list(
389                context,
390                &mime_parser.past_members,
391                Origin::Hidden,
392            )
393            .await?;
394        }
395        ChatAssignment::OneOneChat => {
396            let pgp_to_ids = add_or_lookup_key_contacts(
397                context,
398                &mime_parser.recipients,
399                &mime_parser.gossiped_keys,
400                to_member_fingerprints,
401                Origin::Hidden,
402            )
403            .await?;
404            if pgp_to_ids
405                .first()
406                .is_some_and(|contact_id| contact_id.is_some())
407            {
408                // There is a single recipient and we have
409                // mapped it to a key contact.
410                // This is an encrypted 1:1 chat.
411                to_ids = pgp_to_ids
412            } else if let Some(chat_id) = chat_id {
413                to_ids = match mime_parser.was_encrypted() {
414                    true => {
415                        lookup_key_contacts_by_address_list(
416                            context,
417                            &mime_parser.recipients,
418                            to_member_fingerprints,
419                            Some(chat_id),
420                        )
421                        .await?
422                    }
423                    false => {
424                        add_or_lookup_contacts_by_address_list(
425                            context,
426                            &mime_parser.recipients,
427                            if !mime_parser.incoming {
428                                Origin::OutgoingTo
429                            } else if incoming_origin.is_known() {
430                                Origin::IncomingTo
431                            } else {
432                                Origin::IncomingUnknownTo
433                            },
434                        )
435                        .await?
436                    }
437                }
438            } else {
439                let ids = match mime_parser.was_encrypted() {
440                    true => {
441                        lookup_key_contacts_by_address_list(
442                            context,
443                            &mime_parser.recipients,
444                            to_member_fingerprints,
445                            None,
446                        )
447                        .await?
448                    }
449                    false => vec![],
450                };
451                if mime_parser.was_encrypted() && !ids.contains(&None)
452                // Prefer creating PGP chats if there are any key-contacts. At least this prevents
453                // from replying unencrypted.
454                || ids
455                    .iter()
456                    .any(|&c| c.is_some() && c != Some(ContactId::SELF))
457                {
458                    to_ids = ids;
459                } else {
460                    to_ids = add_or_lookup_contacts_by_address_list(
461                        context,
462                        &mime_parser.recipients,
463                        if !mime_parser.incoming {
464                            Origin::OutgoingTo
465                        } else if incoming_origin.is_known() {
466                            Origin::IncomingTo
467                        } else {
468                            Origin::IncomingUnknownTo
469                        },
470                    )
471                    .await?;
472                }
473            }
474
475            past_ids = add_or_lookup_contacts_by_address_list(
476                context,
477                &mime_parser.past_members,
478                Origin::Hidden,
479            )
480            .await?;
481        }
482    };
483
484    Ok((to_ids, past_ids))
485}
486
487/// Receive a message and add it to the database.
488///
489/// Returns an error on database failure or if the message is broken,
490/// e.g. has nonstandard MIME structure.
491///
492/// If possible, creates a database entry to prevent the message from being
493/// downloaded again, sets `chat_id=DC_CHAT_ID_TRASH` and returns `Ok(Some(…))`.
494/// If the message is so wrong that we didn't even create a database entry,
495/// returns `Ok(None)`.
496///
497/// If `is_partial_download` is set, it contains the full message size in bytes.
498/// Do not confuse that with `replace_msg_id` that will be set when the full message is loaded
499/// later.
500#[expect(clippy::too_many_arguments)]
501pub(crate) async fn receive_imf_inner(
502    context: &Context,
503    folder: &str,
504    uidvalidity: u32,
505    uid: u32,
506    rfc724_mid: &str,
507    imf_raw: &[u8],
508    seen: bool,
509    is_partial_download: Option<u32>,
510) -> Result<Option<ReceivedMsg>> {
511    if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
512        info!(
513            context,
514            "receive_imf: incoming message mime-body:\n{}",
515            String::from_utf8_lossy(imf_raw),
516        );
517    }
518
519    let mut mime_parser = match MimeMessage::from_bytes(context, imf_raw, is_partial_download).await
520    {
521        Err(err) => {
522            warn!(context, "receive_imf: can't parse MIME: {err:#}.");
523            if rfc724_mid.starts_with(GENERATED_PREFIX) {
524                // We don't have an rfc724_mid, there's no point in adding a trash entry
525                return Ok(None);
526            }
527
528            let msg_ids = vec![insert_tombstone(context, rfc724_mid).await?];
529
530            return Ok(Some(ReceivedMsg {
531                chat_id: DC_CHAT_ID_TRASH,
532                state: MessageState::Undefined,
533                hidden: false,
534                sort_timestamp: 0,
535                msg_ids,
536                needs_delete_job: false,
537            }));
538        }
539        Ok(mime_parser) => mime_parser,
540    };
541
542    let rfc724_mid_orig = &mime_parser
543        .get_rfc724_mid()
544        .unwrap_or(rfc724_mid.to_string());
545    info!(
546        context,
547        "Receiving message {rfc724_mid_orig:?}, seen={seen}...",
548    );
549
550    // check, if the mail is already in our database.
551    // make sure, this check is done eg. before securejoin-processing.
552    let (replace_msg_id, replace_chat_id);
553    if let Some((old_msg_id, _)) = message::rfc724_mid_exists(context, rfc724_mid).await? {
554        if is_partial_download.is_some() {
555            // Should never happen, see imap::prefetch_should_download(), but still.
556            info!(
557                context,
558                "Got a partial download and message is already in DB."
559            );
560            return Ok(None);
561        }
562        let msg = Message::load_from_db(context, old_msg_id).await?;
563        replace_msg_id = Some(old_msg_id);
564        replace_chat_id = if msg.download_state() != DownloadState::Done {
565            // the message was partially downloaded before and is fully downloaded now.
566            info!(
567                context,
568                "Message already partly in DB, replacing by full message."
569            );
570            Some(msg.chat_id)
571        } else {
572            None
573        };
574    } else {
575        replace_msg_id = if rfc724_mid_orig == rfc724_mid {
576            None
577        } else if let Some((old_msg_id, old_ts_sent)) =
578            message::rfc724_mid_exists(context, rfc724_mid_orig).await?
579        {
580            if imap::is_dup_msg(
581                mime_parser.has_chat_version(),
582                mime_parser.timestamp_sent,
583                old_ts_sent,
584            ) {
585                info!(context, "Deleting duplicate message {rfc724_mid_orig}.");
586                let target = context.get_delete_msgs_target().await?;
587                context
588                    .sql
589                    .execute(
590                        "UPDATE imap SET target=? WHERE folder=? AND uidvalidity=? AND uid=?",
591                        (target, folder, uidvalidity, uid),
592                    )
593                    .await?;
594            }
595            Some(old_msg_id)
596        } else {
597            None
598        };
599        replace_chat_id = None;
600    }
601
602    if replace_chat_id.is_some() {
603        // Need to update chat id in the db.
604    } else if let Some(msg_id) = replace_msg_id {
605        info!(context, "Message is already downloaded.");
606        if mime_parser.incoming {
607            return Ok(None);
608        }
609        // For the case if we missed a successful SMTP response. Be optimistic that the message is
610        // delivered also.
611        let self_addr = context.get_primary_self_addr().await?;
612        context
613            .sql
614            .execute(
615                "DELETE FROM smtp \
616                WHERE rfc724_mid=?1 AND (recipients LIKE ?2 OR recipients LIKE ('% ' || ?2))",
617                (rfc724_mid_orig, &self_addr),
618            )
619            .await?;
620        if !context
621            .sql
622            .exists(
623                "SELECT COUNT(*) FROM smtp WHERE rfc724_mid=?",
624                (rfc724_mid_orig,),
625            )
626            .await?
627        {
628            msg_id.set_delivered(context).await?;
629        }
630        return Ok(None);
631    };
632
633    let prevent_rename = should_prevent_rename(&mime_parser);
634
635    // get From: (it can be an address list!) and check if it is known (for known From:'s we add
636    // the other To:/Cc: in the 3rd pass)
637    // or if From: is equal to SELF (in this case, it is any outgoing messages,
638    // we do not check Return-Path any more as this is unreliable, see
639    // <https://github.com/deltachat/deltachat-core/issues/150>)
640    //
641    // If this is a mailing list email (i.e. list_id_header is some), don't change the displayname because in
642    // a mailing list the sender displayname sometimes does not belong to the sender email address.
643    // For example, GitHub sends messages from `notifications@github.com`,
644    // but uses display name of the user whose action generated the notification
645    // as the display name.
646    let fingerprint = mime_parser.signatures.iter().next();
647    let (from_id, _from_id_blocked, incoming_origin) = match from_field_to_contact_id(
648        context,
649        &mime_parser.from,
650        fingerprint,
651        prevent_rename,
652        is_partial_download.is_some()
653            && mime_parser
654                .get_header(HeaderDef::ContentType)
655                .unwrap_or_default()
656                .starts_with("multipart/encrypted"),
657    )
658    .await?
659    {
660        Some(contact_id_res) => contact_id_res,
661        None => {
662            warn!(
663                context,
664                "receive_imf: From field does not contain an acceptable address."
665            );
666            return Ok(None);
667        }
668    };
669
670    // Lookup parent message.
671    //
672    // This may be useful to assign the message to
673    // group chats without Chat-Group-ID
674    // when a message is sent by Thunderbird.
675    //
676    // This can be also used to lookup
677    // key-contact by email address
678    // when receiving a private 1:1 reply
679    // to a group chat message.
680    let parent_message = get_parent_message(
681        context,
682        mime_parser.get_header(HeaderDef::References),
683        mime_parser.get_header(HeaderDef::InReplyTo),
684    )
685    .await?
686    .filter(|p| Some(p.id) != replace_msg_id);
687
688    let chat_assignment = decide_chat_assignment(
689        context,
690        &mime_parser,
691        &parent_message,
692        rfc724_mid,
693        from_id,
694        &is_partial_download,
695    )
696    .await?;
697    info!(context, "Chat assignment is {chat_assignment:?}.");
698
699    let (to_ids, past_ids) = get_to_and_past_contact_ids(
700        context,
701        &mime_parser,
702        &chat_assignment,
703        is_partial_download,
704        &parent_message,
705        incoming_origin,
706    )
707    .await?;
708
709    let received_msg;
710    if mime_parser.get_header(HeaderDef::SecureJoin).is_some() {
711        let res = if mime_parser.incoming {
712            handle_securejoin_handshake(context, &mut mime_parser, from_id)
713                .await
714                .context("error in Secure-Join message handling")?
715        } else {
716            let to_id = to_ids.first().copied().flatten().unwrap_or(ContactId::SELF);
717            // handshake may mark contacts as verified and must be processed before chats are created
718            observe_securejoin_on_other_device(context, &mime_parser, to_id)
719                .await
720                .context("error in Secure-Join watching")?
721        };
722
723        match res {
724            securejoin::HandshakeMessage::Done | securejoin::HandshakeMessage::Ignore => {
725                let msg_id = insert_tombstone(context, rfc724_mid).await?;
726                received_msg = Some(ReceivedMsg {
727                    chat_id: DC_CHAT_ID_TRASH,
728                    state: MessageState::InSeen,
729                    hidden: false,
730                    sort_timestamp: mime_parser.timestamp_sent,
731                    msg_ids: vec![msg_id],
732                    needs_delete_job: res == securejoin::HandshakeMessage::Done,
733                });
734            }
735            securejoin::HandshakeMessage::Propagate => {
736                received_msg = None;
737            }
738        }
739    } else {
740        received_msg = None;
741    }
742
743    let verified_encryption = has_verified_encryption(context, &mime_parser, from_id).await?;
744
745    if verified_encryption == VerifiedEncryption::Verified {
746        mark_recipients_as_verified(context, from_id, &to_ids, &mime_parser).await?;
747    }
748
749    let received_msg = if let Some(received_msg) = received_msg {
750        received_msg
751    } else {
752        let is_dc_message = if mime_parser.has_chat_version() {
753            MessengerMessage::Yes
754        } else if let Some(parent_message) = &parent_message {
755            match parent_message.is_dc_message {
756                MessengerMessage::No => MessengerMessage::No,
757                MessengerMessage::Yes | MessengerMessage::Reply => MessengerMessage::Reply,
758            }
759        } else {
760            MessengerMessage::No
761        };
762
763        let show_emails = ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await?)
764            .unwrap_or_default();
765
766        let allow_creation = if mime_parser.decrypting_failed {
767            false
768        } else if mime_parser.is_system_message != SystemMessage::AutocryptSetupMessage
769            && is_dc_message == MessengerMessage::No
770            && !context.get_config_bool(Config::IsChatmail).await?
771        {
772            // the message is a classic email in a classic profile
773            // (in chatmail profiles, we always show all messages, because shared dc-mua usage is not supported)
774            match show_emails {
775                ShowEmails::Off | ShowEmails::AcceptedContacts => false,
776                ShowEmails::All => true,
777            }
778        } else {
779            !mime_parser.parts.iter().all(|part| part.is_reaction)
780        };
781
782        let to_id = if mime_parser.incoming {
783            ContactId::SELF
784        } else {
785            to_ids.first().copied().flatten().unwrap_or(ContactId::SELF)
786        };
787
788        let (chat_id, chat_id_blocked) = do_chat_assignment(
789            context,
790            chat_assignment,
791            from_id,
792            &to_ids,
793            &past_ids,
794            to_id,
795            allow_creation,
796            &mut mime_parser,
797            is_partial_download,
798            &verified_encryption,
799            parent_message,
800        )
801        .await?;
802
803        // Add parts
804        add_parts(
805            context,
806            &mut mime_parser,
807            imf_raw,
808            &to_ids,
809            &past_ids,
810            rfc724_mid_orig,
811            from_id,
812            seen,
813            is_partial_download,
814            replace_msg_id,
815            prevent_rename,
816            verified_encryption,
817            chat_id,
818            chat_id_blocked,
819            is_dc_message,
820        )
821        .await
822        .context("add_parts error")?
823    };
824
825    if !from_id.is_special() {
826        contact::update_last_seen(context, from_id, mime_parser.timestamp_sent).await?;
827    }
828
829    // Update gossiped timestamp for the chat if someone else or our other device sent
830    // Autocrypt-Gossip header to avoid sending Autocrypt-Gossip ourselves
831    // and waste traffic.
832    let chat_id = received_msg.chat_id;
833    if !chat_id.is_special() {
834        for gossiped_key in mime_parser.gossiped_keys.values() {
835            context
836                .sql
837                .transaction(move |transaction| {
838                    let fingerprint = gossiped_key.dc_fingerprint().hex();
839                    transaction.execute(
840                        "INSERT INTO gossip_timestamp (chat_id, fingerprint, timestamp)
841                         VALUES                       (?, ?, ?)
842                         ON CONFLICT                  (chat_id, fingerprint)
843                         DO UPDATE SET timestamp=MAX(timestamp, excluded.timestamp)",
844                        (chat_id, &fingerprint, mime_parser.timestamp_sent),
845                    )?;
846
847                    Ok(())
848                })
849                .await?;
850        }
851    }
852
853    let insert_msg_id = if let Some(msg_id) = received_msg.msg_ids.last() {
854        *msg_id
855    } else {
856        MsgId::new_unset()
857    };
858
859    save_locations(context, &mime_parser, chat_id, from_id, insert_msg_id).await?;
860
861    if let Some(ref sync_items) = mime_parser.sync_items {
862        if from_id == ContactId::SELF {
863            if mime_parser.was_encrypted() {
864                context.execute_sync_items(sync_items).await;
865            } else {
866                warn!(context, "Sync items are not encrypted.");
867            }
868        } else {
869            warn!(context, "Sync items not sent by self.");
870        }
871    }
872
873    if let Some(ref status_update) = mime_parser.webxdc_status_update {
874        let can_info_msg;
875        let instance = if mime_parser
876            .parts
877            .first()
878            .filter(|part| part.typ == Viewtype::Webxdc)
879            .is_some()
880        {
881            can_info_msg = false;
882            Some(Message::load_from_db(context, insert_msg_id).await?)
883        } else if let Some(field) = mime_parser.get_header(HeaderDef::InReplyTo) {
884            if let Some(instance) =
885                message::get_by_rfc724_mids(context, &parse_message_ids(field)).await?
886            {
887                can_info_msg = instance.download_state() == DownloadState::Done;
888                Some(instance)
889            } else {
890                can_info_msg = false;
891                None
892            }
893        } else {
894            can_info_msg = false;
895            None
896        };
897
898        if let Some(instance) = instance {
899            if let Err(err) = context
900                .receive_status_update(
901                    from_id,
902                    &instance,
903                    received_msg.sort_timestamp,
904                    can_info_msg,
905                    status_update,
906                )
907                .await
908            {
909                warn!(context, "receive_imf cannot update status: {err:#}.");
910            }
911        } else {
912            warn!(
913                context,
914                "Received webxdc update, but cannot assign it to message."
915            );
916        }
917    }
918
919    if let Some(avatar_action) = &mime_parser.user_avatar {
920        if from_id != ContactId::UNDEFINED
921            && context
922                .update_contacts_timestamp(
923                    from_id,
924                    Param::AvatarTimestamp,
925                    mime_parser.timestamp_sent,
926                )
927                .await?
928        {
929            if let Err(err) = contact::set_profile_image(
930                context,
931                from_id,
932                avatar_action,
933                mime_parser.was_encrypted(),
934            )
935            .await
936            {
937                warn!(context, "receive_imf cannot update profile image: {err:#}.");
938            };
939        }
940    }
941
942    // Ignore footers from mailinglists as they are often created or modified by the mailinglist software.
943    if let Some(footer) = &mime_parser.footer {
944        if !mime_parser.is_mailinglist_message()
945            && from_id != ContactId::UNDEFINED
946            && context
947                .update_contacts_timestamp(
948                    from_id,
949                    Param::StatusTimestamp,
950                    mime_parser.timestamp_sent,
951                )
952                .await?
953        {
954            if let Err(err) = contact::set_status(
955                context,
956                from_id,
957                footer.to_string(),
958                mime_parser.was_encrypted(),
959                mime_parser.has_chat_version(),
960            )
961            .await
962            {
963                warn!(context, "Cannot update contact status: {err:#}.");
964            }
965        }
966    }
967
968    // Get user-configured server deletion
969    let delete_server_after = context.get_config_delete_server_after().await?;
970
971    if !received_msg.msg_ids.is_empty() {
972        let target = if received_msg.needs_delete_job
973            || (delete_server_after == Some(0) && is_partial_download.is_none())
974        {
975            Some(context.get_delete_msgs_target().await?)
976        } else {
977            None
978        };
979        if target.is_some() || rfc724_mid_orig != rfc724_mid {
980            let target_subst = match &target {
981                Some(_) => "target=?1,",
982                None => "",
983            };
984            context
985                .sql
986                .execute(
987                    &format!("UPDATE imap SET {target_subst} rfc724_mid=?2 WHERE rfc724_mid=?3"),
988                    (
989                        target.as_deref().unwrap_or_default(),
990                        rfc724_mid_orig,
991                        rfc724_mid,
992                    ),
993                )
994                .await?;
995        }
996        if target.is_none() && !mime_parser.mdn_reports.is_empty() && mime_parser.has_chat_version()
997        {
998            // This is a Delta Chat MDN. Mark as read.
999            markseen_on_imap_table(context, rfc724_mid_orig).await?;
1000        }
1001    }
1002
1003    if received_msg.hidden {
1004        // No need to emit an event about the changed message
1005    } else if let Some(replace_chat_id) = replace_chat_id {
1006        context.emit_msgs_changed_without_msg_id(replace_chat_id);
1007    } else if !chat_id.is_trash() {
1008        let fresh = received_msg.state == MessageState::InFresh;
1009        for msg_id in &received_msg.msg_ids {
1010            chat_id.emit_msg_event(context, *msg_id, mime_parser.incoming && fresh);
1011        }
1012    }
1013    context.new_msgs_notify.notify_one();
1014
1015    mime_parser
1016        .handle_reports(context, from_id, &mime_parser.parts)
1017        .await;
1018
1019    if let Some(is_bot) = mime_parser.is_bot {
1020        // If the message is auto-generated and was generated by Delta Chat,
1021        // mark the contact as a bot.
1022        if mime_parser.get_header(HeaderDef::ChatVersion).is_some() {
1023            from_id.mark_bot(context, is_bot).await?;
1024        }
1025    }
1026
1027    Ok(Some(received_msg))
1028}
1029
1030/// Converts "From" field to contact id.
1031///
1032/// Also returns whether it is blocked or not and its origin.
1033///
1034/// * `prevent_rename`: if true, the display_name of this contact will not be changed. Useful for
1035///   mailing lists: In some mailing lists, many users write from the same address but with different
1036///   display names. We don't want the display name to change every time the user gets a new email from
1037///   a mailing list.
1038///
1039/// * `find_key_contact_by_addr`: if true, we only know the e-mail address
1040///   of the contact, but not the fingerprint,
1041///   yet want to assign the message to some key-contact.
1042///   This can happen during prefetch or when the message is partially downloaded.
1043///   If we get it wrong, the message will be placed into the correct
1044///   chat after downloading.
1045///
1046/// Returns `None` if From field does not contain a valid contact address.
1047pub async fn from_field_to_contact_id(
1048    context: &Context,
1049    from: &SingleInfo,
1050    fingerprint: Option<&Fingerprint>,
1051    prevent_rename: bool,
1052    find_key_contact_by_addr: bool,
1053) -> Result<Option<(ContactId, bool, Origin)>> {
1054    let fingerprint = fingerprint.as_ref().map(|fp| fp.hex()).unwrap_or_default();
1055    let display_name = if prevent_rename {
1056        Some("")
1057    } else {
1058        from.display_name.as_deref()
1059    };
1060    let from_addr = match ContactAddress::new(&from.addr) {
1061        Ok(from_addr) => from_addr,
1062        Err(err) => {
1063            warn!(
1064                context,
1065                "Cannot create a contact for the given From field: {err:#}."
1066            );
1067            return Ok(None);
1068        }
1069    };
1070
1071    if fingerprint.is_empty() && find_key_contact_by_addr {
1072        let addr_normalized = addr_normalize(&from_addr);
1073
1074        // Try to assign to some key-contact.
1075        if let Some((from_id, origin)) = context
1076            .sql
1077            .query_row_optional(
1078                "SELECT id, origin FROM contacts
1079                 WHERE addr=?1 COLLATE NOCASE
1080                 AND fingerprint<>'' -- Only key-contacts
1081                 AND id>?2 AND origin>=?3 AND blocked=?4
1082                 ORDER BY last_seen DESC
1083                 LIMIT 1",
1084                (
1085                    &addr_normalized,
1086                    ContactId::LAST_SPECIAL,
1087                    Origin::IncomingUnknownFrom,
1088                    Blocked::Not,
1089                ),
1090                |row| {
1091                    let id: ContactId = row.get(0)?;
1092                    let origin: Origin = row.get(1)?;
1093                    Ok((id, origin))
1094                },
1095            )
1096            .await?
1097        {
1098            return Ok(Some((from_id, false, origin)));
1099        }
1100    }
1101
1102    let (from_id, _) = Contact::add_or_lookup_ex(
1103        context,
1104        display_name.unwrap_or_default(),
1105        &from_addr,
1106        &fingerprint,
1107        Origin::IncomingUnknownFrom,
1108    )
1109    .await?;
1110
1111    if from_id == ContactId::SELF {
1112        Ok(Some((ContactId::SELF, false, Origin::OutgoingBcc)))
1113    } else {
1114        let contact = Contact::get_by_id(context, from_id).await?;
1115        let from_id_blocked = contact.blocked;
1116        let incoming_origin = contact.origin;
1117
1118        context
1119            .sql
1120            .execute(
1121                "UPDATE contacts SET addr=? WHERE id=?",
1122                (from_addr, from_id),
1123            )
1124            .await?;
1125
1126        Ok(Some((from_id, from_id_blocked, incoming_origin)))
1127    }
1128}
1129
1130async fn decide_chat_assignment(
1131    context: &Context,
1132    mime_parser: &MimeMessage,
1133    parent_message: &Option<Message>,
1134    rfc724_mid: &str,
1135    from_id: ContactId,
1136    is_partial_download: &Option<u32>,
1137) -> Result<ChatAssignment> {
1138    let should_trash = if !mime_parser.mdn_reports.is_empty() {
1139        info!(context, "Message is an MDN (TRASH).");
1140        true
1141    } else if mime_parser.delivery_report.is_some() {
1142        info!(context, "Message is a DSN (TRASH).");
1143        markseen_on_imap_table(context, rfc724_mid).await.ok();
1144        true
1145    } else if mime_parser.get_header(HeaderDef::ChatEdit).is_some()
1146        || mime_parser.get_header(HeaderDef::ChatDelete).is_some()
1147        || mime_parser.get_header(HeaderDef::IrohNodeAddr).is_some()
1148        || mime_parser.sync_items.is_some()
1149    {
1150        info!(context, "Chat edit/delete/iroh/sync message (TRASH).");
1151        true
1152    } else if mime_parser.decrypting_failed && !mime_parser.incoming {
1153        // Outgoing undecryptable message.
1154        let last_time = context
1155            .get_config_i64(Config::LastCantDecryptOutgoingMsgs)
1156            .await?;
1157        let now = tools::time();
1158        let update_config = if last_time.saturating_add(24 * 60 * 60) <= now {
1159            let mut msg = Message::new_text(stock_str::cant_decrypt_outgoing_msgs(context).await);
1160            chat::add_device_msg(context, None, Some(&mut msg))
1161                .await
1162                .log_err(context)
1163                .ok();
1164            true
1165        } else {
1166            last_time > now
1167        };
1168        if update_config {
1169            context
1170                .set_config_internal(Config::LastCantDecryptOutgoingMsgs, Some(&now.to_string()))
1171                .await?;
1172        }
1173        info!(context, "Outgoing undecryptable message (TRASH).");
1174        true
1175    } else if mime_parser.is_system_message != SystemMessage::AutocryptSetupMessage
1176        && !mime_parser.has_chat_version()
1177        && parent_message
1178            .as_ref()
1179            .is_none_or(|p| p.is_dc_message == MessengerMessage::No)
1180        && !context.get_config_bool(Config::IsChatmail).await?
1181        && ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await?)
1182            .unwrap_or_default()
1183            == ShowEmails::Off
1184    {
1185        info!(context, "Classical email not shown (TRASH).");
1186        // the message is a classic email in a classic profile
1187        // (in chatmail profiles, we always show all messages, because shared dc-mua usage is not supported)
1188        true
1189    } else if mime_parser
1190        .get_header(HeaderDef::XMozillaDraftInfo)
1191        .is_some()
1192    {
1193        // Mozilla Thunderbird does not set \Draft flag on "Templates", but sets
1194        // X-Mozilla-Draft-Info header, which can be used to detect both drafts and templates
1195        // created by Thunderbird.
1196
1197        // Most mailboxes have a "Drafts" folder where constantly new emails appear but we don't actually want to show them
1198        info!(context, "Email is probably just a draft (TRASH).");
1199        true
1200    } else if mime_parser.webxdc_status_update.is_some() && mime_parser.parts.len() == 1 {
1201        if let Some(part) = mime_parser.parts.first() {
1202            if part.typ == Viewtype::Text && part.msg.is_empty() {
1203                info!(context, "Message is a status update only (TRASH).");
1204                markseen_on_imap_table(context, rfc724_mid).await.ok();
1205                true
1206            } else {
1207                false
1208            }
1209        } else {
1210            false
1211        }
1212    } else {
1213        false
1214    };
1215
1216    // Decide on the type of chat we assign the message to.
1217    //
1218    // The chat may not exist yet, i.e. there may be
1219    // no database row and ChatId yet.
1220    let mut num_recipients = mime_parser.recipients.len();
1221    if from_id != ContactId::SELF {
1222        let mut has_self_addr = false;
1223        for recipient in &mime_parser.recipients {
1224            if context.is_self_addr(&recipient.addr).await? {
1225                has_self_addr = true;
1226            }
1227        }
1228        if !has_self_addr {
1229            num_recipients += 1;
1230        }
1231    }
1232
1233    let chat_assignment = if should_trash {
1234        ChatAssignment::Trash
1235    } else if mime_parser.get_mailinglist_header().is_some() {
1236        ChatAssignment::MailingListOrBroadcast
1237    } else if let Some(grpid) = mime_parser.get_chat_group_id() {
1238        if mime_parser.was_encrypted() {
1239            ChatAssignment::GroupChat {
1240                grpid: grpid.to_string(),
1241            }
1242        } else if let Some(parent) = &parent_message {
1243            if let Some((chat_id, chat_id_blocked)) =
1244                lookup_chat_by_reply(context, mime_parser, parent, is_partial_download).await?
1245            {
1246                // Try to assign to a chat based on In-Reply-To/References.
1247                ChatAssignment::ExistingChat {
1248                    chat_id,
1249                    chat_id_blocked,
1250                }
1251            } else {
1252                ChatAssignment::AdHocGroup
1253            }
1254        } else {
1255            // Could be a message from old version
1256            // with opportunistic encryption.
1257            //
1258            // We still want to assign this to a group
1259            // even if it had only two members.
1260            //
1261            // Group ID is ignored, however.
1262            ChatAssignment::AdHocGroup
1263        }
1264    } else if let Some(parent) = &parent_message {
1265        if let Some((chat_id, chat_id_blocked)) =
1266            lookup_chat_by_reply(context, mime_parser, parent, is_partial_download).await?
1267        {
1268            // Try to assign to a chat based on In-Reply-To/References.
1269            ChatAssignment::ExistingChat {
1270                chat_id,
1271                chat_id_blocked,
1272            }
1273        } else if mime_parser.get_header(HeaderDef::ChatGroupName).is_some() {
1274            ChatAssignment::AdHocGroup
1275        } else if num_recipients <= 1 {
1276            ChatAssignment::OneOneChat
1277        } else {
1278            ChatAssignment::AdHocGroup
1279        }
1280    } else if mime_parser.get_header(HeaderDef::ChatGroupName).is_some() {
1281        ChatAssignment::AdHocGroup
1282    } else if num_recipients <= 1 {
1283        ChatAssignment::OneOneChat
1284    } else {
1285        ChatAssignment::AdHocGroup
1286    };
1287    Ok(chat_assignment)
1288}
1289
1290/// Assigns the message to a chat.
1291///
1292/// Creates a new chat if necessary.
1293#[expect(clippy::too_many_arguments)]
1294async fn do_chat_assignment(
1295    context: &Context,
1296    chat_assignment: ChatAssignment,
1297    from_id: ContactId,
1298    to_ids: &[Option<ContactId>],
1299    past_ids: &[Option<ContactId>],
1300    to_id: ContactId,
1301    allow_creation: bool,
1302    mime_parser: &mut MimeMessage,
1303    is_partial_download: Option<u32>,
1304    verified_encryption: &VerifiedEncryption,
1305    parent_message: Option<Message>,
1306) -> Result<(ChatId, Blocked)> {
1307    let is_bot = context.get_config_bool(Config::Bot).await?;
1308
1309    let mut chat_id = None;
1310    let mut chat_id_blocked = Blocked::Not;
1311
1312    if mime_parser.incoming {
1313        let test_normal_chat = ChatIdBlocked::lookup_by_contact(context, from_id).await?;
1314
1315        let create_blocked_default = if is_bot {
1316            Blocked::Not
1317        } else {
1318            Blocked::Request
1319        };
1320        let create_blocked = if let Some(ChatIdBlocked { id: _, blocked }) = test_normal_chat {
1321            match blocked {
1322                Blocked::Request => create_blocked_default,
1323                Blocked::Not => Blocked::Not,
1324                Blocked::Yes => {
1325                    if Contact::is_blocked_load(context, from_id).await? {
1326                        // User has blocked the contact.
1327                        // Block the group contact created as well.
1328                        Blocked::Yes
1329                    } else {
1330                        // 1:1 chat is blocked, but the contact is not.
1331                        // This happens when 1:1 chat is hidden
1332                        // during scanning of a group invitation code.
1333                        create_blocked_default
1334                    }
1335                }
1336            }
1337        } else {
1338            create_blocked_default
1339        };
1340
1341        match &chat_assignment {
1342            ChatAssignment::Trash => {
1343                chat_id = Some(DC_CHAT_ID_TRASH);
1344            }
1345            ChatAssignment::GroupChat { grpid } => {
1346                // Try to assign to a chat based on Chat-Group-ID.
1347                if let Some((id, _protected, blocked)) =
1348                    chat::get_chat_id_by_grpid(context, grpid).await?
1349                {
1350                    chat_id = Some(id);
1351                    chat_id_blocked = blocked;
1352                } else if allow_creation || test_normal_chat.is_some() {
1353                    if let Some((new_chat_id, new_chat_id_blocked)) = create_group(
1354                        context,
1355                        mime_parser,
1356                        is_partial_download.is_some(),
1357                        create_blocked,
1358                        from_id,
1359                        to_ids,
1360                        past_ids,
1361                        verified_encryption,
1362                        grpid,
1363                    )
1364                    .await?
1365                    {
1366                        chat_id = Some(new_chat_id);
1367                        chat_id_blocked = new_chat_id_blocked;
1368                    }
1369                }
1370            }
1371            ChatAssignment::MailingListOrBroadcast => {
1372                if let Some(mailinglist_header) = mime_parser.get_mailinglist_header() {
1373                    if let Some((new_chat_id, new_chat_id_blocked)) =
1374                        create_or_lookup_mailinglist_or_broadcast(
1375                            context,
1376                            allow_creation,
1377                            mailinglist_header,
1378                            from_id,
1379                            mime_parser,
1380                        )
1381                        .await?
1382                    {
1383                        chat_id = Some(new_chat_id);
1384                        chat_id_blocked = new_chat_id_blocked;
1385
1386                        apply_mailinglist_changes(context, mime_parser, new_chat_id).await?;
1387                    }
1388                }
1389            }
1390            ChatAssignment::ExistingChat {
1391                chat_id: new_chat_id,
1392                chat_id_blocked: new_chat_id_blocked,
1393            } => {
1394                chat_id = Some(*new_chat_id);
1395                chat_id_blocked = *new_chat_id_blocked;
1396            }
1397            ChatAssignment::AdHocGroup => {
1398                if let Some((new_chat_id, new_chat_id_blocked)) = lookup_or_create_adhoc_group(
1399                    context,
1400                    mime_parser,
1401                    to_ids,
1402                    allow_creation || test_normal_chat.is_some(),
1403                    create_blocked,
1404                    is_partial_download.is_some(),
1405                )
1406                .await?
1407                {
1408                    chat_id = Some(new_chat_id);
1409                    chat_id_blocked = new_chat_id_blocked;
1410                }
1411            }
1412            ChatAssignment::OneOneChat => {}
1413        }
1414
1415        // if the chat is somehow blocked but we want to create a non-blocked chat,
1416        // unblock the chat
1417        if chat_id_blocked != Blocked::Not
1418            && create_blocked != Blocked::Yes
1419            && !matches!(chat_assignment, ChatAssignment::MailingListOrBroadcast)
1420        {
1421            if let Some(chat_id) = chat_id {
1422                chat_id.set_blocked(context, create_blocked).await?;
1423                chat_id_blocked = create_blocked;
1424            }
1425        }
1426
1427        if chat_id.is_none() {
1428            // Try to create a 1:1 chat.
1429            let contact = Contact::get_by_id(context, from_id).await?;
1430            let create_blocked = match contact.is_blocked() {
1431                true => Blocked::Yes,
1432                false if is_bot => Blocked::Not,
1433                false => Blocked::Request,
1434            };
1435
1436            if let Some(chat) = test_normal_chat {
1437                chat_id = Some(chat.id);
1438                chat_id_blocked = chat.blocked;
1439            } else if allow_creation {
1440                let chat = ChatIdBlocked::get_for_contact(context, from_id, create_blocked)
1441                    .await
1442                    .context("Failed to get (new) chat for contact")?;
1443                chat_id = Some(chat.id);
1444                chat_id_blocked = chat.blocked;
1445            }
1446
1447            if let Some(chat_id) = chat_id {
1448                if chat_id_blocked != Blocked::Not {
1449                    if chat_id_blocked != create_blocked {
1450                        chat_id.set_blocked(context, create_blocked).await?;
1451                    }
1452                    if create_blocked == Blocked::Request && parent_message.is_some() {
1453                        // we do not want any chat to be created implicitly.  Because of the origin-scale-up,
1454                        // the contact requests will pop up and this should be just fine.
1455                        ContactId::scaleup_origin(context, &[from_id], Origin::IncomingReplyTo)
1456                            .await?;
1457                        info!(
1458                            context,
1459                            "Message is a reply to a known message, mark sender as known.",
1460                        );
1461                    }
1462                }
1463
1464                // Check if the message was sent with verified encryption and set the protection of
1465                // the 1:1 chat accordingly.
1466                let chat = match is_partial_download.is_none()
1467                    && mime_parser.get_header(HeaderDef::SecureJoin).is_none()
1468                {
1469                    true => Some(Chat::load_from_db(context, chat_id).await?)
1470                        .filter(|chat| chat.typ == Chattype::Single),
1471                    false => None,
1472                };
1473                if let Some(chat) = chat {
1474                    ensure_and_debug_assert!(
1475                        chat.typ == Chattype::Single,
1476                        "Chat {chat_id} is not Single",
1477                    );
1478                    let new_protection = match verified_encryption {
1479                        VerifiedEncryption::Verified => ProtectionStatus::Protected,
1480                        VerifiedEncryption::NotVerified(_) => ProtectionStatus::Unprotected,
1481                    };
1482
1483                    ensure_and_debug_assert!(
1484                        chat.protected == ProtectionStatus::Unprotected
1485                            || new_protection == ProtectionStatus::Protected,
1486                        "Chat {chat_id} can't downgrade to Unprotected",
1487                    );
1488                    if chat.protected != new_protection {
1489                        // The message itself will be sorted under the device message since the device
1490                        // message is `MessageState::InNoticed`, which means that all following
1491                        // messages are sorted under it.
1492                        chat_id
1493                            .set_protection(
1494                                context,
1495                                new_protection,
1496                                mime_parser.timestamp_sent,
1497                                Some(from_id),
1498                            )
1499                            .await?;
1500                    }
1501                }
1502            }
1503        }
1504    } else {
1505        // Outgoing
1506
1507        // Older Delta Chat versions with core <=1.152.2 only accepted
1508        // self-sent messages in Saved Messages with own address in the `To` field.
1509        // New Delta Chat versions may use empty `To` field
1510        // with only a single `hidden-recipients` group in this case.
1511        let self_sent = to_ids.len() <= 1 && to_id == ContactId::SELF;
1512
1513        match &chat_assignment {
1514            ChatAssignment::Trash => {
1515                chat_id = Some(DC_CHAT_ID_TRASH);
1516            }
1517            ChatAssignment::GroupChat { grpid } => {
1518                if let Some((id, _protected, blocked)) =
1519                    chat::get_chat_id_by_grpid(context, grpid).await?
1520                {
1521                    chat_id = Some(id);
1522                    chat_id_blocked = blocked;
1523                } else if allow_creation {
1524                    if let Some((new_chat_id, new_chat_id_blocked)) = create_group(
1525                        context,
1526                        mime_parser,
1527                        is_partial_download.is_some(),
1528                        Blocked::Not,
1529                        from_id,
1530                        to_ids,
1531                        past_ids,
1532                        verified_encryption,
1533                        grpid,
1534                    )
1535                    .await?
1536                    {
1537                        chat_id = Some(new_chat_id);
1538                        chat_id_blocked = new_chat_id_blocked;
1539                    }
1540                }
1541            }
1542            ChatAssignment::ExistingChat {
1543                chat_id: new_chat_id,
1544                chat_id_blocked: new_chat_id_blocked,
1545            } => {
1546                chat_id = Some(*new_chat_id);
1547                chat_id_blocked = *new_chat_id_blocked;
1548            }
1549            ChatAssignment::MailingListOrBroadcast => {
1550                // Check if the message belongs to a broadcast channel
1551                // (it can't be a mailing list, since it's outgoing)
1552                if let Some(mailinglist_header) = mime_parser.get_mailinglist_header() {
1553                    let listid = mailinglist_header_listid(mailinglist_header)?;
1554                    chat_id = Some(
1555                        if let Some((id, ..)) = chat::get_chat_id_by_grpid(context, &listid).await?
1556                        {
1557                            id
1558                        } else {
1559                            let name =
1560                                compute_mailinglist_name(mailinglist_header, &listid, mime_parser);
1561                            chat::create_broadcast_ex(context, Nosync, listid, name).await?
1562                        },
1563                    );
1564                }
1565            }
1566            ChatAssignment::AdHocGroup => {
1567                if let Some((new_chat_id, new_chat_id_blocked)) = lookup_or_create_adhoc_group(
1568                    context,
1569                    mime_parser,
1570                    to_ids,
1571                    allow_creation,
1572                    Blocked::Not,
1573                    is_partial_download.is_some(),
1574                )
1575                .await?
1576                {
1577                    chat_id = Some(new_chat_id);
1578                    chat_id_blocked = new_chat_id_blocked;
1579                }
1580            }
1581            ChatAssignment::OneOneChat => {}
1582        }
1583
1584        if !to_ids.is_empty() {
1585            if chat_id.is_none() && allow_creation {
1586                let to_contact = Contact::get_by_id(context, to_id).await?;
1587                if let Some(list_id) = to_contact.param.get(Param::ListId) {
1588                    if let Some((id, _, blocked)) =
1589                        chat::get_chat_id_by_grpid(context, list_id).await?
1590                    {
1591                        chat_id = Some(id);
1592                        chat_id_blocked = blocked;
1593                    }
1594                } else {
1595                    let chat = ChatIdBlocked::get_for_contact(context, to_id, Blocked::Not).await?;
1596                    chat_id = Some(chat.id);
1597                    chat_id_blocked = chat.blocked;
1598                }
1599            }
1600            if chat_id.is_none() && mime_parser.has_chat_version() {
1601                if let Some(chat) = ChatIdBlocked::lookup_by_contact(context, to_id).await? {
1602                    chat_id = Some(chat.id);
1603                    chat_id_blocked = chat.blocked;
1604                }
1605            }
1606        }
1607
1608        if chat_id.is_none() && self_sent {
1609            // from_id==to_id==ContactId::SELF - this is a self-sent messages,
1610            // maybe an Autocrypt Setup Message
1611            let chat = ChatIdBlocked::get_for_contact(context, ContactId::SELF, Blocked::Not)
1612                .await
1613                .context("Failed to get (new) chat for contact")?;
1614
1615            chat_id = Some(chat.id);
1616            chat_id_blocked = chat.blocked;
1617
1618            if Blocked::Not != chat.blocked {
1619                chat.id.unblock_ex(context, Nosync).await?;
1620            }
1621        }
1622
1623        // automatically unblock chat when the user sends a message
1624        if chat_id_blocked != Blocked::Not {
1625            if let Some(chat_id) = chat_id {
1626                chat_id.unblock_ex(context, Nosync).await?;
1627                chat_id_blocked = Blocked::Not;
1628            }
1629        }
1630    }
1631    let chat_id = chat_id.unwrap_or_else(|| {
1632        info!(context, "No chat id for message (TRASH).");
1633        DC_CHAT_ID_TRASH
1634    });
1635    Ok((chat_id, chat_id_blocked))
1636}
1637
1638/// Creates a `ReceivedMsg` from given parts which might consist of
1639/// multiple messages (if there are multiple attachments).
1640/// Every entry in `mime_parser.parts` produces a new row in the `msgs` table.
1641#[expect(clippy::too_many_arguments)]
1642async fn add_parts(
1643    context: &Context,
1644    mime_parser: &mut MimeMessage,
1645    imf_raw: &[u8],
1646    to_ids: &[Option<ContactId>],
1647    past_ids: &[Option<ContactId>],
1648    rfc724_mid: &str,
1649    from_id: ContactId,
1650    seen: bool,
1651    is_partial_download: Option<u32>,
1652    mut replace_msg_id: Option<MsgId>,
1653    prevent_rename: bool,
1654    verified_encryption: VerifiedEncryption,
1655    chat_id: ChatId,
1656    chat_id_blocked: Blocked,
1657    is_dc_message: MessengerMessage,
1658) -> Result<ReceivedMsg> {
1659    let to_id = if mime_parser.incoming {
1660        ContactId::SELF
1661    } else {
1662        to_ids.first().copied().flatten().unwrap_or(ContactId::SELF)
1663    };
1664
1665    // if contact renaming is prevented (for mailinglists and bots),
1666    // we use name from From:-header as override name
1667    if prevent_rename {
1668        if let Some(name) = &mime_parser.from.display_name {
1669            for part in &mut mime_parser.parts {
1670                part.param.set(Param::OverrideSenderDisplayname, name);
1671            }
1672        }
1673    }
1674
1675    let mut chat = Chat::load_from_db(context, chat_id).await?;
1676
1677    if mime_parser.incoming && !chat_id.is_trash() {
1678        // It can happen that the message is put into a chat
1679        // but the From-address is not a member of this chat.
1680        if !chat::is_contact_in_chat(context, chat_id, from_id).await? {
1681            // Mark the sender as overridden.
1682            // The UI will prepend `~` to the sender's name,
1683            // indicating that the sender is not part of the group.
1684            let from = &mime_parser.from;
1685            let name: &str = from.display_name.as_ref().unwrap_or(&from.addr);
1686            for part in &mut mime_parser.parts {
1687                part.param.set(Param::OverrideSenderDisplayname, name);
1688
1689                if chat.is_protected() {
1690                    // In protected chat, also mark the message with an error.
1691                    let s = stock_str::unknown_sender_for_chat(context).await;
1692                    part.error = Some(s);
1693                }
1694            }
1695        }
1696    }
1697
1698    let is_location_kml = mime_parser.location_kml.is_some();
1699    let is_mdn = !mime_parser.mdn_reports.is_empty();
1700
1701    let mut group_changes = match chat.typ {
1702        _ if chat.id.is_special() => GroupChangesInfo::default(),
1703        Chattype::Single => GroupChangesInfo::default(),
1704        Chattype::Mailinglist => GroupChangesInfo::default(),
1705        Chattype::OutBroadcast => {
1706            apply_out_broadcast_changes(context, mime_parser, &mut chat, from_id).await?
1707        }
1708        Chattype::Group => {
1709            apply_group_changes(
1710                context,
1711                mime_parser,
1712                &mut chat,
1713                from_id,
1714                to_ids,
1715                past_ids,
1716                &verified_encryption,
1717            )
1718            .await?
1719        }
1720        Chattype::InBroadcast => {
1721            apply_in_broadcast_changes(context, mime_parser, &mut chat, from_id).await?
1722        }
1723    };
1724
1725    let rfc724_mid_orig = &mime_parser
1726        .get_rfc724_mid()
1727        .unwrap_or(rfc724_mid.to_string());
1728
1729    // Extract ephemeral timer from the message or use the existing timer if the message is not fully downloaded.
1730    let mut ephemeral_timer = if is_partial_download.is_some() {
1731        chat_id.get_ephemeral_timer(context).await?
1732    } else if let Some(value) = mime_parser.get_header(HeaderDef::EphemeralTimer) {
1733        match value.parse::<EphemeralTimer>() {
1734            Ok(timer) => timer,
1735            Err(err) => {
1736                warn!(context, "Can't parse ephemeral timer \"{value}\": {err:#}.");
1737                EphemeralTimer::Disabled
1738            }
1739        }
1740    } else {
1741        EphemeralTimer::Disabled
1742    };
1743
1744    let state = if !mime_parser.incoming {
1745        MessageState::OutDelivered
1746    } else if seen || is_mdn || chat_id_blocked == Blocked::Yes || group_changes.silent
1747    // No check for `hidden` because only reactions are such and they should be `InFresh`.
1748    {
1749        MessageState::InSeen
1750    } else {
1751        MessageState::InFresh
1752    };
1753    let in_fresh = state == MessageState::InFresh;
1754
1755    let sort_to_bottom = false;
1756    let received = true;
1757    let sort_timestamp = chat_id
1758        .calc_sort_timestamp(
1759            context,
1760            mime_parser.timestamp_sent,
1761            sort_to_bottom,
1762            received,
1763            mime_parser.incoming,
1764        )
1765        .await?;
1766
1767    // Apply ephemeral timer changes to the chat.
1768    //
1769    // Only apply the timer when there are visible parts (e.g., the message does not consist only
1770    // of `location.kml` attachment).  Timer changes without visible received messages may be
1771    // confusing to the user.
1772    if !chat_id.is_special()
1773        && !mime_parser.parts.is_empty()
1774        && chat_id.get_ephemeral_timer(context).await? != ephemeral_timer
1775    {
1776        let chat_contacts =
1777            HashSet::<ContactId>::from_iter(chat::get_chat_contacts(context, chat_id).await?);
1778        let is_from_in_chat =
1779            !chat_contacts.contains(&ContactId::SELF) || chat_contacts.contains(&from_id);
1780
1781        info!(
1782            context,
1783            "Received new ephemeral timer value {ephemeral_timer:?} for chat {chat_id}, checking if it should be applied."
1784        );
1785        if !is_from_in_chat {
1786            warn!(
1787                context,
1788                "Ignoring ephemeral timer change to {ephemeral_timer:?} for chat {chat_id} because sender {from_id} is not a member.",
1789            );
1790        } else if is_dc_message == MessengerMessage::Yes
1791            && get_previous_message(context, mime_parser)
1792                .await?
1793                .map(|p| p.ephemeral_timer)
1794                == Some(ephemeral_timer)
1795            && mime_parser.is_system_message != SystemMessage::EphemeralTimerChanged
1796        {
1797            // The message is a Delta Chat message, so we know that previous message according to
1798            // References header is the last message in the chat as seen by the sender. The timer
1799            // is the same in both the received message and the last message, so we know that the
1800            // sender has not seen any change of the timer between these messages. As our timer
1801            // value is different, it means the sender has not received some timer update that we
1802            // have seen or sent ourselves, so we ignore incoming timer to prevent a rollback.
1803            warn!(
1804                context,
1805                "Ignoring ephemeral timer change to {ephemeral_timer:?} for chat {chat_id} to avoid rollback.",
1806            );
1807        } else if chat_id
1808            .update_timestamp(
1809                context,
1810                Param::EphemeralSettingsTimestamp,
1811                mime_parser.timestamp_sent,
1812            )
1813            .await?
1814        {
1815            if let Err(err) = chat_id
1816                .inner_set_ephemeral_timer(context, ephemeral_timer)
1817                .await
1818            {
1819                warn!(
1820                    context,
1821                    "Failed to modify timer for chat {chat_id}: {err:#}."
1822                );
1823            } else {
1824                info!(
1825                    context,
1826                    "Updated ephemeral timer to {ephemeral_timer:?} for chat {chat_id}."
1827                );
1828                if mime_parser.is_system_message != SystemMessage::EphemeralTimerChanged {
1829                    chat::add_info_msg(
1830                        context,
1831                        chat_id,
1832                        &stock_ephemeral_timer_changed(context, ephemeral_timer, from_id).await,
1833                        sort_timestamp,
1834                    )
1835                    .await?;
1836                }
1837            }
1838        } else {
1839            warn!(
1840                context,
1841                "Ignoring ephemeral timer change to {ephemeral_timer:?} because it is outdated."
1842            );
1843        }
1844    }
1845
1846    let mut better_msg = if mime_parser.is_system_message == SystemMessage::LocationStreamingEnabled
1847    {
1848        Some(stock_str::msg_location_enabled_by(context, from_id).await)
1849    } else if mime_parser.is_system_message == SystemMessage::EphemeralTimerChanged {
1850        let better_msg = stock_ephemeral_timer_changed(context, ephemeral_timer, from_id).await;
1851
1852        // Do not delete the system message itself.
1853        //
1854        // This prevents confusion when timer is changed
1855        // to 1 week, and then changed to 1 hour: after 1
1856        // hour, only the message about the change to 1
1857        // week is left.
1858        ephemeral_timer = EphemeralTimer::Disabled;
1859
1860        Some(better_msg)
1861    } else {
1862        None
1863    };
1864
1865    let mut verification_failed = false;
1866    if !chat_id.is_special() && is_partial_download.is_none() {
1867        // For outgoing emails in the 1:1 chat we have an exception that
1868        // they are allowed to be unencrypted:
1869        // 1. They can't be an attack (they are outgoing, not incoming)
1870        // 2. Probably the unencryptedness is just a temporary state, after all
1871        //    the user obviously still uses DC
1872        //    -> Showing info messages every time would be a lot of noise
1873        // 3. The info messages that are shown to the user ("Your chat partner
1874        //    likely reinstalled DC" or similar) would be wrong.
1875        if chat.is_protected() && (mime_parser.incoming || chat.typ != Chattype::Single) {
1876            if let VerifiedEncryption::NotVerified(err) = verified_encryption {
1877                verification_failed = true;
1878                warn!(context, "Verification problem: {err:#}.");
1879                let s = format!("{err}. Re-download the message or see 'Info' for more details");
1880                mime_parser.replace_msg_by_error(&s);
1881            }
1882        }
1883    }
1884    drop(chat); // Avoid using stale `chat` object.
1885
1886    let sort_timestamp = tweak_sort_timestamp(
1887        context,
1888        mime_parser,
1889        group_changes.silent,
1890        chat_id,
1891        sort_timestamp,
1892    )
1893    .await?;
1894
1895    let mime_in_reply_to = mime_parser
1896        .get_header(HeaderDef::InReplyTo)
1897        .unwrap_or_default();
1898    let mime_references = mime_parser
1899        .get_header(HeaderDef::References)
1900        .unwrap_or_default();
1901
1902    // fine, so far.  now, split the message into simple parts usable as "short messages"
1903    // and add them to the database (mails sent by other messenger clients should result
1904    // into only one message; mails sent by other clients may result in several messages
1905    // (eg. one per attachment))
1906    let icnt = mime_parser.parts.len();
1907
1908    let subject = mime_parser.get_subject().unwrap_or_default();
1909
1910    let is_system_message = mime_parser.is_system_message;
1911
1912    // if indicated by the parser,
1913    // we save the full mime-message and add a flag
1914    // that the ui should show button to display the full message.
1915
1916    // We add "Show Full Message" button to the last message bubble (part) if this flag evaluates to
1917    // `true` finally.
1918    let mut save_mime_modified = false;
1919
1920    let mime_headers = if mime_parser.is_mime_modified {
1921        let headers = if !mime_parser.decoded_data.is_empty() {
1922            mime_parser.decoded_data.clone()
1923        } else {
1924            imf_raw.to_vec()
1925        };
1926        tokio::task::block_in_place(move || buf_compress(&headers))?
1927    } else {
1928        Vec::new()
1929    };
1930
1931    let mut created_db_entries = Vec::with_capacity(mime_parser.parts.len());
1932
1933    if let Some(m) = group_changes.better_msg {
1934        match &better_msg {
1935            None => better_msg = Some(m),
1936            Some(_) => {
1937                if !m.is_empty() {
1938                    group_changes.extra_msgs.push((m, is_system_message, None))
1939                }
1940            }
1941        }
1942    }
1943
1944    let chat_id = if better_msg
1945        .as_ref()
1946        .is_some_and(|better_msg| better_msg.is_empty())
1947        && is_partial_download.is_none()
1948    {
1949        DC_CHAT_ID_TRASH
1950    } else {
1951        chat_id
1952    };
1953
1954    for (group_changes_msg, cmd, added_removed_id) in group_changes.extra_msgs {
1955        chat::add_info_msg_with_cmd(
1956            context,
1957            chat_id,
1958            &group_changes_msg,
1959            cmd,
1960            sort_timestamp,
1961            None,
1962            None,
1963            None,
1964            added_removed_id,
1965        )
1966        .await?;
1967    }
1968
1969    if let Some(node_addr) = mime_parser.get_header(HeaderDef::IrohNodeAddr) {
1970        match mime_parser.get_header(HeaderDef::InReplyTo) {
1971            Some(in_reply_to) => match rfc724_mid_exists(context, in_reply_to).await? {
1972                Some((instance_id, _ts_sent)) => {
1973                    if let Err(err) =
1974                        add_gossip_peer_from_header(context, instance_id, node_addr).await
1975                    {
1976                        warn!(context, "Failed to add iroh peer from header: {err:#}.");
1977                    }
1978                }
1979                None => {
1980                    warn!(
1981                        context,
1982                        "Cannot add iroh peer because WebXDC instance does not exist."
1983                    );
1984                }
1985            },
1986            None => {
1987                warn!(
1988                    context,
1989                    "Cannot add iroh peer because the message has no In-Reply-To."
1990                );
1991            }
1992        }
1993    }
1994
1995    handle_edit_delete(context, mime_parser, from_id).await?;
1996
1997    let hidden = mime_parser.parts.iter().all(|part| part.is_reaction);
1998    let mut parts = mime_parser.parts.iter().peekable();
1999    while let Some(part) = parts.next() {
2000        let hidden = part.is_reaction;
2001        if part.is_reaction {
2002            let reaction_str = simplify::remove_footers(part.msg.as_str());
2003            let is_incoming_fresh = mime_parser.incoming && !seen;
2004            set_msg_reaction(
2005                context,
2006                mime_in_reply_to,
2007                chat_id,
2008                from_id,
2009                sort_timestamp,
2010                Reaction::from(reaction_str.as_str()),
2011                is_incoming_fresh,
2012            )
2013            .await?;
2014        }
2015
2016        let mut param = part.param.clone();
2017        if is_system_message != SystemMessage::Unknown {
2018            param.set_int(Param::Cmd, is_system_message as i32);
2019        }
2020
2021        if let Some(replace_msg_id) = replace_msg_id {
2022            let placeholder = Message::load_from_db(context, replace_msg_id).await?;
2023            for key in [
2024                Param::WebxdcSummary,
2025                Param::WebxdcSummaryTimestamp,
2026                Param::WebxdcDocument,
2027                Param::WebxdcDocumentTimestamp,
2028            ] {
2029                if let Some(value) = placeholder.param.get(key) {
2030                    param.set(key, value);
2031                }
2032            }
2033        }
2034
2035        let (msg, typ): (&str, Viewtype) = if let Some(better_msg) = &better_msg {
2036            (better_msg, Viewtype::Text)
2037        } else {
2038            (&part.msg, part.typ)
2039        };
2040        let part_is_empty =
2041            typ == Viewtype::Text && msg.is_empty() && part.param.get(Param::Quote).is_none();
2042
2043        if let Some(contact_id) = group_changes.added_removed_id {
2044            param.set(Param::ContactAddedRemoved, contact_id.to_u32().to_string());
2045        }
2046
2047        save_mime_modified |= mime_parser.is_mime_modified && !part_is_empty && !hidden;
2048        let save_mime_modified = save_mime_modified && parts.peek().is_none();
2049
2050        let ephemeral_timestamp = if in_fresh {
2051            0
2052        } else {
2053            match ephemeral_timer {
2054                EphemeralTimer::Disabled => 0,
2055                EphemeralTimer::Enabled { duration } => {
2056                    mime_parser.timestamp_rcvd.saturating_add(duration.into())
2057                }
2058            }
2059        };
2060
2061        // If you change which information is skipped if the message is trashed,
2062        // also change `MsgId::trash()` and `delete_expired_messages()`
2063        let trash = chat_id.is_trash() || (is_location_kml && part_is_empty && !save_mime_modified);
2064
2065        let row_id = context
2066            .sql
2067            .call_write(|conn| {
2068                let mut stmt = conn.prepare_cached(
2069            r#"
2070INSERT INTO msgs
2071  (
2072    id,
2073    rfc724_mid, chat_id,
2074    from_id, to_id, timestamp, timestamp_sent, 
2075    timestamp_rcvd, type, state, msgrmsg, 
2076    txt, txt_normalized, subject, param, hidden,
2077    bytes, mime_headers, mime_compressed, mime_in_reply_to,
2078    mime_references, mime_modified, error, ephemeral_timer,
2079    ephemeral_timestamp, download_state, hop_info
2080  )
2081  VALUES (
2082    ?,
2083    ?, ?, ?, ?,
2084    ?, ?, ?, ?,
2085    ?, ?, ?, ?,
2086    ?, ?, ?, ?, ?, 1,
2087    ?, ?, ?, ?,
2088    ?, ?, ?, ?
2089  )
2090ON CONFLICT (id) DO UPDATE
2091SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id,
2092    from_id=excluded.from_id, to_id=excluded.to_id, timestamp_sent=excluded.timestamp_sent,
2093    type=excluded.type, state=max(state,excluded.state), msgrmsg=excluded.msgrmsg,
2094    txt=excluded.txt, txt_normalized=excluded.txt_normalized, subject=excluded.subject,
2095    param=excluded.param,
2096    hidden=excluded.hidden,bytes=excluded.bytes, mime_headers=excluded.mime_headers,
2097    mime_compressed=excluded.mime_compressed, mime_in_reply_to=excluded.mime_in_reply_to,
2098    mime_references=excluded.mime_references, mime_modified=excluded.mime_modified, error=excluded.error, ephemeral_timer=excluded.ephemeral_timer,
2099    ephemeral_timestamp=excluded.ephemeral_timestamp, download_state=excluded.download_state, hop_info=excluded.hop_info
2100RETURNING id
2101"#)?;
2102                let row_id: MsgId = stmt.query_row(params![
2103                    replace_msg_id,
2104                    rfc724_mid_orig,
2105                    if trash { DC_CHAT_ID_TRASH } else { chat_id },
2106                    if trash { ContactId::UNDEFINED } else { from_id },
2107                    if trash { ContactId::UNDEFINED } else { to_id },
2108                    sort_timestamp,
2109                    if trash { 0 } else { mime_parser.timestamp_sent },
2110                    if trash { 0 } else { mime_parser.timestamp_rcvd },
2111                    if trash { Viewtype::Unknown } else { typ },
2112                    if trash { MessageState::Undefined } else { state },
2113                    if trash { MessengerMessage::No } else { is_dc_message },
2114                    if trash || hidden { "" } else { msg },
2115                    if trash || hidden { None } else { message::normalize_text(msg) },
2116                    if trash || hidden { "" } else { &subject },
2117                    if trash {
2118                        "".to_string()
2119                    } else {
2120                        param.to_string()
2121                    },
2122                    !trash && hidden,
2123                    if trash { 0 } else { part.bytes as isize },
2124                    if save_mime_modified && !(trash || hidden) {
2125                        mime_headers.clone()
2126                    } else {
2127                        Vec::new()
2128                    },
2129                    if trash { "" } else { mime_in_reply_to },
2130                    if trash { "" } else { mime_references },
2131                    !trash && save_mime_modified,
2132                    if trash { "" } else { part.error.as_deref().unwrap_or_default() },
2133                    if trash { 0 } else { ephemeral_timer.to_u32() },
2134                    if trash { 0 } else { ephemeral_timestamp },
2135                    if trash {
2136                        DownloadState::Done
2137                    } else if is_partial_download.is_some() {
2138                        DownloadState::Available
2139                    } else if mime_parser.decrypting_failed {
2140                        DownloadState::Undecipherable
2141                    } else if verification_failed {
2142                        // Verification can fail because of message reordering. Re-downloading the
2143                        // message should help if so.
2144                        DownloadState::Available
2145                    } else {
2146                        DownloadState::Done
2147                    },
2148                    if trash { "" } else { &mime_parser.hop_info },
2149                ],
2150                |row| {
2151                    let msg_id: MsgId = row.get(0)?;
2152                    Ok(msg_id)
2153                }
2154                )?;
2155                Ok(row_id)
2156            })
2157            .await?;
2158
2159        // We only replace placeholder with a first part,
2160        // afterwards insert additional parts.
2161        replace_msg_id = None;
2162
2163        ensure_and_debug_assert!(!row_id.is_special(), "Rowid {row_id} is special");
2164        created_db_entries.push(row_id);
2165    }
2166
2167    // Maybe set logging xdc and add gossip topics for webxdcs.
2168    for (part, msg_id) in mime_parser.parts.iter().zip(&created_db_entries) {
2169        // check if any part contains a webxdc topic id
2170        if part.typ == Viewtype::Webxdc {
2171            if let Some(topic) = mime_parser.get_header(HeaderDef::IrohGossipTopic) {
2172                // default encoding of topic ids is `hex`.
2173                let mut topic_raw = [0u8; 32];
2174                BASE32_NOPAD
2175                    .decode_mut(topic.to_ascii_uppercase().as_bytes(), &mut topic_raw)
2176                    .map_err(|e| e.error)
2177                    .context("Wrong gossip topic header")?;
2178
2179                let topic = TopicId::from_bytes(topic_raw);
2180                insert_topic_stub(context, *msg_id, topic).await?;
2181            } else {
2182                warn!(context, "webxdc doesn't have a gossip topic")
2183            }
2184        }
2185
2186        maybe_set_logging_xdc_inner(
2187            context,
2188            part.typ,
2189            chat_id,
2190            part.param.get(Param::Filename),
2191            *msg_id,
2192        )
2193        .await?;
2194    }
2195
2196    if let Some(replace_msg_id) = replace_msg_id {
2197        // Trash the "replace" placeholder with a message that has no parts. If it has the original
2198        // "Message-ID", mark the placeholder for server-side deletion so as if the user deletes the
2199        // fully downloaded message later, the server-side deletion is issued.
2200        let on_server = rfc724_mid == rfc724_mid_orig;
2201        replace_msg_id.trash(context, on_server).await?;
2202    }
2203
2204    let unarchive = match mime_parser.get_header(HeaderDef::ChatGroupMemberRemoved) {
2205        Some(addr) => context.is_self_addr(addr).await?,
2206        None => true,
2207    };
2208    if unarchive {
2209        chat_id.unarchive_if_not_muted(context, state).await?;
2210    }
2211
2212    info!(
2213        context,
2214        "Message has {icnt} parts and is assigned to chat #{chat_id}."
2215    );
2216
2217    if !chat_id.is_trash() && !hidden {
2218        let mut chat = Chat::load_from_db(context, chat_id).await?;
2219
2220        // In contrast to most other update-timestamps,
2221        // use `sort_timestamp` instead of `sent_timestamp` for the subject-timestamp comparison.
2222        // This way, `LastSubject` actually refers to the most recent message _shown_ in the chat.
2223        if chat
2224            .param
2225            .update_timestamp(Param::SubjectTimestamp, sort_timestamp)?
2226        {
2227            // write the last subject even if empty -
2228            // otherwise a reply may get an outdated subject.
2229            let subject = mime_parser.get_subject().unwrap_or_default();
2230
2231            chat.param.set(Param::LastSubject, subject);
2232            chat.update_param(context).await?;
2233        }
2234    }
2235
2236    // Normally outgoing MDNs sent by us never appear in mailboxes, but Gmail saves all
2237    // outgoing messages, including MDNs, to the Sent folder. If we detect such saved MDN,
2238    // delete it.
2239    let needs_delete_job =
2240        !mime_parser.incoming && is_mdn && is_dc_message == MessengerMessage::Yes;
2241
2242    Ok(ReceivedMsg {
2243        chat_id,
2244        state,
2245        hidden,
2246        sort_timestamp,
2247        msg_ids: created_db_entries,
2248        needs_delete_job,
2249    })
2250}
2251
2252/// Checks for "Chat-Edit" and "Chat-Delete" headers,
2253/// and edits/deletes existing messages accordingly.
2254///
2255/// Returns `true` if this message is an edit/deletion request.
2256async fn handle_edit_delete(
2257    context: &Context,
2258    mime_parser: &MimeMessage,
2259    from_id: ContactId,
2260) -> Result<()> {
2261    if let Some(rfc724_mid) = mime_parser.get_header(HeaderDef::ChatEdit) {
2262        if let Some((original_msg_id, _)) = rfc724_mid_exists(context, rfc724_mid).await? {
2263            if let Some(mut original_msg) =
2264                Message::load_from_db_optional(context, original_msg_id).await?
2265            {
2266                if original_msg.from_id == from_id {
2267                    if let Some(part) = mime_parser.parts.first() {
2268                        let edit_msg_showpadlock = part
2269                            .param
2270                            .get_bool(Param::GuaranteeE2ee)
2271                            .unwrap_or_default();
2272                        if edit_msg_showpadlock || !original_msg.get_showpadlock() {
2273                            let new_text =
2274                                part.msg.strip_prefix(EDITED_PREFIX).unwrap_or(&part.msg);
2275                            chat::save_text_edit_to_db(context, &mut original_msg, new_text)
2276                                .await?;
2277                        } else {
2278                            warn!(context, "Edit message: Not encrypted.");
2279                        }
2280                    }
2281                } else {
2282                    warn!(context, "Edit message: Bad sender.");
2283                }
2284            } else {
2285                warn!(context, "Edit message: Database entry does not exist.");
2286            }
2287        } else {
2288            warn!(
2289                context,
2290                "Edit message: rfc724_mid {rfc724_mid:?} not found."
2291            );
2292        }
2293    } else if let Some(rfc724_mid_list) = mime_parser.get_header(HeaderDef::ChatDelete) {
2294        if let Some(part) = mime_parser.parts.first() {
2295            // See `message::delete_msgs_ex()`, unlike edit requests, DC doesn't send unencrypted
2296            // deletion requests, so there's no need to support them.
2297            if part.param.get_bool(Param::GuaranteeE2ee).unwrap_or(false) {
2298                let mut modified_chat_ids = HashSet::new();
2299                let mut msg_ids = Vec::new();
2300
2301                let rfc724_mid_vec: Vec<&str> = rfc724_mid_list.split_whitespace().collect();
2302                for rfc724_mid in rfc724_mid_vec {
2303                    if let Some((msg_id, _)) =
2304                        message::rfc724_mid_exists(context, rfc724_mid).await?
2305                    {
2306                        if let Some(msg) = Message::load_from_db_optional(context, msg_id).await? {
2307                            if msg.from_id == from_id {
2308                                message::delete_msg_locally(context, &msg).await?;
2309                                msg_ids.push(msg.id);
2310                                modified_chat_ids.insert(msg.chat_id);
2311                            } else {
2312                                warn!(context, "Delete message: Bad sender.");
2313                            }
2314                        } else {
2315                            warn!(context, "Delete message: Database entry does not exist.");
2316                        }
2317                    } else {
2318                        warn!(context, "Delete message: {rfc724_mid:?} not found.");
2319                    }
2320                }
2321                message::delete_msgs_locally_done(context, &msg_ids, modified_chat_ids).await?;
2322            } else {
2323                warn!(context, "Delete message: Not encrypted.");
2324            }
2325        }
2326    }
2327    Ok(())
2328}
2329
2330async fn tweak_sort_timestamp(
2331    context: &Context,
2332    mime_parser: &mut MimeMessage,
2333    silent: bool,
2334    chat_id: ChatId,
2335    sort_timestamp: i64,
2336) -> Result<i64> {
2337    // Ensure replies to messages are sorted after the parent message.
2338    //
2339    // This is useful in a case where sender clocks are not
2340    // synchronized and parent message has a Date: header with a
2341    // timestamp higher than reply timestamp.
2342    //
2343    // This does not help if parent message arrives later than the
2344    // reply.
2345    let parent_timestamp = mime_parser.get_parent_timestamp(context).await?;
2346    let mut sort_timestamp = parent_timestamp.map_or(sort_timestamp, |parent_timestamp| {
2347        std::cmp::max(sort_timestamp, parent_timestamp)
2348    });
2349
2350    // If the message should be silent,
2351    // set the timestamp to be no more than the same as last message
2352    // so that the chat is not sorted to the top of the chatlist.
2353    if silent {
2354        let last_msg_timestamp = if let Some(t) = chat_id.get_timestamp(context).await? {
2355            t
2356        } else {
2357            chat_id.created_timestamp(context).await?
2358        };
2359        sort_timestamp = std::cmp::min(sort_timestamp, last_msg_timestamp);
2360    }
2361    Ok(sort_timestamp)
2362}
2363
2364/// Saves attached locations to the database.
2365///
2366/// Emits an event if at least one new location was added.
2367async fn save_locations(
2368    context: &Context,
2369    mime_parser: &MimeMessage,
2370    chat_id: ChatId,
2371    from_id: ContactId,
2372    msg_id: MsgId,
2373) -> Result<()> {
2374    if chat_id.is_special() {
2375        // Do not save locations for trashed messages.
2376        return Ok(());
2377    }
2378
2379    let mut send_event = false;
2380
2381    if let Some(message_kml) = &mime_parser.message_kml {
2382        if let Some(newest_location_id) =
2383            location::save(context, chat_id, from_id, &message_kml.locations, true).await?
2384        {
2385            location::set_msg_location_id(context, msg_id, newest_location_id).await?;
2386            send_event = true;
2387        }
2388    }
2389
2390    if let Some(location_kml) = &mime_parser.location_kml {
2391        if let Some(addr) = &location_kml.addr {
2392            let contact = Contact::get_by_id(context, from_id).await?;
2393            if contact.get_addr().to_lowercase() == addr.to_lowercase() {
2394                if location::save(context, chat_id, from_id, &location_kml.locations, false)
2395                    .await?
2396                    .is_some()
2397                {
2398                    send_event = true;
2399                }
2400            } else {
2401                warn!(
2402                    context,
2403                    "Address in location.kml {:?} is not the same as the sender address {:?}.",
2404                    addr,
2405                    contact.get_addr()
2406                );
2407            }
2408        }
2409    }
2410    if send_event {
2411        context.emit_location_changed(Some(from_id)).await?;
2412    }
2413    Ok(())
2414}
2415
2416async fn lookup_chat_by_reply(
2417    context: &Context,
2418    mime_parser: &MimeMessage,
2419    parent: &Message,
2420    is_partial_download: &Option<u32>,
2421) -> Result<Option<(ChatId, Blocked)>> {
2422    // If the message is encrypted and has group ID,
2423    // lookup by reply should never be needed
2424    // as we can directly assign the message to the chat
2425    // by its group ID.
2426    ensure_and_debug_assert!(
2427        mime_parser.get_chat_group_id().is_none() || !mime_parser.was_encrypted(),
2428        "Encrypted message has group ID {}",
2429        mime_parser.get_chat_group_id().unwrap_or_default(),
2430    );
2431
2432    // Try to assign message to the same chat as the parent message.
2433    let Some(parent_chat_id) = ChatId::lookup_by_message(parent) else {
2434        return Ok(None);
2435    };
2436
2437    // If this was a private message just to self, it was probably a private reply.
2438    // It should not go into the group then, but into the private chat.
2439    if is_probably_private_reply(context, mime_parser, parent_chat_id).await? {
2440        return Ok(None);
2441    }
2442
2443    // If the parent chat is a 1:1 chat, and the sender added
2444    // a new person to TO/CC, then the message should not go to the 1:1 chat, but to a
2445    // newly created ad-hoc group.
2446    let parent_chat = Chat::load_from_db(context, parent_chat_id).await?;
2447    if parent_chat.typ == Chattype::Single && mime_parser.recipients.len() > 1 {
2448        return Ok(None);
2449    }
2450
2451    // Do not assign unencrypted messages to encrypted chats.
2452    if is_partial_download.is_none()
2453        && parent_chat.is_encrypted(context).await?
2454        && !mime_parser.was_encrypted()
2455    {
2456        return Ok(None);
2457    }
2458
2459    info!(
2460        context,
2461        "Assigning message to {parent_chat_id} as it's a reply to {}.", parent.rfc724_mid
2462    );
2463    Ok(Some((parent_chat.id, parent_chat.blocked)))
2464}
2465
2466async fn lookup_or_create_adhoc_group(
2467    context: &Context,
2468    mime_parser: &MimeMessage,
2469    to_ids: &[Option<ContactId>],
2470    allow_creation: bool,
2471    create_blocked: Blocked,
2472    is_partial_download: bool,
2473) -> Result<Option<(ChatId, Blocked)>> {
2474    // Partial download may be an encrypted message with protected Subject header. We do not want to
2475    // create a group with "..." or "Encrypted message" as a subject. The same is for undecipherable
2476    // messages. Instead, assign the message to 1:1 chat with the sender.
2477    if is_partial_download {
2478        info!(
2479            context,
2480            "Ad-hoc group cannot be created from partial download."
2481        );
2482        return Ok(None);
2483    }
2484    if mime_parser.decrypting_failed {
2485        warn!(
2486            context,
2487            "Not creating ad-hoc group for message that cannot be decrypted."
2488        );
2489        return Ok(None);
2490    }
2491
2492    // Lookup address-contact by the From address.
2493    let fingerprint = None;
2494    let find_key_contact_by_addr = false;
2495    let prevent_rename = should_prevent_rename(mime_parser);
2496    let (from_id, _from_id_blocked, _incoming_origin) = from_field_to_contact_id(
2497        context,
2498        &mime_parser.from,
2499        fingerprint,
2500        prevent_rename,
2501        find_key_contact_by_addr,
2502    )
2503    .await?
2504    .context("Cannot lookup address-contact by the From field")?;
2505
2506    let grpname = mime_parser
2507        .get_header(HeaderDef::ChatGroupName)
2508        .map(|s| s.to_string())
2509        .unwrap_or_else(|| {
2510            mime_parser
2511                .get_subject()
2512                .map(|s| remove_subject_prefix(&s))
2513                .unwrap_or_else(|| "👥📧".to_string())
2514        });
2515    let to_ids: Vec<ContactId> = to_ids.iter().filter_map(|x| *x).collect();
2516    let mut contact_ids = Vec::with_capacity(to_ids.len() + 1);
2517    contact_ids.extend(&to_ids);
2518    if !contact_ids.contains(&from_id) {
2519        contact_ids.push(from_id);
2520    }
2521    let trans_fn = |t: &mut rusqlite::Transaction| {
2522        t.pragma_update(None, "query_only", "0")?;
2523        t.execute(
2524            "CREATE TEMP TABLE temp.contacts (
2525                id INTEGER PRIMARY KEY
2526            ) STRICT",
2527            (),
2528        )?;
2529        let mut stmt = t.prepare("INSERT INTO temp.contacts(id) VALUES (?)")?;
2530        for &id in &contact_ids {
2531            stmt.execute((id,))?;
2532        }
2533        let val = t
2534            .query_row(
2535                "SELECT c.id, c.blocked
2536                FROM chats c INNER JOIN msgs m ON c.id=m.chat_id
2537                WHERE m.hidden=0 AND c.grpid='' AND c.name=?
2538                AND (SELECT COUNT(*) FROM chats_contacts
2539                     WHERE chat_id=c.id
2540                     AND add_timestamp >= remove_timestamp)=?
2541                AND (SELECT COUNT(*) FROM chats_contacts
2542                     WHERE chat_id=c.id
2543                     AND contact_id NOT IN (SELECT id FROM temp.contacts)
2544                     AND add_timestamp >= remove_timestamp)=0
2545                ORDER BY m.timestamp DESC",
2546                (&grpname, contact_ids.len()),
2547                |row| {
2548                    let id: ChatId = row.get(0)?;
2549                    let blocked: Blocked = row.get(1)?;
2550                    Ok((id, blocked))
2551                },
2552            )
2553            .optional()?;
2554        t.execute("DROP TABLE temp.contacts", ())?;
2555        Ok(val)
2556    };
2557    let query_only = true;
2558    if let Some((chat_id, blocked)) = context.sql.transaction_ex(query_only, trans_fn).await? {
2559        info!(
2560            context,
2561            "Assigning message to ad-hoc group {chat_id} with matching name and members."
2562        );
2563        return Ok(Some((chat_id, blocked)));
2564    }
2565    if !allow_creation {
2566        return Ok(None);
2567    }
2568    create_adhoc_group(
2569        context,
2570        mime_parser,
2571        create_blocked,
2572        from_id,
2573        &to_ids,
2574        &grpname,
2575    )
2576    .await
2577    .context("Could not create ad hoc group")
2578}
2579
2580/// If this method returns true, the message shall be assigned to the 1:1 chat with the sender.
2581/// If it returns false, it shall be assigned to the parent chat.
2582async fn is_probably_private_reply(
2583    context: &Context,
2584    mime_parser: &MimeMessage,
2585    parent_chat_id: ChatId,
2586) -> Result<bool> {
2587    // Message cannot be a private reply if it has an explicit Chat-Group-ID header.
2588    if mime_parser.get_chat_group_id().is_some() {
2589        return Ok(false);
2590    }
2591
2592    // Usually we don't want to show private replies in the parent chat, but in the
2593    // 1:1 chat with the sender.
2594    //
2595    // There is one exception: Classical MUA replies to two-member groups
2596    // should be assigned to the group chat. We restrict this exception to classical emails, as chat-group-messages
2597    // contain a Chat-Group-Id header and can be sorted into the correct chat this way.
2598
2599    if mime_parser.recipients.len() != 1 {
2600        return Ok(false);
2601    }
2602
2603    if !mime_parser.has_chat_version() {
2604        let chat_contacts = chat::get_chat_contacts(context, parent_chat_id).await?;
2605        if chat_contacts.len() == 2 && chat_contacts.contains(&ContactId::SELF) {
2606            return Ok(false);
2607        }
2608    }
2609
2610    Ok(true)
2611}
2612
2613/// This function tries to extract the group-id from the message and create a new group
2614/// chat with this ID. If there is no group-id and there are more
2615/// than two members, a new ad hoc group is created.
2616///
2617/// On success the function returns the created (chat_id, chat_blocked) tuple.
2618#[expect(clippy::too_many_arguments)]
2619async fn create_group(
2620    context: &Context,
2621    mime_parser: &mut MimeMessage,
2622    is_partial_download: bool,
2623    create_blocked: Blocked,
2624    from_id: ContactId,
2625    to_ids: &[Option<ContactId>],
2626    past_ids: &[Option<ContactId>],
2627    verified_encryption: &VerifiedEncryption,
2628    grpid: &str,
2629) -> Result<Option<(ChatId, Blocked)>> {
2630    let to_ids_flat: Vec<ContactId> = to_ids.iter().filter_map(|x| *x).collect();
2631    let mut chat_id = None;
2632    let mut chat_id_blocked = Default::default();
2633
2634    let create_protected = if mime_parser.get_header(HeaderDef::ChatVerified).is_some() {
2635        if let VerifiedEncryption::NotVerified(err) = verified_encryption {
2636            warn!(
2637                context,
2638                "Creating unprotected group because of the verification problem: {err:#}."
2639            );
2640            ProtectionStatus::Unprotected
2641        } else {
2642            ProtectionStatus::Protected
2643        }
2644    } else {
2645        ProtectionStatus::Unprotected
2646    };
2647
2648    async fn self_explicitly_added(
2649        context: &Context,
2650        mime_parser: &&mut MimeMessage,
2651    ) -> Result<bool> {
2652        let ret = match mime_parser.get_header(HeaderDef::ChatGroupMemberAdded) {
2653            Some(member_addr) => context.is_self_addr(member_addr).await?,
2654            None => false,
2655        };
2656        Ok(ret)
2657    }
2658
2659    if chat_id.is_none()
2660            && !mime_parser.is_mailinglist_message()
2661            && !grpid.is_empty()
2662            && mime_parser.get_header(HeaderDef::ChatGroupName).is_some()
2663            // otherwise, a pending "quit" message may pop up
2664            && mime_parser.get_header(HeaderDef::ChatGroupMemberRemoved).is_none()
2665            // re-create explicitly left groups only if ourself is re-added
2666            && (!chat::is_group_explicitly_left(context, grpid).await?
2667                || self_explicitly_added(context, &mime_parser).await?)
2668    {
2669        // Group does not exist but should be created.
2670        let grpname = mime_parser
2671            .get_header(HeaderDef::ChatGroupName)
2672            .context("Chat-Group-Name vanished")?
2673            // Workaround for the "Space added before long group names after MIME
2674            // serialization/deserialization #3650" issue. DC itself never creates group names with
2675            // leading/trailing whitespace.
2676            .trim();
2677        let new_chat_id = ChatId::create_multiuser_record(
2678            context,
2679            Chattype::Group,
2680            grpid,
2681            grpname,
2682            create_blocked,
2683            create_protected,
2684            None,
2685            mime_parser.timestamp_sent,
2686        )
2687        .await
2688        .with_context(|| format!("Failed to create group '{grpname}' for grpid={grpid}"))?;
2689
2690        chat_id = Some(new_chat_id);
2691        chat_id_blocked = create_blocked;
2692
2693        // Create initial member list.
2694        if let Some(mut chat_group_member_timestamps) = mime_parser.chat_group_member_timestamps() {
2695            let mut new_to_ids = to_ids.to_vec();
2696            if !new_to_ids.contains(&Some(from_id)) {
2697                new_to_ids.insert(0, Some(from_id));
2698                chat_group_member_timestamps.insert(0, mime_parser.timestamp_sent);
2699            }
2700
2701            update_chats_contacts_timestamps(
2702                context,
2703                new_chat_id,
2704                None,
2705                &new_to_ids,
2706                past_ids,
2707                &chat_group_member_timestamps,
2708            )
2709            .await?;
2710        } else {
2711            let mut members = vec![ContactId::SELF];
2712            if !from_id.is_special() {
2713                members.push(from_id);
2714            }
2715            members.extend(to_ids_flat);
2716
2717            // Add all members with 0 timestamp
2718            // because we don't know the real timestamp of their addition.
2719            // This will allow other senders who support
2720            // `Chat-Group-Member-Timestamps` to overwrite
2721            // timestamps later.
2722            let timestamp = 0;
2723
2724            chat::add_to_chat_contacts_table(context, timestamp, new_chat_id, &members).await?;
2725        }
2726
2727        context.emit_event(EventType::ChatModified(new_chat_id));
2728        chatlist_events::emit_chatlist_changed(context);
2729        chatlist_events::emit_chatlist_item_changed(context, new_chat_id);
2730    }
2731
2732    if let Some(chat_id) = chat_id {
2733        Ok(Some((chat_id, chat_id_blocked)))
2734    } else if is_partial_download || mime_parser.decrypting_failed {
2735        // It is possible that the message was sent to a valid,
2736        // yet unknown group, which was rejected because
2737        // Chat-Group-Name, which is in the encrypted part, was
2738        // not found. We can't create a properly named group in
2739        // this case, so assign error message to 1:1 chat with the
2740        // sender instead.
2741        Ok(None)
2742    } else {
2743        // The message was decrypted successfully, but contains a late "quit" or otherwise
2744        // unwanted message.
2745        info!(context, "Message belongs to unwanted group (TRASH).");
2746        Ok(Some((DC_CHAT_ID_TRASH, Blocked::Not)))
2747    }
2748}
2749
2750async fn update_chats_contacts_timestamps(
2751    context: &Context,
2752    chat_id: ChatId,
2753    ignored_id: Option<ContactId>,
2754    to_ids: &[Option<ContactId>],
2755    past_ids: &[Option<ContactId>],
2756    chat_group_member_timestamps: &[i64],
2757) -> Result<bool> {
2758    let expected_timestamps_count = to_ids.len() + past_ids.len();
2759
2760    if chat_group_member_timestamps.len() != expected_timestamps_count {
2761        warn!(
2762            context,
2763            "Chat-Group-Member-Timestamps has wrong number of timestamps, got {}, expected {}.",
2764            chat_group_member_timestamps.len(),
2765            expected_timestamps_count
2766        );
2767        return Ok(false);
2768    }
2769
2770    let mut modified = false;
2771
2772    context
2773        .sql
2774        .transaction(|transaction| {
2775            let mut add_statement = transaction.prepare(
2776                "INSERT INTO chats_contacts (chat_id, contact_id, add_timestamp)
2777                 VALUES                     (?1,      ?2,         ?3)
2778                 ON CONFLICT (chat_id, contact_id)
2779                 DO
2780                   UPDATE SET add_timestamp=?3
2781                   WHERE ?3>add_timestamp AND ?3>=remove_timestamp",
2782            )?;
2783
2784            for (contact_id, ts) in iter::zip(
2785                to_ids.iter(),
2786                chat_group_member_timestamps.iter().take(to_ids.len()),
2787            ) {
2788                if let Some(contact_id) = contact_id {
2789                    if Some(*contact_id) != ignored_id {
2790                        // It could be that member was already added,
2791                        // but updated addition timestamp
2792                        // is also a modification worth notifying about.
2793                        modified |= add_statement.execute((chat_id, contact_id, ts))? > 0;
2794                    }
2795                }
2796            }
2797
2798            let mut remove_statement = transaction.prepare(
2799                "INSERT INTO chats_contacts (chat_id, contact_id, remove_timestamp)
2800                 VALUES                     (?1,      ?2,         ?3)
2801                 ON CONFLICT (chat_id, contact_id)
2802                 DO
2803                   UPDATE SET remove_timestamp=?3
2804                   WHERE ?3>remove_timestamp AND ?3>add_timestamp",
2805            )?;
2806
2807            for (contact_id, ts) in iter::zip(
2808                past_ids.iter(),
2809                chat_group_member_timestamps.iter().skip(to_ids.len()),
2810            ) {
2811                if let Some(contact_id) = contact_id {
2812                    // It could be that member was already removed,
2813                    // but updated removal timestamp
2814                    // is also a modification worth notifying about.
2815                    modified |= remove_statement.execute((chat_id, contact_id, ts))? > 0;
2816                }
2817            }
2818
2819            Ok(())
2820        })
2821        .await?;
2822
2823    Ok(modified)
2824}
2825
2826/// The return type of [apply_group_changes].
2827/// Contains information on which system messages
2828/// should be shown in the chat.
2829#[derive(Default)]
2830struct GroupChangesInfo {
2831    /// Optional: A better message that should replace the original system message.
2832    /// If this is an empty string, the original system message should be trashed.
2833    better_msg: Option<String>,
2834    /// Added/removed contact `better_msg` refers to.
2835    added_removed_id: Option<ContactId>,
2836    /// If true, the user should not be notified about the group change.
2837    silent: bool,
2838    /// A list of additional group changes messages that should be shown in the chat.
2839    extra_msgs: Vec<(String, SystemMessage, Option<ContactId>)>,
2840}
2841
2842/// Apply group member list, name, avatar and protection status changes from the MIME message.
2843///
2844/// Returns [GroupChangesInfo].
2845///
2846/// * `to_ids` - contents of the `To` and `Cc` headers.
2847/// * `past_ids` - contents of the `Chat-Group-Past-Members` header.
2848async fn apply_group_changes(
2849    context: &Context,
2850    mime_parser: &mut MimeMessage,
2851    chat: &mut Chat,
2852    from_id: ContactId,
2853    to_ids: &[Option<ContactId>],
2854    past_ids: &[Option<ContactId>],
2855    verified_encryption: &VerifiedEncryption,
2856) -> Result<GroupChangesInfo> {
2857    let to_ids_flat: Vec<ContactId> = to_ids.iter().filter_map(|x| *x).collect();
2858    ensure!(chat.typ == Chattype::Group);
2859    ensure!(!chat.id.is_special());
2860
2861    let mut send_event_chat_modified = false;
2862    let (mut removed_id, mut added_id) = (None, None);
2863    let mut better_msg = None;
2864    let mut silent = false;
2865
2866    // True if a Delta Chat client has explicitly added our current primary address.
2867    let self_added =
2868        if let Some(added_addr) = mime_parser.get_header(HeaderDef::ChatGroupMemberAdded) {
2869            addr_cmp(&context.get_primary_self_addr().await?, added_addr)
2870        } else {
2871            false
2872        };
2873
2874    let chat_contacts =
2875        HashSet::<ContactId>::from_iter(chat::get_chat_contacts(context, chat.id).await?);
2876    let is_from_in_chat =
2877        !chat_contacts.contains(&ContactId::SELF) || chat_contacts.contains(&from_id);
2878
2879    if mime_parser.get_header(HeaderDef::ChatVerified).is_some() && !chat.is_protected() {
2880        if let VerifiedEncryption::NotVerified(err) = verified_encryption {
2881            warn!(
2882                context,
2883                "Not marking chat {} as protected due to verification problem: {err:#}.", chat.id,
2884            );
2885        } else {
2886            chat.id
2887                .set_protection(
2888                    context,
2889                    ProtectionStatus::Protected,
2890                    mime_parser.timestamp_sent,
2891                    Some(from_id),
2892                )
2893                .await?;
2894        }
2895    }
2896
2897    if let Some(removed_addr) = mime_parser.get_header(HeaderDef::ChatGroupMemberRemoved) {
2898        // TODO: if address "alice@example.org" is a member of the group twice,
2899        // with old and new key,
2900        // and someone (maybe Alice's new contact) just removed Alice's old contact,
2901        // we may lookup the wrong contact because we only look up by the address.
2902        // The result is that info message may contain the new Alice's display name
2903        // rather than old display name.
2904        // This could be fixed by looking up the contact with the highest
2905        // `remove_timestamp` after applying Chat-Group-Member-Timestamps.
2906        removed_id = lookup_key_contact_by_address(context, removed_addr, Some(chat.id)).await?;
2907        if let Some(id) = removed_id {
2908            better_msg = if id == from_id {
2909                silent = true;
2910                Some(stock_str::msg_group_left_local(context, from_id).await)
2911            } else {
2912                Some(stock_str::msg_del_member_local(context, id, from_id).await)
2913            };
2914        } else {
2915            warn!(context, "Removed {removed_addr:?} has no contact id.")
2916        }
2917    } else if let Some(added_addr) = mime_parser.get_header(HeaderDef::ChatGroupMemberAdded) {
2918        if let Some(key) = mime_parser.gossiped_keys.get(added_addr) {
2919            // TODO: if gossiped keys contain the same address multiple times,
2920            // we may lookup the wrong contact.
2921            // This could be fixed by looking up the contact with
2922            // highest `add_timestamp` to disambiguate.
2923            // The result of the error is that info message
2924            // may contain display name of the wrong contact.
2925            let fingerprint = key.dc_fingerprint().hex();
2926            if let Some(contact_id) =
2927                lookup_key_contact_by_fingerprint(context, &fingerprint).await?
2928            {
2929                added_id = Some(contact_id);
2930                better_msg =
2931                    Some(stock_str::msg_add_member_local(context, contact_id, from_id).await);
2932            } else {
2933                warn!(context, "Added {added_addr:?} has no contact id.");
2934            }
2935        } else {
2936            warn!(context, "Added {added_addr:?} has no gossiped key.");
2937        }
2938    }
2939
2940    if is_from_in_chat {
2941        apply_chat_name_and_avatar_changes(
2942            context,
2943            mime_parser,
2944            from_id,
2945            chat,
2946            &mut send_event_chat_modified,
2947            &mut better_msg,
2948        )
2949        .await?;
2950
2951        if chat.member_list_is_stale(context).await? {
2952            info!(context, "Member list is stale.");
2953            let mut new_members: HashSet<ContactId> =
2954                HashSet::from_iter(to_ids_flat.iter().copied());
2955            new_members.insert(ContactId::SELF);
2956            if !from_id.is_special() {
2957                new_members.insert(from_id);
2958            }
2959
2960            context
2961                .sql
2962                .transaction(|transaction| {
2963                    // Remove all contacts and tombstones.
2964                    transaction.execute(
2965                        "DELETE FROM chats_contacts
2966                         WHERE chat_id=?",
2967                        (chat.id,),
2968                    )?;
2969
2970                    // Insert contacts with default timestamps of 0.
2971                    let mut statement = transaction.prepare(
2972                        "INSERT INTO chats_contacts (chat_id, contact_id)
2973                         VALUES                     (?,       ?)",
2974                    )?;
2975                    for contact_id in &new_members {
2976                        statement.execute((chat.id, contact_id))?;
2977                    }
2978
2979                    Ok(())
2980                })
2981                .await?;
2982            send_event_chat_modified = true;
2983        } else if let Some(ref chat_group_member_timestamps) =
2984            mime_parser.chat_group_member_timestamps()
2985        {
2986            send_event_chat_modified |= update_chats_contacts_timestamps(
2987                context,
2988                chat.id,
2989                Some(from_id),
2990                to_ids,
2991                past_ids,
2992                chat_group_member_timestamps,
2993            )
2994            .await?;
2995        } else {
2996            let mut new_members: HashSet<ContactId>;
2997            if self_added {
2998                new_members = HashSet::from_iter(to_ids_flat.iter().copied());
2999                new_members.insert(ContactId::SELF);
3000                if !from_id.is_special() {
3001                    new_members.insert(from_id);
3002                }
3003            } else {
3004                new_members = chat_contacts.clone();
3005            }
3006
3007            // Allow non-Delta Chat MUAs to add members.
3008            if mime_parser.get_header(HeaderDef::ChatVersion).is_none() {
3009                // Don't delete any members locally, but instead add absent ones to provide group
3010                // membership consistency for all members:
3011                new_members.extend(to_ids_flat.iter());
3012            }
3013
3014            // Apply explicit addition if any.
3015            if let Some(added_id) = added_id {
3016                new_members.insert(added_id);
3017            }
3018
3019            // Apply explicit removal if any.
3020            if let Some(removed_id) = removed_id {
3021                new_members.remove(&removed_id);
3022            }
3023
3024            if new_members != chat_contacts {
3025                chat::update_chat_contacts_table(
3026                    context,
3027                    mime_parser.timestamp_sent,
3028                    chat.id,
3029                    &new_members,
3030                )
3031                .await?;
3032                send_event_chat_modified = true;
3033            }
3034        }
3035
3036        chat.id
3037            .update_timestamp(
3038                context,
3039                Param::MemberListTimestamp,
3040                mime_parser.timestamp_sent,
3041            )
3042            .await?;
3043    }
3044
3045    let new_chat_contacts = HashSet::<ContactId>::from_iter(
3046        chat::get_chat_contacts(context, chat.id)
3047            .await?
3048            .iter()
3049            .copied(),
3050    );
3051
3052    // These are for adding info messages about implicit membership changes.
3053    let mut added_ids: HashSet<ContactId> = new_chat_contacts
3054        .difference(&chat_contacts)
3055        .copied()
3056        .collect();
3057    let mut removed_ids: HashSet<ContactId> = chat_contacts
3058        .difference(&new_chat_contacts)
3059        .copied()
3060        .collect();
3061
3062    if let Some(added_id) = added_id {
3063        if !added_ids.remove(&added_id) && !self_added {
3064            // No-op "Member added" message.
3065            //
3066            // Trash it.
3067            better_msg = Some(String::new());
3068        }
3069    }
3070    if let Some(removed_id) = removed_id {
3071        removed_ids.remove(&removed_id);
3072    }
3073    let group_changes_msgs = if self_added {
3074        Vec::new()
3075    } else {
3076        group_changes_msgs(context, &added_ids, &removed_ids, chat.id).await?
3077    };
3078
3079    if send_event_chat_modified {
3080        context.emit_event(EventType::ChatModified(chat.id));
3081        chatlist_events::emit_chatlist_item_changed(context, chat.id);
3082    }
3083    Ok(GroupChangesInfo {
3084        better_msg,
3085        added_removed_id: if added_id.is_some() {
3086            added_id
3087        } else {
3088            removed_id
3089        },
3090        silent,
3091        extra_msgs: group_changes_msgs,
3092    })
3093}
3094
3095/// Applies incoming changes to the group's or broadcast channel's name and avatar.
3096///
3097/// - `send_event_chat_modified` is set to `true` if ChatModified event should be sent
3098/// - `better_msg` is filled with an info message about name change, if necessary
3099async fn apply_chat_name_and_avatar_changes(
3100    context: &Context,
3101    mime_parser: &MimeMessage,
3102    from_id: ContactId,
3103    chat: &mut Chat,
3104    send_event_chat_modified: &mut bool,
3105    better_msg: &mut Option<String>,
3106) -> Result<()> {
3107    // ========== Apply chat name changes ==========
3108
3109    let group_name_timestamp = mime_parser
3110        .get_header(HeaderDef::ChatGroupNameTimestamp)
3111        .and_then(|s| s.parse::<i64>().ok());
3112
3113    if let Some(old_name) = mime_parser
3114        .get_header(HeaderDef::ChatGroupNameChanged)
3115        .map(|s| s.trim())
3116        .or(match group_name_timestamp {
3117            Some(0) => None,
3118            Some(_) => Some(chat.name.as_str()),
3119            None => None,
3120        })
3121    {
3122        if let Some(grpname) = mime_parser
3123            .get_header(HeaderDef::ChatGroupName)
3124            .map(|grpname| grpname.trim())
3125            .filter(|grpname| grpname.len() < 200)
3126        {
3127            let grpname = &sanitize_single_line(grpname);
3128
3129            let chat_group_name_timestamp =
3130                chat.param.get_i64(Param::GroupNameTimestamp).unwrap_or(0);
3131            let group_name_timestamp = group_name_timestamp.unwrap_or(mime_parser.timestamp_sent);
3132            // To provide group name consistency, compare names if timestamps are equal.
3133            if (chat_group_name_timestamp, grpname) < (group_name_timestamp, &chat.name)
3134                && chat
3135                    .id
3136                    .update_timestamp(context, Param::GroupNameTimestamp, group_name_timestamp)
3137                    .await?
3138                && grpname != &chat.name
3139            {
3140                info!(context, "Updating grpname for chat {}.", chat.id);
3141                context
3142                    .sql
3143                    .execute("UPDATE chats SET name=? WHERE id=?;", (grpname, chat.id))
3144                    .await?;
3145                *send_event_chat_modified = true;
3146            }
3147            if mime_parser
3148                .get_header(HeaderDef::ChatGroupNameChanged)
3149                .is_some()
3150            {
3151                let old_name = &sanitize_single_line(old_name);
3152                better_msg.get_or_insert(
3153                    stock_str::msg_grp_name(context, old_name, grpname, from_id).await,
3154                );
3155            }
3156        }
3157    }
3158
3159    // ========== Apply chat avatar changes ==========
3160
3161    if let (Some(value), None) = (mime_parser.get_header(HeaderDef::ChatContent), &better_msg) {
3162        if value == "group-avatar-changed" {
3163            if let Some(avatar_action) = &mime_parser.group_avatar {
3164                // this is just an explicit message containing the group-avatar,
3165                // apart from that, the group-avatar is send along with various other messages
3166                better_msg.get_or_insert(match avatar_action {
3167                    AvatarAction::Delete => stock_str::msg_grp_img_deleted(context, from_id).await,
3168                    AvatarAction::Change(_) => {
3169                        stock_str::msg_grp_img_changed(context, from_id).await
3170                    }
3171                });
3172            }
3173        }
3174    }
3175
3176    if let Some(avatar_action) = &mime_parser.group_avatar {
3177        info!(context, "Group-avatar change for {}.", chat.id);
3178        if chat
3179            .param
3180            .update_timestamp(Param::AvatarTimestamp, mime_parser.timestamp_sent)?
3181        {
3182            match avatar_action {
3183                AvatarAction::Change(profile_image) => {
3184                    chat.param.set(Param::ProfileImage, profile_image);
3185                }
3186                AvatarAction::Delete => {
3187                    chat.param.remove(Param::ProfileImage);
3188                }
3189            };
3190            chat.update_param(context).await?;
3191            *send_event_chat_modified = true;
3192        }
3193    }
3194
3195    Ok(())
3196}
3197
3198/// Returns a list of strings that should be shown as info messages, informing about group membership changes.
3199async fn group_changes_msgs(
3200    context: &Context,
3201    added_ids: &HashSet<ContactId>,
3202    removed_ids: &HashSet<ContactId>,
3203    chat_id: ChatId,
3204) -> Result<Vec<(String, SystemMessage, Option<ContactId>)>> {
3205    let mut group_changes_msgs: Vec<(String, SystemMessage, Option<ContactId>)> = Vec::new();
3206    if !added_ids.is_empty() {
3207        warn!(
3208            context,
3209            "Implicit addition of {added_ids:?} to chat {chat_id}."
3210        );
3211    }
3212    if !removed_ids.is_empty() {
3213        warn!(
3214            context,
3215            "Implicit removal of {removed_ids:?} from chat {chat_id}."
3216        );
3217    }
3218    group_changes_msgs.reserve(added_ids.len() + removed_ids.len());
3219    for contact_id in added_ids {
3220        group_changes_msgs.push((
3221            stock_str::msg_add_member_local(context, *contact_id, ContactId::UNDEFINED).await,
3222            SystemMessage::MemberAddedToGroup,
3223            Some(*contact_id),
3224        ));
3225    }
3226    for contact_id in removed_ids {
3227        group_changes_msgs.push((
3228            stock_str::msg_del_member_local(context, *contact_id, ContactId::UNDEFINED).await,
3229            SystemMessage::MemberRemovedFromGroup,
3230            Some(*contact_id),
3231        ));
3232    }
3233
3234    Ok(group_changes_msgs)
3235}
3236
3237static LIST_ID_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^(.+)<(.+)>$").unwrap());
3238
3239fn mailinglist_header_listid(list_id_header: &str) -> Result<String> {
3240    Ok(match LIST_ID_REGEX.captures(list_id_header) {
3241        Some(cap) => cap.get(2).context("no match??")?.as_str().trim(),
3242        None => list_id_header
3243            .trim()
3244            .trim_start_matches('<')
3245            .trim_end_matches('>'),
3246    }
3247    .to_string())
3248}
3249
3250/// Create or lookup a mailing list or incoming broadcast channel chat.
3251///
3252/// `list_id_header` contains the Id that must be used for the mailing list
3253/// and has the form `Name <Id>`, `<Id>` or just `Id`.
3254/// Depending on the mailing list type, `list_id_header`
3255/// was picked from `ListId:`-header or the `Sender:`-header.
3256///
3257/// `mime_parser` is the corresponding message
3258/// and is used to figure out the mailing list name from different header fields.
3259async fn create_or_lookup_mailinglist_or_broadcast(
3260    context: &Context,
3261    allow_creation: bool,
3262    list_id_header: &str,
3263    from_id: ContactId,
3264    mime_parser: &MimeMessage,
3265) -> Result<Option<(ChatId, Blocked)>> {
3266    let listid = mailinglist_header_listid(list_id_header)?;
3267
3268    if let Some((chat_id, _, blocked)) = chat::get_chat_id_by_grpid(context, &listid).await? {
3269        return Ok(Some((chat_id, blocked)));
3270    }
3271
3272    let chattype = if mime_parser.was_encrypted() {
3273        Chattype::InBroadcast
3274    } else {
3275        Chattype::Mailinglist
3276    };
3277
3278    let name = if chattype == Chattype::InBroadcast {
3279        mime_parser
3280            .get_header(HeaderDef::ChatGroupName)
3281            .unwrap_or("Broadcast Channel")
3282    } else {
3283        &compute_mailinglist_name(list_id_header, &listid, mime_parser)
3284    };
3285
3286    if allow_creation {
3287        // list does not exist but should be created
3288        let param = mime_parser.list_post.as_ref().map(|list_post| {
3289            let mut p = Params::new();
3290            p.set(Param::ListPost, list_post);
3291            p.to_string()
3292        });
3293
3294        let is_bot = context.get_config_bool(Config::Bot).await?;
3295        let blocked = if is_bot {
3296            Blocked::Not
3297        } else {
3298            Blocked::Request
3299        };
3300        let chat_id = ChatId::create_multiuser_record(
3301            context,
3302            chattype,
3303            &listid,
3304            name,
3305            blocked,
3306            ProtectionStatus::Unprotected,
3307            param,
3308            mime_parser.timestamp_sent,
3309        )
3310        .await
3311        .with_context(|| {
3312            format!(
3313                "failed to create mailinglist '{}' for grpid={}",
3314                &name, &listid
3315            )
3316        })?;
3317
3318        chat::add_to_chat_contacts_table(
3319            context,
3320            mime_parser.timestamp_sent,
3321            chat_id,
3322            &[ContactId::SELF],
3323        )
3324        .await?;
3325        if chattype == Chattype::InBroadcast {
3326            chat::add_to_chat_contacts_table(
3327                context,
3328                mime_parser.timestamp_sent,
3329                chat_id,
3330                &[from_id],
3331            )
3332            .await?;
3333        }
3334        Ok(Some((chat_id, blocked)))
3335    } else {
3336        info!(context, "Creating list forbidden by caller.");
3337        Ok(None)
3338    }
3339}
3340
3341fn compute_mailinglist_name(
3342    list_id_header: &str,
3343    listid: &str,
3344    mime_parser: &MimeMessage,
3345) -> String {
3346    let mut name = match LIST_ID_REGEX
3347        .captures(list_id_header)
3348        .and_then(|caps| caps.get(1))
3349    {
3350        Some(cap) => cap.as_str().trim().to_string(),
3351        None => "".to_string(),
3352    };
3353
3354    // for mailchimp lists, the name in `ListId` is just a long number.
3355    // a usable name for these lists is in the `From` header
3356    // and we can detect these lists by a unique `ListId`-suffix.
3357    if listid.ends_with(".list-id.mcsv.net") {
3358        if let Some(display_name) = &mime_parser.from.display_name {
3359            name.clone_from(display_name);
3360        }
3361    }
3362
3363    // additional names in square brackets in the subject are preferred
3364    // (as that part is much more visible, we assume, that names is shorter and comes more to the point,
3365    // than the sometimes longer part from ListId)
3366    let subject = mime_parser.get_subject().unwrap_or_default();
3367    static SUBJECT: LazyLock<Regex> =
3368        LazyLock::new(|| Regex::new(r"^.{0,5}\[(.+?)\](\s*\[.+\])?").unwrap()); // remove square brackets around first name
3369    if let Some(cap) = SUBJECT.captures(&subject) {
3370        name = cap[1].to_string() + cap.get(2).map_or("", |m| m.as_str());
3371    }
3372
3373    // if we do not have a name yet and `From` indicates, that this is a notification list,
3374    // a usable name is often in the `From` header (seen for several parcel service notifications).
3375    // same, if we do not have a name yet and `List-Id` has a known suffix (`.xt.local`)
3376    //
3377    // this pattern is similar to mailchimp above, however,
3378    // with weaker conditions and does not overwrite existing names.
3379    if name.is_empty()
3380        && (mime_parser.from.addr.contains("noreply")
3381            || mime_parser.from.addr.contains("no-reply")
3382            || mime_parser.from.addr.starts_with("notifications@")
3383            || mime_parser.from.addr.starts_with("newsletter@")
3384            || listid.ends_with(".xt.local"))
3385    {
3386        if let Some(display_name) = &mime_parser.from.display_name {
3387            name.clone_from(display_name);
3388        }
3389    }
3390
3391    // as a last resort, use the ListId as the name
3392    // but strip some known, long hash prefixes
3393    if name.is_empty() {
3394        // 51231231231231231231231232869f58.xing.com -> xing.com
3395        static PREFIX_32_CHARS_HEX: LazyLock<Regex> =
3396            LazyLock::new(|| Regex::new(r"([0-9a-fA-F]{32})\.(.{6,})").unwrap());
3397        if let Some(cap) = PREFIX_32_CHARS_HEX
3398            .captures(listid)
3399            .and_then(|caps| caps.get(2))
3400        {
3401            name = cap.as_str().to_string();
3402        } else {
3403            name = listid.to_string();
3404        }
3405    }
3406
3407    sanitize_single_line(&name)
3408}
3409
3410/// Set ListId param on the contact and ListPost param the chat.
3411/// Only called for incoming messages since outgoing messages never have a
3412/// List-Post header, anyway.
3413async fn apply_mailinglist_changes(
3414    context: &Context,
3415    mime_parser: &MimeMessage,
3416    chat_id: ChatId,
3417) -> Result<()> {
3418    let Some(mailinglist_header) = mime_parser.get_mailinglist_header() else {
3419        return Ok(());
3420    };
3421
3422    let mut chat = Chat::load_from_db(context, chat_id).await?;
3423    if chat.typ != Chattype::Mailinglist {
3424        return Ok(());
3425    }
3426    let listid = &chat.grpid;
3427
3428    let new_name = compute_mailinglist_name(mailinglist_header, listid, mime_parser);
3429    if chat.name != new_name
3430        && chat_id
3431            .update_timestamp(
3432                context,
3433                Param::GroupNameTimestamp,
3434                mime_parser.timestamp_sent,
3435            )
3436            .await?
3437    {
3438        info!(context, "Updating listname for chat {chat_id}.");
3439        context
3440            .sql
3441            .execute("UPDATE chats SET name=? WHERE id=?;", (new_name, chat_id))
3442            .await?;
3443        context.emit_event(EventType::ChatModified(chat_id));
3444    }
3445
3446    let Some(list_post) = &mime_parser.list_post else {
3447        return Ok(());
3448    };
3449
3450    let list_post = match ContactAddress::new(list_post) {
3451        Ok(list_post) => list_post,
3452        Err(err) => {
3453            warn!(context, "Invalid List-Post: {:#}.", err);
3454            return Ok(());
3455        }
3456    };
3457    let (contact_id, _) = Contact::add_or_lookup(context, "", &list_post, Origin::Hidden).await?;
3458    let mut contact = Contact::get_by_id(context, contact_id).await?;
3459    if contact.param.get(Param::ListId) != Some(listid) {
3460        contact.param.set(Param::ListId, listid);
3461        contact.update_param(context).await?;
3462    }
3463
3464    if let Some(old_list_post) = chat.param.get(Param::ListPost) {
3465        if list_post.as_ref() != old_list_post {
3466            // Apparently the mailing list is using a different List-Post header in each message.
3467            // Make the mailing list read-only because we wouldn't know which message the user wants to reply to.
3468            chat.param.remove(Param::ListPost);
3469            chat.update_param(context).await?;
3470        }
3471    } else {
3472        chat.param.set(Param::ListPost, list_post);
3473        chat.update_param(context).await?;
3474    }
3475
3476    Ok(())
3477}
3478
3479async fn apply_out_broadcast_changes(
3480    context: &Context,
3481    mime_parser: &MimeMessage,
3482    chat: &mut Chat,
3483    from_id: ContactId,
3484) -> Result<GroupChangesInfo> {
3485    ensure!(chat.typ == Chattype::OutBroadcast);
3486
3487    if let Some(_removed_addr) = mime_parser.get_header(HeaderDef::ChatGroupMemberRemoved) {
3488        // The sender of the message left the broadcast channel
3489        remove_from_chat_contacts_table(context, chat.id, from_id).await?;
3490
3491        return Ok(GroupChangesInfo {
3492            better_msg: Some("".to_string()),
3493            added_removed_id: None,
3494            silent: true,
3495            extra_msgs: vec![],
3496        });
3497    }
3498
3499    Ok(GroupChangesInfo::default())
3500}
3501
3502async fn apply_in_broadcast_changes(
3503    context: &Context,
3504    mime_parser: &MimeMessage,
3505    chat: &mut Chat,
3506    from_id: ContactId,
3507) -> Result<GroupChangesInfo> {
3508    ensure!(chat.typ == Chattype::InBroadcast);
3509
3510    let mut send_event_chat_modified = false;
3511    let mut better_msg = None;
3512
3513    apply_chat_name_and_avatar_changes(
3514        context,
3515        mime_parser,
3516        from_id,
3517        chat,
3518        &mut send_event_chat_modified,
3519        &mut better_msg,
3520    )
3521    .await?;
3522
3523    if let Some(_removed_addr) = mime_parser.get_header(HeaderDef::ChatGroupMemberRemoved) {
3524        // The only member added/removed message that is ever sent is "I left.",
3525        // so, this is the only case we need to handle here
3526        if from_id == ContactId::SELF {
3527            better_msg
3528                .get_or_insert(stock_str::msg_group_left_local(context, ContactId::SELF).await);
3529        }
3530    }
3531
3532    if send_event_chat_modified {
3533        context.emit_event(EventType::ChatModified(chat.id));
3534        chatlist_events::emit_chatlist_item_changed(context, chat.id);
3535    }
3536    Ok(GroupChangesInfo {
3537        better_msg,
3538        added_removed_id: None,
3539        silent: false,
3540        extra_msgs: vec![],
3541    })
3542}
3543
3544/// Creates ad-hoc group and returns chat ID on success.
3545async fn create_adhoc_group(
3546    context: &Context,
3547    mime_parser: &MimeMessage,
3548    create_blocked: Blocked,
3549    from_id: ContactId,
3550    to_ids: &[ContactId],
3551    grpname: &str,
3552) -> Result<Option<(ChatId, Blocked)>> {
3553    let mut member_ids: Vec<ContactId> = to_ids.to_vec();
3554    if !member_ids.contains(&(from_id)) {
3555        member_ids.push(from_id);
3556    }
3557    if !member_ids.contains(&(ContactId::SELF)) {
3558        member_ids.push(ContactId::SELF);
3559    }
3560
3561    if mime_parser.is_mailinglist_message() {
3562        return Ok(None);
3563    }
3564    if mime_parser
3565        .get_header(HeaderDef::ChatGroupMemberRemoved)
3566        .is_some()
3567    {
3568        info!(
3569            context,
3570            "Message removes member from unknown ad-hoc group (TRASH)."
3571        );
3572        return Ok(Some((DC_CHAT_ID_TRASH, Blocked::Not)));
3573    }
3574    if member_ids.len() < 2 {
3575        info!(
3576            context,
3577            "Not creating ad hoc group with less than 2 members."
3578        );
3579        return Ok(None);
3580    }
3581
3582    let new_chat_id: ChatId = ChatId::create_multiuser_record(
3583        context,
3584        Chattype::Group,
3585        "", // Ad hoc groups have no ID.
3586        grpname,
3587        create_blocked,
3588        ProtectionStatus::Unprotected,
3589        None,
3590        mime_parser.timestamp_sent,
3591    )
3592    .await?;
3593
3594    info!(
3595        context,
3596        "Created ad-hoc group id={new_chat_id}, name={grpname:?}."
3597    );
3598    chat::add_to_chat_contacts_table(
3599        context,
3600        mime_parser.timestamp_sent,
3601        new_chat_id,
3602        &member_ids,
3603    )
3604    .await?;
3605
3606    context.emit_event(EventType::ChatModified(new_chat_id));
3607    chatlist_events::emit_chatlist_changed(context);
3608    chatlist_events::emit_chatlist_item_changed(context, new_chat_id);
3609
3610    Ok(Some((new_chat_id, create_blocked)))
3611}
3612
3613#[derive(Debug, PartialEq, Eq)]
3614enum VerifiedEncryption {
3615    Verified,
3616    NotVerified(String), // The string contains the reason why it's not verified
3617}
3618
3619/// Checks whether the message is allowed to appear in a protected chat.
3620///
3621/// This means that it is encrypted and signed with a verified key.
3622async fn has_verified_encryption(
3623    context: &Context,
3624    mimeparser: &MimeMessage,
3625    from_id: ContactId,
3626) -> Result<VerifiedEncryption> {
3627    use VerifiedEncryption::*;
3628
3629    if !mimeparser.was_encrypted() {
3630        return Ok(NotVerified("This message is not encrypted".to_string()));
3631    };
3632
3633    if from_id == ContactId::SELF {
3634        return Ok(Verified);
3635    }
3636
3637    let from_contact = Contact::get_by_id(context, from_id).await?;
3638
3639    let Some(fingerprint) = from_contact.fingerprint() else {
3640        return Ok(NotVerified(
3641            "The message was sent without encryption".to_string(),
3642        ));
3643    };
3644
3645    if from_contact.get_verifier_id(context).await?.is_none() {
3646        return Ok(NotVerified(
3647            "The message was sent by non-verified contact".to_string(),
3648        ));
3649    }
3650
3651    let signed_with_verified_key = mimeparser.signatures.contains(&fingerprint);
3652    if signed_with_verified_key {
3653        Ok(Verified)
3654    } else {
3655        Ok(NotVerified(
3656            "The message was sent with non-verified encryption".to_string(),
3657        ))
3658    }
3659}
3660
3661async fn mark_recipients_as_verified(
3662    context: &Context,
3663    from_id: ContactId,
3664    to_ids: &[Option<ContactId>],
3665    mimeparser: &MimeMessage,
3666) -> Result<()> {
3667    if mimeparser.get_header(HeaderDef::ChatVerified).is_none() {
3668        return Ok(());
3669    }
3670    for to_id in to_ids.iter().filter_map(|&x| x) {
3671        if to_id == ContactId::SELF || to_id == from_id {
3672            continue;
3673        }
3674
3675        mark_contact_id_as_verified(context, to_id, from_id).await?;
3676        ChatId::set_protection_for_contact(context, to_id, mimeparser.timestamp_sent).await?;
3677    }
3678
3679    Ok(())
3680}
3681
3682/// Returns the last message referenced from `References` header if it is in the database.
3683///
3684/// For Delta Chat messages it is the last message in the chat of the sender.
3685async fn get_previous_message(
3686    context: &Context,
3687    mime_parser: &MimeMessage,
3688) -> Result<Option<Message>> {
3689    if let Some(field) = mime_parser.get_header(HeaderDef::References) {
3690        if let Some(rfc724mid) = parse_message_ids(field).last() {
3691            if let Some((msg_id, _)) = rfc724_mid_exists(context, rfc724mid).await? {
3692                return Message::load_from_db_optional(context, msg_id).await;
3693            }
3694        }
3695    }
3696    Ok(None)
3697}
3698
3699/// Returns the last message referenced from References: header found in the database.
3700///
3701/// If none found, tries In-Reply-To: as a fallback for classic MUAs that don't set the
3702/// References: header.
3703async fn get_parent_message(
3704    context: &Context,
3705    references: Option<&str>,
3706    in_reply_to: Option<&str>,
3707) -> Result<Option<Message>> {
3708    let mut mids = Vec::new();
3709    if let Some(field) = in_reply_to {
3710        mids = parse_message_ids(field);
3711    }
3712    if let Some(field) = references {
3713        mids.append(&mut parse_message_ids(field));
3714    }
3715    message::get_by_rfc724_mids(context, &mids).await
3716}
3717
3718pub(crate) async fn get_prefetch_parent_message(
3719    context: &Context,
3720    headers: &[mailparse::MailHeader<'_>],
3721) -> Result<Option<Message>> {
3722    get_parent_message(
3723        context,
3724        headers.get_header_value(HeaderDef::References).as_deref(),
3725        headers.get_header_value(HeaderDef::InReplyTo).as_deref(),
3726    )
3727    .await
3728}
3729
3730/// Looks up contact IDs from the database given the list of recipients.
3731async fn add_or_lookup_contacts_by_address_list(
3732    context: &Context,
3733    address_list: &[SingleInfo],
3734    origin: Origin,
3735) -> Result<Vec<Option<ContactId>>> {
3736    let mut contact_ids = Vec::new();
3737    for info in address_list {
3738        let addr = &info.addr;
3739        if !may_be_valid_addr(addr) {
3740            contact_ids.push(None);
3741            continue;
3742        }
3743        let display_name = info.display_name.as_deref();
3744        if let Ok(addr) = ContactAddress::new(addr) {
3745            let (contact_id, _) =
3746                Contact::add_or_lookup(context, display_name.unwrap_or_default(), &addr, origin)
3747                    .await?;
3748            contact_ids.push(Some(contact_id));
3749        } else {
3750            warn!(context, "Contact with address {:?} cannot exist.", addr);
3751            contact_ids.push(None);
3752        }
3753    }
3754
3755    Ok(contact_ids)
3756}
3757
3758/// Looks up contact IDs from the database given the list of recipients.
3759async fn add_or_lookup_key_contacts(
3760    context: &Context,
3761    address_list: &[SingleInfo],
3762    gossiped_keys: &HashMap<String, SignedPublicKey>,
3763    fingerprints: &[Fingerprint],
3764    origin: Origin,
3765) -> Result<Vec<Option<ContactId>>> {
3766    let mut contact_ids = Vec::new();
3767    let mut fingerprint_iter = fingerprints.iter();
3768    for info in address_list {
3769        let addr = &info.addr;
3770        if !may_be_valid_addr(addr) {
3771            contact_ids.push(None);
3772            continue;
3773        }
3774        let fingerprint: String = if let Some(fp) = fingerprint_iter.next() {
3775            // Iterator has not ran out of fingerprints yet.
3776            fp.hex()
3777        } else if let Some(key) = gossiped_keys.get(addr) {
3778            key.dc_fingerprint().hex()
3779        } else if context.is_self_addr(addr).await? {
3780            contact_ids.push(Some(ContactId::SELF));
3781            continue;
3782        } else {
3783            contact_ids.push(None);
3784            continue;
3785        };
3786        let display_name = info.display_name.as_deref();
3787        if let Ok(addr) = ContactAddress::new(addr) {
3788            let (contact_id, _) = Contact::add_or_lookup_ex(
3789                context,
3790                display_name.unwrap_or_default(),
3791                &addr,
3792                &fingerprint,
3793                origin,
3794            )
3795            .await?;
3796            contact_ids.push(Some(contact_id));
3797        } else {
3798            warn!(context, "Contact with address {:?} cannot exist.", addr);
3799            contact_ids.push(None);
3800        }
3801    }
3802
3803    ensure_and_debug_assert_eq!(contact_ids.len(), address_list.len(),);
3804    Ok(contact_ids)
3805}
3806
3807/// Looks up a key-contact by email address.
3808///
3809/// If provided, `chat_id` must be an encrypted chat ID that has key-contacts inside.
3810/// Otherwise the function searches in all contacts, preferring accepted and most recently seen ones.
3811async fn lookup_key_contact_by_address(
3812    context: &Context,
3813    addr: &str,
3814    chat_id: Option<ChatId>,
3815) -> Result<Option<ContactId>> {
3816    if context.is_self_addr(addr).await? {
3817        if chat_id.is_none() {
3818            return Ok(Some(ContactId::SELF));
3819        }
3820        let is_self_in_chat = context
3821            .sql
3822            .exists(
3823                "SELECT COUNT(*) FROM chats_contacts WHERE chat_id=? AND contact_id=1",
3824                (chat_id,),
3825            )
3826            .await?;
3827        if is_self_in_chat {
3828            return Ok(Some(ContactId::SELF));
3829        }
3830    }
3831    let contact_id: Option<ContactId> = match chat_id {
3832        Some(chat_id) => {
3833            context
3834                .sql
3835                .query_row_optional(
3836                    "SELECT id FROM contacts
3837                     WHERE contacts.addr=?
3838                     AND EXISTS (SELECT 1 FROM chats_contacts
3839                                 WHERE contact_id=contacts.id
3840                                 AND chat_id=?)
3841                     AND fingerprint<>'' -- Should always be true
3842                     ",
3843                    (addr, chat_id),
3844                    |row| {
3845                        let contact_id: ContactId = row.get(0)?;
3846                        Ok(contact_id)
3847                    },
3848                )
3849                .await?
3850        }
3851        None => {
3852            context
3853                .sql
3854                .query_row_optional(
3855                    "SELECT id FROM contacts
3856                     WHERE addr=?
3857                     AND fingerprint<>''
3858                     ORDER BY
3859                         (
3860                             SELECT COUNT(*) FROM chats c
3861                             INNER JOIN chats_contacts cc
3862                             ON c.id=cc.chat_id
3863                             WHERE c.type=?
3864                                 AND c.id>?
3865                                 AND c.blocked=?
3866                                 AND cc.contact_id=contacts.id
3867                         ) DESC,
3868                         last_seen DESC, id DESC
3869                     ",
3870                    (
3871                        addr,
3872                        Chattype::Single,
3873                        constants::DC_CHAT_ID_LAST_SPECIAL,
3874                        Blocked::Not,
3875                    ),
3876                    |row| {
3877                        let contact_id: ContactId = row.get(0)?;
3878                        Ok(contact_id)
3879                    },
3880                )
3881                .await?
3882        }
3883    };
3884    Ok(contact_id)
3885}
3886
3887async fn lookup_key_contact_by_fingerprint(
3888    context: &Context,
3889    fingerprint: &str,
3890) -> Result<Option<ContactId>> {
3891    logged_debug_assert!(
3892        context,
3893        !fingerprint.is_empty(),
3894        "lookup_key_contact_by_fingerprint: fingerprint is empty."
3895    );
3896    if fingerprint.is_empty() {
3897        // Avoid accidentally looking up a non-key-contact.
3898        return Ok(None);
3899    }
3900    if let Some(contact_id) = context
3901        .sql
3902        .query_row_optional(
3903            "SELECT id FROM contacts
3904             WHERE fingerprint=? AND fingerprint!=''",
3905            (fingerprint,),
3906            |row| {
3907                let contact_id: ContactId = row.get(0)?;
3908                Ok(contact_id)
3909            },
3910        )
3911        .await?
3912    {
3913        Ok(Some(contact_id))
3914    } else if let Some(self_fp) = self_fingerprint_opt(context).await? {
3915        if self_fp == fingerprint {
3916            Ok(Some(ContactId::SELF))
3917        } else {
3918            Ok(None)
3919        }
3920    } else {
3921        Ok(None)
3922    }
3923}
3924
3925/// Looks up key-contacts by email addresses.
3926///
3927/// `fingerprints` may be empty.
3928/// This is used as a fallback when email addresses are available,
3929/// but not the fingerprints, e.g. when core 1.157.3
3930/// client sends the `To` and `Chat-Group-Past-Members` header
3931/// but not the corresponding fingerprint list.
3932///
3933/// Lookup is restricted to the chat ID.
3934///
3935/// If contact cannot be found, `None` is returned.
3936/// This ensures that the length of the result vector
3937/// is the same as the number of addresses in the header
3938/// and it is possible to find corresponding
3939/// `Chat-Group-Member-Timestamps` items.
3940async fn lookup_key_contacts_by_address_list(
3941    context: &Context,
3942    address_list: &[SingleInfo],
3943    fingerprints: &[Fingerprint],
3944    chat_id: Option<ChatId>,
3945) -> Result<Vec<Option<ContactId>>> {
3946    let mut contact_ids = Vec::new();
3947    let mut fingerprint_iter = fingerprints.iter();
3948    for info in address_list {
3949        let addr = &info.addr;
3950        if !may_be_valid_addr(addr) {
3951            contact_ids.push(None);
3952            continue;
3953        }
3954
3955        if let Some(fp) = fingerprint_iter.next() {
3956            // Iterator has not ran out of fingerprints yet.
3957            let display_name = info.display_name.as_deref();
3958            let fingerprint: String = fp.hex();
3959
3960            if let Ok(addr) = ContactAddress::new(addr) {
3961                let (contact_id, _) = Contact::add_or_lookup_ex(
3962                    context,
3963                    display_name.unwrap_or_default(),
3964                    &addr,
3965                    &fingerprint,
3966                    Origin::Hidden,
3967                )
3968                .await?;
3969                contact_ids.push(Some(contact_id));
3970            } else {
3971                warn!(context, "Contact with address {:?} cannot exist.", addr);
3972                contact_ids.push(None);
3973            }
3974        } else {
3975            let contact_id = lookup_key_contact_by_address(context, addr, chat_id).await?;
3976            contact_ids.push(contact_id);
3977        }
3978    }
3979    ensure_and_debug_assert_eq!(address_list.len(), contact_ids.len(),);
3980    Ok(contact_ids)
3981}
3982
3983/// Returns true if the message should not result in renaming
3984/// of the sender contact.
3985fn should_prevent_rename(mime_parser: &MimeMessage) -> bool {
3986    (mime_parser.is_mailinglist_message() && !mime_parser.was_encrypted())
3987        || mime_parser.get_header(HeaderDef::Sender).is_some()
3988}
3989
3990#[cfg(test)]
3991mod receive_imf_tests;