deltachat/
imap.rs

1//! # IMAP handling module.
2//!
3//! uses [async-email/async-imap](https://github.com/async-email/async-imap)
4//! to implement connect, fetch, delete functionality with standard IMAP servers.
5
6use std::{
7    cmp::max,
8    cmp::min,
9    collections::{BTreeMap, BTreeSet, HashMap},
10    iter::Peekable,
11    mem::take,
12    sync::atomic::Ordering,
13    time::{Duration, UNIX_EPOCH},
14};
15
16use anyhow::{Context as _, Result, bail, ensure, format_err};
17use async_channel::Receiver;
18use async_imap::types::{Fetch, Flag, Name, NameAttribute, UnsolicitedResponse};
19use deltachat_contact_tools::ContactAddress;
20use futures::{FutureExt as _, StreamExt, TryStreamExt};
21use futures_lite::FutureExt;
22use num_traits::FromPrimitive;
23use rand::Rng;
24use ratelimit::Ratelimit;
25use url::Url;
26
27use crate::chat::{self, ChatId, ChatIdBlocked};
28use crate::chatlist_events;
29use crate::config::Config;
30use crate::constants::{self, Blocked, Chattype, ShowEmails};
31use crate::contact::{Contact, ContactId, Modifier, Origin};
32use crate::context::Context;
33use crate::events::EventType;
34use crate::headerdef::{HeaderDef, HeaderDefMap};
35use crate::log::{LogExt, error, info, warn};
36use crate::login_param::{
37    ConfiguredLoginParam, ConfiguredServerLoginParam, prioritize_server_login_params,
38};
39use crate::message::{self, Message, MessageState, MessengerMessage, MsgId};
40use crate::mimeparser;
41use crate::net::proxy::ProxyConfig;
42use crate::net::session::SessionStream;
43use crate::oauth2::get_oauth2_access_token;
44use crate::push::encrypt_device_token;
45use crate::receive_imf::{
46    ReceivedMsg, from_field_to_contact_id, get_prefetch_parent_message, receive_imf_inner,
47};
48use crate::scheduler::connectivity::ConnectivityStore;
49use crate::stock_str;
50use crate::tools::{self, create_id, duration_to_str};
51
52pub(crate) mod capabilities;
53mod client;
54mod idle;
55pub mod scan_folders;
56pub mod select_folder;
57pub(crate) mod session;
58
59use client::{Client, determine_capabilities};
60use mailparse::SingleInfo;
61use session::Session;
62
63pub(crate) const GENERATED_PREFIX: &str = "GEN_";
64
65const RFC724MID_UID: &str = "(UID BODY.PEEK[HEADER.FIELDS (\
66                             MESSAGE-ID \
67                             X-MICROSOFT-ORIGINAL-MESSAGE-ID\
68                             )])";
69const BODY_FULL: &str = "(FLAGS BODY.PEEK[])";
70const BODY_PARTIAL: &str = "(FLAGS RFC822.SIZE BODY.PEEK[HEADER])";
71
72#[derive(Debug)]
73pub(crate) struct Imap {
74    pub(crate) idle_interrupt_receiver: Receiver<()>,
75
76    /// Email address.
77    addr: String,
78
79    /// Login parameters.
80    lp: Vec<ConfiguredServerLoginParam>,
81
82    /// Password.
83    password: String,
84
85    /// Proxy configuration.
86    proxy_config: Option<ProxyConfig>,
87
88    strict_tls: bool,
89
90    oauth2: bool,
91
92    authentication_failed_once: bool,
93
94    pub(crate) connectivity: ConnectivityStore,
95
96    conn_last_try: tools::Time,
97    conn_backoff_ms: u64,
98
99    /// Rate limit for successful IMAP connections.
100    ///
101    /// This rate limit prevents busy loop in case the server refuses logins
102    /// or in case connection gets dropped over and over due to IMAP bug,
103    /// e.g. the server returning invalid response to SELECT command
104    /// immediately after logging in or returning an error in response to LOGIN command
105    /// due to internal server error.
106    ratelimit: Ratelimit,
107}
108
109#[derive(Debug)]
110struct OAuth2 {
111    user: String,
112    access_token: String,
113}
114
115#[derive(Debug)]
116pub(crate) struct ServerMetadata {
117    /// IMAP METADATA `/shared/comment` as defined in
118    /// <https://www.rfc-editor.org/rfc/rfc5464#section-6.2.1>.
119    pub comment: Option<String>,
120
121    /// IMAP METADATA `/shared/admin` as defined in
122    /// <https://www.rfc-editor.org/rfc/rfc5464#section-6.2.2>.
123    pub admin: Option<String>,
124
125    pub iroh_relay: Option<Url>,
126}
127
128impl async_imap::Authenticator for OAuth2 {
129    type Response = String;
130
131    fn process(&mut self, _data: &[u8]) -> Self::Response {
132        format!(
133            "user={}\x01auth=Bearer {}\x01\x01",
134            self.user, self.access_token
135        )
136    }
137}
138
139#[derive(Debug, Display, PartialEq, Eq, Clone, Copy)]
140pub enum FolderMeaning {
141    Unknown,
142
143    /// Spam folder.
144    Spam,
145    Inbox,
146    Mvbox,
147    Sent,
148    Trash,
149    Drafts,
150
151    /// Virtual folders.
152    ///
153    /// On Gmail there are virtual folders marked as \\All, \\Important and \\Flagged.
154    /// Delta Chat ignores these folders because the same messages can be fetched
155    /// from the real folder and the result of moving and deleting messages via
156    /// virtual folder is unclear.
157    Virtual,
158}
159
160impl FolderMeaning {
161    pub fn to_config(self) -> Option<Config> {
162        match self {
163            FolderMeaning::Unknown => None,
164            FolderMeaning::Spam => None,
165            FolderMeaning::Inbox => Some(Config::ConfiguredInboxFolder),
166            FolderMeaning::Mvbox => Some(Config::ConfiguredMvboxFolder),
167            FolderMeaning::Sent => Some(Config::ConfiguredSentboxFolder),
168            FolderMeaning::Trash => Some(Config::ConfiguredTrashFolder),
169            FolderMeaning::Drafts => None,
170            FolderMeaning::Virtual => None,
171        }
172    }
173}
174
175struct UidGrouper<T: Iterator<Item = (i64, u32, String)>> {
176    inner: Peekable<T>,
177}
178
179impl<T, I> From<I> for UidGrouper<T>
180where
181    T: Iterator<Item = (i64, u32, String)>,
182    I: IntoIterator<IntoIter = T>,
183{
184    fn from(inner: I) -> Self {
185        Self {
186            inner: inner.into_iter().peekable(),
187        }
188    }
189}
190
191impl<T: Iterator<Item = (i64, u32, String)>> Iterator for UidGrouper<T> {
192    // Tuple of folder, row IDs, and UID range as a string.
193    type Item = (String, Vec<i64>, String);
194
195    fn next(&mut self) -> Option<Self::Item> {
196        let (_, _, folder) = self.inner.peek().cloned()?;
197
198        let mut uid_set = String::new();
199        let mut rowid_set = Vec::new();
200
201        while uid_set.len() < 1000 {
202            // Construct a new range.
203            if let Some((start_rowid, start_uid, _)) = self
204                .inner
205                .next_if(|(_, _, start_folder)| start_folder == &folder)
206            {
207                rowid_set.push(start_rowid);
208                let mut end_uid = start_uid;
209
210                while let Some((next_rowid, next_uid, _)) =
211                    self.inner.next_if(|(_, next_uid, next_folder)| {
212                        next_folder == &folder && (*next_uid == end_uid + 1 || *next_uid == end_uid)
213                    })
214                {
215                    end_uid = next_uid;
216                    rowid_set.push(next_rowid);
217                }
218
219                let uid_range = UidRange {
220                    start: start_uid,
221                    end: end_uid,
222                };
223                if !uid_set.is_empty() {
224                    uid_set.push(',');
225                }
226                uid_set.push_str(&uid_range.to_string());
227            } else {
228                break;
229            }
230        }
231
232        Some((folder, rowid_set, uid_set))
233    }
234}
235
236impl Imap {
237    /// Creates new disconnected IMAP client using the specific login parameters.
238    ///
239    /// `addr` is used to renew token if OAuth2 authentication is used.
240    pub fn new(
241        lp: Vec<ConfiguredServerLoginParam>,
242        password: String,
243        proxy_config: Option<ProxyConfig>,
244        addr: &str,
245        strict_tls: bool,
246        oauth2: bool,
247        idle_interrupt_receiver: Receiver<()>,
248    ) -> Self {
249        Imap {
250            idle_interrupt_receiver,
251            addr: addr.to_string(),
252            lp,
253            password,
254            proxy_config,
255            strict_tls,
256            oauth2,
257            authentication_failed_once: false,
258            connectivity: Default::default(),
259            conn_last_try: UNIX_EPOCH,
260            conn_backoff_ms: 0,
261            // 1 connection per minute + a burst of 2.
262            ratelimit: Ratelimit::new(Duration::new(120, 0), 2.0),
263        }
264    }
265
266    /// Creates new disconnected IMAP client using configured parameters.
267    pub async fn new_configured(
268        context: &Context,
269        idle_interrupt_receiver: Receiver<()>,
270    ) -> Result<Self> {
271        let param = ConfiguredLoginParam::load(context)
272            .await?
273            .context("Not configured")?;
274        let proxy_config = ProxyConfig::load(context).await?;
275        let strict_tls = param.strict_tls(proxy_config.is_some());
276        let imap = Self::new(
277            param.imap.clone(),
278            param.imap_password.clone(),
279            proxy_config,
280            &param.addr,
281            strict_tls,
282            param.oauth2,
283            idle_interrupt_receiver,
284        );
285        Ok(imap)
286    }
287
288    /// Connects to IMAP server and returns a new IMAP session.
289    ///
290    /// Calling this function is not enough to perform IMAP operations. Use [`Imap::prepare`]
291    /// instead if you are going to actually use connection rather than trying connection
292    /// parameters.
293    pub(crate) async fn connect(
294        &mut self,
295        context: &Context,
296        configuring: bool,
297    ) -> Result<Session> {
298        let now = tools::Time::now();
299        let until_can_send = max(
300            min(self.conn_last_try, now)
301                .checked_add(Duration::from_millis(self.conn_backoff_ms))
302                .unwrap_or(now),
303            now,
304        )
305        .duration_since(now)?;
306        let ratelimit_duration = max(until_can_send, self.ratelimit.until_can_send());
307        if !ratelimit_duration.is_zero() {
308            warn!(
309                context,
310                "IMAP got rate limited, waiting for {} until can connect.",
311                duration_to_str(ratelimit_duration),
312            );
313            let interrupted = async {
314                tokio::time::sleep(ratelimit_duration).await;
315                false
316            }
317            .race(self.idle_interrupt_receiver.recv().map(|_| true))
318            .await;
319            if interrupted {
320                info!(
321                    context,
322                    "Connecting to IMAP without waiting for ratelimit due to interrupt."
323                );
324            }
325        }
326
327        info!(context, "Connecting to IMAP server.");
328        self.connectivity.set_connecting(context).await;
329
330        self.conn_last_try = tools::Time::now();
331        const BACKOFF_MIN_MS: u64 = 2000;
332        const BACKOFF_MAX_MS: u64 = 80_000;
333        self.conn_backoff_ms = min(self.conn_backoff_ms, BACKOFF_MAX_MS / 2);
334        self.conn_backoff_ms = self.conn_backoff_ms.saturating_add(
335            rand::thread_rng().gen_range((self.conn_backoff_ms / 2)..=self.conn_backoff_ms),
336        );
337        self.conn_backoff_ms = max(BACKOFF_MIN_MS, self.conn_backoff_ms);
338
339        let login_params = prioritize_server_login_params(&context.sql, &self.lp, "imap").await?;
340        let mut first_error = None;
341        for lp in login_params {
342            info!(context, "IMAP trying to connect to {}.", &lp.connection);
343            let connection_candidate = lp.connection.clone();
344            let client = match Client::connect(
345                context,
346                self.proxy_config.clone(),
347                self.strict_tls,
348                connection_candidate,
349            )
350            .await
351            .context("IMAP failed to connect")
352            {
353                Ok(client) => client,
354                Err(err) => {
355                    warn!(context, "{err:#}.");
356                    first_error.get_or_insert(err);
357                    continue;
358                }
359            };
360
361            self.conn_backoff_ms = BACKOFF_MIN_MS;
362            self.ratelimit.send();
363
364            let imap_user: &str = lp.user.as_ref();
365            let imap_pw: &str = &self.password;
366
367            let login_res = if self.oauth2 {
368                info!(context, "Logging into IMAP server with OAuth 2.");
369                let addr: &str = self.addr.as_ref();
370
371                let token = get_oauth2_access_token(context, addr, imap_pw, true)
372                    .await?
373                    .context("IMAP could not get OAUTH token")?;
374                let auth = OAuth2 {
375                    user: imap_user.into(),
376                    access_token: token,
377                };
378                client.authenticate("XOAUTH2", auth).await
379            } else {
380                info!(context, "Logging into IMAP server with LOGIN.");
381                client.login(imap_user, imap_pw).await
382            };
383
384            match login_res {
385                Ok(mut session) => {
386                    let capabilities = determine_capabilities(&mut session).await?;
387
388                    let session = if capabilities.can_compress {
389                        info!(context, "Enabling IMAP compression.");
390                        let compressed_session = session
391                            .compress(|s| {
392                                let session_stream: Box<dyn SessionStream> = Box::new(s);
393                                session_stream
394                            })
395                            .await
396                            .context("Failed to enable IMAP compression")?;
397                        Session::new(compressed_session, capabilities)
398                    } else {
399                        Session::new(session, capabilities)
400                    };
401
402                    // Store server ID in the context to display in account info.
403                    let mut lock = context.server_id.write().await;
404                    lock.clone_from(&session.capabilities.server_id);
405
406                    self.authentication_failed_once = false;
407                    context.emit_event(EventType::ImapConnected(format!(
408                        "IMAP-LOGIN as {}",
409                        lp.user
410                    )));
411                    self.connectivity.set_preparing(context).await;
412                    info!(context, "Successfully logged into IMAP server.");
413                    return Ok(session);
414                }
415
416                Err(err) => {
417                    let imap_user = lp.user.to_owned();
418                    let message = stock_str::cannot_login(context, &imap_user).await;
419
420                    warn!(context, "IMAP failed to login: {err:#}.");
421                    first_error.get_or_insert(format_err!("{message} ({err:#})"));
422
423                    // If it looks like the password is wrong, send a notification:
424                    let _lock = context.wrong_pw_warning_mutex.lock().await;
425                    if err.to_string().to_lowercase().contains("authentication") {
426                        if self.authentication_failed_once
427                            && !configuring
428                            && context.get_config_bool(Config::NotifyAboutWrongPw).await?
429                        {
430                            let mut msg = Message::new_text(message);
431                            if let Err(e) = chat::add_device_msg_with_importance(
432                                context,
433                                None,
434                                Some(&mut msg),
435                                true,
436                            )
437                            .await
438                            {
439                                warn!(context, "Failed to add device message: {e:#}.");
440                            } else {
441                                context
442                                    .set_config_internal(Config::NotifyAboutWrongPw, None)
443                                    .await
444                                    .log_err(context)
445                                    .ok();
446                            }
447                        } else {
448                            self.authentication_failed_once = true;
449                        }
450                    } else {
451                        self.authentication_failed_once = false;
452                    }
453                }
454            }
455        }
456
457        Err(first_error.unwrap_or_else(|| format_err!("No IMAP connection candidates provided")))
458    }
459
460    /// Prepare a new IMAP session.
461    ///
462    /// This creates a new IMAP connection and ensures
463    /// that folders are created and IMAP capabilities are determined.
464    pub(crate) async fn prepare(&mut self, context: &Context) -> Result<Session> {
465        let configuring = false;
466        let mut session = match self.connect(context, configuring).await {
467            Ok(session) => session,
468            Err(err) => {
469                self.connectivity.set_err(context, &err).await;
470                return Err(err);
471            }
472        };
473
474        let folders_configured = context
475            .sql
476            .get_raw_config_int(constants::DC_FOLDERS_CONFIGURED_KEY)
477            .await?;
478        if folders_configured.unwrap_or_default() < constants::DC_FOLDERS_CONFIGURED_VERSION {
479            let is_chatmail = match context.get_config_bool(Config::FixIsChatmail).await? {
480                false => session.is_chatmail(),
481                true => context.get_config_bool(Config::IsChatmail).await?,
482            };
483            let create_mvbox = !is_chatmail || context.get_config_bool(Config::MvboxMove).await?;
484            self.configure_folders(context, &mut session, create_mvbox)
485                .await?;
486        }
487
488        Ok(session)
489    }
490
491    /// FETCH-MOVE-DELETE iteration.
492    ///
493    /// Prefetches headers and downloads new message from the folder, moves messages away from the
494    /// folder and deletes messages in the folder.
495    pub async fn fetch_move_delete(
496        &mut self,
497        context: &Context,
498        session: &mut Session,
499        watch_folder: &str,
500        folder_meaning: FolderMeaning,
501    ) -> Result<()> {
502        if !context.sql.is_open().await {
503            // probably shutdown
504            bail!("IMAP operation attempted while it is torn down");
505        }
506
507        let msgs_fetched = self
508            .fetch_new_messages(context, session, watch_folder, folder_meaning)
509            .await
510            .context("fetch_new_messages")?;
511        if msgs_fetched && context.get_config_delete_device_after().await?.is_some() {
512            // New messages were fetched and shall be deleted later, restart ephemeral loop.
513            // Note that the `Config::DeleteDeviceAfter` timer starts as soon as the messages are
514            // fetched while the per-chat ephemeral timers start as soon as the messages are marked
515            // as noticed.
516            context.scheduler.interrupt_ephemeral_task().await;
517        }
518
519        session
520            .move_delete_messages(context, watch_folder)
521            .await
522            .context("move_delete_messages")?;
523
524        Ok(())
525    }
526
527    /// Fetches new messages.
528    ///
529    /// Returns true if at least one message was fetched.
530    pub(crate) async fn fetch_new_messages(
531        &mut self,
532        context: &Context,
533        session: &mut Session,
534        folder: &str,
535        folder_meaning: FolderMeaning,
536    ) -> Result<bool> {
537        if should_ignore_folder(context, folder, folder_meaning).await? {
538            info!(context, "Not fetching from {folder:?}.");
539            session.new_mail = false;
540            return Ok(false);
541        }
542
543        let create = false;
544        let folder_exists = session
545            .select_with_uidvalidity(context, folder, create)
546            .await
547            .with_context(|| format!("Failed to select folder {folder:?}"))?;
548        if !folder_exists {
549            return Ok(false);
550        }
551
552        if !session.new_mail {
553            info!(context, "No new emails in folder {folder:?}.");
554            return Ok(false);
555        }
556        session.new_mail = false;
557
558        let uid_validity = get_uidvalidity(context, folder).await?;
559        let old_uid_next = get_uid_next(context, folder).await?;
560
561        let msgs = session.prefetch(old_uid_next).await.context("prefetch")?;
562        let read_cnt = msgs.len();
563
564        let download_limit = context.download_limit().await?;
565        let mut uids_fetch = Vec::<(_, bool /* partially? */)>::with_capacity(msgs.len() + 1);
566        let mut uid_message_ids = BTreeMap::new();
567        let mut largest_uid_skipped = None;
568        let delete_target = context.get_delete_msgs_target().await?;
569
570        // Store the info about IMAP messages in the database.
571        for (uid, ref fetch_response) in msgs {
572            let headers = match get_fetch_headers(fetch_response) {
573                Ok(headers) => headers,
574                Err(err) => {
575                    warn!(context, "Failed to parse FETCH headers: {err:#}.");
576                    continue;
577                }
578            };
579
580            let message_id = prefetch_get_message_id(&headers);
581
582            // Determine the target folder where the message should be moved to.
583            //
584            // If we have seen the message on the IMAP server before, do not move it.
585            // This is required to avoid infinite MOVE loop on IMAP servers
586            // that alias `DeltaChat` folder to other names.
587            // For example, some Dovecot servers alias `DeltaChat` folder to `INBOX.DeltaChat`.
588            // In this case Delta Chat configured with `DeltaChat` as the destination folder
589            // would detect messages in the `INBOX.DeltaChat` folder
590            // and try to move them to the `DeltaChat` folder.
591            // Such move to the same folder results in the messages
592            // getting a new UID, so the messages will be detected as new
593            // in the `INBOX.DeltaChat` folder again.
594            let _target;
595            let target = if let Some(message_id) = &message_id {
596                let msg_info =
597                    message::rfc724_mid_exists_ex(context, message_id, "deleted=1").await?;
598                let delete = if let Some((_, _, true)) = msg_info {
599                    info!(context, "Deleting locally deleted message {message_id}.");
600                    true
601                } else if let Some((_, ts_sent_old, _)) = msg_info {
602                    let is_chat_msg = headers.get_header_value(HeaderDef::ChatVersion).is_some();
603                    let ts_sent = headers
604                        .get_header_value(HeaderDef::Date)
605                        .and_then(|v| mailparse::dateparse(&v).ok())
606                        .unwrap_or_default();
607                    let is_dup = is_dup_msg(is_chat_msg, ts_sent, ts_sent_old);
608                    if is_dup {
609                        info!(context, "Deleting duplicate message {message_id}.");
610                    }
611                    is_dup
612                } else {
613                    false
614                };
615                if delete {
616                    &delete_target
617                } else if context
618                    .sql
619                    .exists(
620                        "SELECT COUNT (*) FROM imap WHERE rfc724_mid=?",
621                        (message_id,),
622                    )
623                    .await?
624                {
625                    info!(
626                        context,
627                        "Not moving the message {} that we have seen before.", &message_id
628                    );
629                    folder
630                } else {
631                    _target = target_folder(context, folder, folder_meaning, &headers).await?;
632                    &_target
633                }
634            } else {
635                // Do not move the messages without Message-ID.
636                // We cannot reliably determine if we have seen them before,
637                // so it is safer not to move them.
638                warn!(
639                    context,
640                    "Not moving the message that does not have a Message-ID."
641                );
642                folder
643            };
644
645            // Generate a fake Message-ID to identify the message in the database
646            // if the message has no real Message-ID.
647            let message_id = message_id.unwrap_or_else(create_message_id);
648
649            context
650                .sql
651                .execute(
652                    "INSERT INTO imap (rfc724_mid, folder, uid, uidvalidity, target)
653                       VALUES         (?1,         ?2,     ?3,  ?4,          ?5)
654                       ON CONFLICT(folder, uid, uidvalidity)
655                       DO UPDATE SET rfc724_mid=excluded.rfc724_mid,
656                                     target=excluded.target",
657                    (&message_id, &folder, uid, uid_validity, target),
658                )
659                .await?;
660
661            // Download only the messages which have reached their target folder if there are
662            // multiple devices. This prevents race conditions in multidevice case, where one
663            // device tries to download the message while another device moves the message at the
664            // same time. Even in single device case it is possible to fail downloading the first
665            // message, move it to the movebox and then download the second message before
666            // downloading the first one, if downloading from inbox before moving is allowed.
667            if folder == target
668                // Never download messages directly from the spam folder.
669                // If the sender is known, the message will be moved to the Inbox or Mvbox
670                // and then we download the message from there.
671                // Also see `spam_target_folder_cfg()`.
672                && folder_meaning != FolderMeaning::Spam
673                && prefetch_should_download(
674                    context,
675                    &headers,
676                    &message_id,
677                    fetch_response.flags(),
678                )
679                .await.context("prefetch_should_download")?
680            {
681                match download_limit {
682                    Some(download_limit) => uids_fetch.push((
683                        uid,
684                        fetch_response.size.unwrap_or_default() > download_limit,
685                    )),
686                    None => uids_fetch.push((uid, false)),
687                }
688                uid_message_ids.insert(uid, message_id);
689            } else {
690                largest_uid_skipped = Some(uid);
691            }
692        }
693
694        if !uids_fetch.is_empty() {
695            self.connectivity.set_working(context).await;
696        }
697
698        // Actually download messages.
699        let mut largest_uid_fetched: u32 = 0;
700        let mut received_msgs = Vec::with_capacity(uids_fetch.len());
701        let mut uids_fetch_in_batch = Vec::with_capacity(max(uids_fetch.len(), 1));
702        let mut fetch_partially = false;
703        uids_fetch.push((0, !uids_fetch.last().unwrap_or(&(0, false)).1));
704        for (uid, fp) in uids_fetch {
705            if fp != fetch_partially {
706                let (largest_uid_fetched_in_batch, received_msgs_in_batch) = session
707                    .fetch_many_msgs(
708                        context,
709                        folder,
710                        uid_validity,
711                        uids_fetch_in_batch.split_off(0),
712                        &uid_message_ids,
713                        fetch_partially,
714                    )
715                    .await
716                    .context("fetch_many_msgs")?;
717                received_msgs.extend(received_msgs_in_batch);
718                largest_uid_fetched = max(
719                    largest_uid_fetched,
720                    largest_uid_fetched_in_batch.unwrap_or(0),
721                );
722                fetch_partially = fp;
723            }
724            uids_fetch_in_batch.push(uid);
725        }
726
727        // Advance uid_next to the maximum of the largest known UID plus 1
728        // and mailbox UIDNEXT.
729        // Largest known UID is normally less than UIDNEXT,
730        // but a message may have arrived between determining UIDNEXT
731        // and executing the FETCH command.
732        let mailbox_uid_next = session
733            .selected_mailbox
734            .as_ref()
735            .with_context(|| format!("Expected {folder:?} to be selected"))?
736            .uid_next
737            .unwrap_or_default();
738        let new_uid_next = max(
739            max(largest_uid_fetched, largest_uid_skipped.unwrap_or(0)) + 1,
740            mailbox_uid_next,
741        );
742
743        if new_uid_next > old_uid_next {
744            set_uid_next(context, folder, new_uid_next).await?;
745        }
746
747        info!(context, "{} mails read from \"{}\".", read_cnt, folder);
748
749        if !received_msgs.is_empty() {
750            context.emit_event(EventType::IncomingMsgBunch);
751        }
752
753        chat::mark_old_messages_as_noticed(context, received_msgs).await?;
754
755        Ok(read_cnt > 0)
756    }
757
758    /// Read the recipients from old emails sent by the user and add them as contacts.
759    /// This way, we can already offer them some email addresses they can write to.
760    ///
761    /// Then, Fetch the last messages DC_FETCH_EXISTING_MSGS_COUNT emails from the server
762    /// and show them in the chat list.
763    pub(crate) async fn fetch_existing_msgs(
764        &mut self,
765        context: &Context,
766        session: &mut Session,
767    ) -> Result<()> {
768        add_all_recipients_as_contacts(context, session, Config::ConfiguredSentboxFolder)
769            .await
770            .context("failed to get recipients from the sentbox")?;
771        add_all_recipients_as_contacts(context, session, Config::ConfiguredMvboxFolder)
772            .await
773            .context("failed to get recipients from the movebox")?;
774        add_all_recipients_as_contacts(context, session, Config::ConfiguredInboxFolder)
775            .await
776            .context("failed to get recipients from the inbox")?;
777
778        info!(context, "Done fetching existing messages.");
779        Ok(())
780    }
781}
782
783impl Session {
784    /// Synchronizes UIDs for all folders.
785    pub(crate) async fn resync_folders(&mut self, context: &Context) -> Result<()> {
786        let all_folders = self
787            .list_folders()
788            .await
789            .context("listing folders for resync")?;
790        for folder in all_folders {
791            let folder_meaning = get_folder_meaning(&folder);
792            if folder_meaning != FolderMeaning::Virtual {
793                self.resync_folder_uids(context, folder.name(), folder_meaning)
794                    .await?;
795            }
796        }
797        Ok(())
798    }
799
800    /// Synchronizes UIDs in the database with UIDs on the server.
801    ///
802    /// It is assumed that no operations are taking place on the same
803    /// folder at the moment. Make sure to run it in the same
804    /// thread/task as other network operations on this folder to
805    /// avoid race conditions.
806    pub(crate) async fn resync_folder_uids(
807        &mut self,
808        context: &Context,
809        folder: &str,
810        folder_meaning: FolderMeaning,
811    ) -> Result<()> {
812        let uid_validity;
813        // Collect pairs of UID and Message-ID.
814        let mut msgs = BTreeMap::new();
815
816        let create = false;
817        let folder_exists = self
818            .select_with_uidvalidity(context, folder, create)
819            .await?;
820        if folder_exists {
821            let mut list = self
822                .uid_fetch("1:*", RFC724MID_UID)
823                .await
824                .with_context(|| format!("Can't resync folder {folder}"))?;
825            while let Some(fetch) = list.try_next().await? {
826                let headers = match get_fetch_headers(&fetch) {
827                    Ok(headers) => headers,
828                    Err(err) => {
829                        warn!(context, "Failed to parse FETCH headers: {}", err);
830                        continue;
831                    }
832                };
833                let message_id = prefetch_get_message_id(&headers);
834
835                if let (Some(uid), Some(rfc724_mid)) = (fetch.uid, message_id) {
836                    msgs.insert(
837                        uid,
838                        (
839                            rfc724_mid,
840                            target_folder(context, folder, folder_meaning, &headers).await?,
841                        ),
842                    );
843                }
844            }
845
846            info!(
847                context,
848                "resync_folder_uids: Collected {} message IDs in {folder}.",
849                msgs.len(),
850            );
851
852            uid_validity = get_uidvalidity(context, folder).await?;
853        } else {
854            warn!(context, "resync_folder_uids: No folder {folder}.");
855            uid_validity = 0;
856        }
857
858        // Write collected UIDs to SQLite database.
859        context
860            .sql
861            .transaction(move |transaction| {
862                transaction.execute("DELETE FROM imap WHERE folder=?", (folder,))?;
863                for (uid, (rfc724_mid, target)) in &msgs {
864                    // This may detect previously undetected moved
865                    // messages, so we update server_folder too.
866                    transaction.execute(
867                        "INSERT INTO imap (rfc724_mid, folder, uid, uidvalidity, target)
868                         VALUES           (?1,         ?2,     ?3,  ?4,          ?5)
869                         ON CONFLICT(folder, uid, uidvalidity)
870                         DO UPDATE SET rfc724_mid=excluded.rfc724_mid,
871                                       target=excluded.target",
872                        (rfc724_mid, folder, uid, uid_validity, target),
873                    )?;
874                }
875                Ok(())
876            })
877            .await?;
878        Ok(())
879    }
880
881    /// Deletes batch of messages identified by their UID from the currently
882    /// selected folder.
883    async fn delete_message_batch(
884        &mut self,
885        context: &Context,
886        uid_set: &str,
887        row_ids: Vec<i64>,
888    ) -> Result<()> {
889        // mark the message for deletion
890        self.add_flag_finalized_with_set(uid_set, "\\Deleted")
891            .await?;
892        context
893            .sql
894            .transaction(|transaction| {
895                let mut stmt = transaction.prepare("DELETE FROM imap WHERE id = ?")?;
896                for row_id in row_ids {
897                    stmt.execute((row_id,))?;
898                }
899                Ok(())
900            })
901            .await
902            .context("Cannot remove deleted messages from imap table")?;
903
904        context.emit_event(EventType::ImapMessageDeleted(format!(
905            "IMAP messages {uid_set} marked as deleted"
906        )));
907        Ok(())
908    }
909
910    /// Moves batch of messages identified by their UID from the currently
911    /// selected folder to the target folder.
912    async fn move_message_batch(
913        &mut self,
914        context: &Context,
915        set: &str,
916        row_ids: Vec<i64>,
917        target: &str,
918    ) -> Result<()> {
919        if self.can_move() {
920            match self.uid_mv(set, &target).await {
921                Ok(()) => {
922                    // Messages are moved or don't exist, IMAP returns OK response in both cases.
923                    context
924                        .sql
925                        .transaction(|transaction| {
926                            let mut stmt = transaction.prepare("DELETE FROM imap WHERE id = ?")?;
927                            for row_id in row_ids {
928                                stmt.execute((row_id,))?;
929                            }
930                            Ok(())
931                        })
932                        .await
933                        .context("Cannot delete moved messages from imap table")?;
934                    context.emit_event(EventType::ImapMessageMoved(format!(
935                        "IMAP messages {set} moved to {target}"
936                    )));
937                    return Ok(());
938                }
939                Err(err) => {
940                    if context.should_delete_to_trash().await? {
941                        error!(
942                            context,
943                            "Cannot move messages {} to {}, no fallback to COPY/DELETE because \
944                            delete_to_trash is set. Error: {:#}",
945                            set,
946                            target,
947                            err,
948                        );
949                        return Err(err.into());
950                    }
951                    warn!(
952                        context,
953                        "Cannot move messages, fallback to COPY/DELETE {} to {}: {}",
954                        set,
955                        target,
956                        err
957                    );
958                }
959            }
960        }
961
962        // Server does not support MOVE or MOVE failed.
963        // Copy messages to the destination folder if needed and mark records for deletion.
964        let copy = !context.is_trash(target).await?;
965        if copy {
966            info!(
967                context,
968                "Server does not support MOVE, fallback to COPY/DELETE {} to {}", set, target
969            );
970            self.uid_copy(&set, &target).await?;
971        } else {
972            error!(
973                context,
974                "Server does not support MOVE, fallback to DELETE {} to {}", set, target,
975            );
976        }
977        context
978            .sql
979            .transaction(|transaction| {
980                let mut stmt = transaction.prepare("UPDATE imap SET target='' WHERE id = ?")?;
981                for row_id in row_ids {
982                    stmt.execute((row_id,))?;
983                }
984                Ok(())
985            })
986            .await
987            .context("Cannot plan deletion of messages")?;
988        if copy {
989            context.emit_event(EventType::ImapMessageMoved(format!(
990                "IMAP messages {set} copied to {target}"
991            )));
992        }
993        Ok(())
994    }
995
996    /// Moves and deletes messages as planned in the `imap` table.
997    ///
998    /// This is the only place where messages are moved or deleted on the IMAP server.
999    async fn move_delete_messages(&mut self, context: &Context, folder: &str) -> Result<()> {
1000        let rows = context
1001            .sql
1002            .query_map(
1003                "SELECT id, uid, target FROM imap
1004        WHERE folder = ?
1005        AND target != folder
1006        ORDER BY target, uid",
1007                (folder,),
1008                |row| {
1009                    let rowid: i64 = row.get(0)?;
1010                    let uid: u32 = row.get(1)?;
1011                    let target: String = row.get(2)?;
1012                    Ok((rowid, uid, target))
1013                },
1014                |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
1015            )
1016            .await?;
1017
1018        for (target, rowid_set, uid_set) in UidGrouper::from(rows) {
1019            // Select folder inside the loop to avoid selecting it if there are no pending
1020            // MOVE/DELETE operations. This does not result in multiple SELECT commands
1021            // being sent because `select_folder()` does nothing if the folder is already
1022            // selected.
1023            let create = false;
1024            let folder_exists = self
1025                .select_with_uidvalidity(context, folder, create)
1026                .await?;
1027            ensure!(folder_exists, "No folder {folder}");
1028
1029            // Empty target folder name means messages should be deleted.
1030            if target.is_empty() {
1031                self.delete_message_batch(context, &uid_set, rowid_set)
1032                    .await
1033                    .with_context(|| format!("cannot delete batch of messages {:?}", &uid_set))?;
1034            } else {
1035                self.move_message_batch(context, &uid_set, rowid_set, &target)
1036                    .await
1037                    .with_context(|| {
1038                        format!(
1039                            "cannot move batch of messages {:?} to folder {:?}",
1040                            &uid_set, target
1041                        )
1042                    })?;
1043            }
1044        }
1045
1046        // Expunge folder if needed, e.g. if some jobs have
1047        // deleted messages on the server.
1048        if let Err(err) = self.maybe_close_folder(context).await {
1049            warn!(context, "Failed to close folder: {err:#}.");
1050        }
1051
1052        Ok(())
1053    }
1054
1055    /// Uploads sync messages from the `imap_send` table with `\Seen` flag set.
1056    pub(crate) async fn send_sync_msgs(&mut self, context: &Context, folder: &str) -> Result<()> {
1057        context.send_sync_msg().await?;
1058        while let Some((id, mime, msg_id, attempts)) = context
1059            .sql
1060            .query_row_optional(
1061                "SELECT id, mime, msg_id, attempts FROM imap_send ORDER BY id LIMIT 1",
1062                (),
1063                |row| {
1064                    let id: i64 = row.get(0)?;
1065                    let mime: String = row.get(1)?;
1066                    let msg_id: MsgId = row.get(2)?;
1067                    let attempts: i64 = row.get(3)?;
1068                    Ok((id, mime, msg_id, attempts))
1069                },
1070            )
1071            .await
1072            .context("Failed to SELECT from imap_send")?
1073        {
1074            let res = self
1075                .append(folder, Some("(\\Seen)"), None, mime)
1076                .await
1077                .with_context(|| format!("IMAP APPEND to {folder} failed for {msg_id}"))
1078                .log_err(context);
1079            if res.is_ok() {
1080                msg_id.set_delivered(context).await?;
1081            }
1082            const MAX_ATTEMPTS: i64 = 2;
1083            if res.is_ok() || attempts >= MAX_ATTEMPTS - 1 {
1084                context
1085                    .sql
1086                    .execute("DELETE FROM imap_send WHERE id=?", (id,))
1087                    .await
1088                    .context("Failed to delete from imap_send")?;
1089            } else {
1090                context
1091                    .sql
1092                    .execute("UPDATE imap_send SET attempts=attempts+1 WHERE id=?", (id,))
1093                    .await
1094                    .context("Failed to update imap_send.attempts")?;
1095                res?;
1096            }
1097        }
1098        Ok(())
1099    }
1100
1101    /// Stores pending `\Seen` flags for messages in `imap_markseen` table.
1102    pub(crate) async fn store_seen_flags_on_imap(&mut self, context: &Context) -> Result<()> {
1103        let rows = context
1104            .sql
1105            .query_map(
1106                "SELECT imap.id, uid, folder FROM imap, imap_markseen
1107                 WHERE imap.id = imap_markseen.id AND target = folder
1108                 ORDER BY folder, uid",
1109                [],
1110                |row| {
1111                    let rowid: i64 = row.get(0)?;
1112                    let uid: u32 = row.get(1)?;
1113                    let folder: String = row.get(2)?;
1114                    Ok((rowid, uid, folder))
1115                },
1116                |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
1117            )
1118            .await?;
1119
1120        for (folder, rowid_set, uid_set) in UidGrouper::from(rows) {
1121            let create = false;
1122            let folder_exists = match self.select_with_uidvalidity(context, &folder, create).await {
1123                Err(err) => {
1124                    warn!(
1125                        context,
1126                        "store_seen_flags_on_imap: Failed to select {folder}, will retry later: {err:#}."
1127                    );
1128                    continue;
1129                }
1130                Ok(folder_exists) => folder_exists,
1131            };
1132            if !folder_exists {
1133                warn!(context, "store_seen_flags_on_imap: No folder {folder}.");
1134            } else if let Err(err) = self.add_flag_finalized_with_set(&uid_set, "\\Seen").await {
1135                warn!(
1136                    context,
1137                    "Cannot mark messages {uid_set} in {folder} as seen, will retry later: {err:#}."
1138                );
1139                continue;
1140            } else {
1141                info!(
1142                    context,
1143                    "Marked messages {} in folder {} as seen.", uid_set, folder
1144                );
1145            }
1146            context
1147                .sql
1148                .transaction(|transaction| {
1149                    let mut stmt = transaction.prepare("DELETE FROM imap_markseen WHERE id = ?")?;
1150                    for rowid in rowid_set {
1151                        stmt.execute((rowid,))?;
1152                    }
1153                    Ok(())
1154                })
1155                .await
1156                .context("Cannot remove messages marked as seen from imap_markseen table")?;
1157        }
1158
1159        Ok(())
1160    }
1161
1162    /// Synchronizes `\Seen` flags using `CONDSTORE` extension.
1163    pub(crate) async fn sync_seen_flags(&mut self, context: &Context, folder: &str) -> Result<()> {
1164        if !self.can_condstore() {
1165            info!(
1166                context,
1167                "Server does not support CONDSTORE, skipping flag synchronization."
1168            );
1169            return Ok(());
1170        }
1171
1172        let create = false;
1173        let folder_exists = self
1174            .select_with_uidvalidity(context, folder, create)
1175            .await
1176            .context("Failed to select folder")?;
1177        if !folder_exists {
1178            return Ok(());
1179        }
1180
1181        let mailbox = self
1182            .selected_mailbox
1183            .as_ref()
1184            .with_context(|| format!("No mailbox selected, folder: {folder}"))?;
1185
1186        // Check if the mailbox supports MODSEQ.
1187        // We are not interested in actual value of HIGHESTMODSEQ.
1188        if mailbox.highest_modseq.is_none() {
1189            info!(
1190                context,
1191                "Mailbox {} does not support mod-sequences, skipping flag synchronization.", folder
1192            );
1193            return Ok(());
1194        }
1195
1196        let mut updated_chat_ids = BTreeSet::new();
1197        let uid_validity = get_uidvalidity(context, folder)
1198            .await
1199            .with_context(|| format!("failed to get UID validity for folder {folder}"))?;
1200        let mut highest_modseq = get_modseq(context, folder)
1201            .await
1202            .with_context(|| format!("failed to get MODSEQ for folder {folder}"))?;
1203        let mut list = self
1204            .uid_fetch("1:*", format!("(FLAGS) (CHANGEDSINCE {highest_modseq})"))
1205            .await
1206            .context("failed to fetch flags")?;
1207
1208        let mut got_unsolicited_fetch = false;
1209
1210        while let Some(fetch) = list
1211            .try_next()
1212            .await
1213            .context("failed to get FETCH result")?
1214        {
1215            let uid = if let Some(uid) = fetch.uid {
1216                uid
1217            } else {
1218                info!(context, "FETCH result contains no UID, skipping");
1219                got_unsolicited_fetch = true;
1220                continue;
1221            };
1222            let is_seen = fetch.flags().any(|flag| flag == Flag::Seen);
1223            if is_seen {
1224                if let Some(chat_id) = mark_seen_by_uid(context, folder, uid_validity, uid)
1225                    .await
1226                    .with_context(|| {
1227                        format!("failed to update seen status for msg {folder}/{uid}")
1228                    })?
1229                {
1230                    updated_chat_ids.insert(chat_id);
1231                }
1232            }
1233
1234            if let Some(modseq) = fetch.modseq {
1235                if modseq > highest_modseq {
1236                    highest_modseq = modseq;
1237                }
1238            } else {
1239                warn!(context, "FETCH result contains no MODSEQ");
1240            }
1241        }
1242        drop(list);
1243
1244        if got_unsolicited_fetch {
1245            // We got unsolicited FETCH, which means some flags
1246            // have been modified while our request was in progress.
1247            // We may or may not have these new flags as a part of the response,
1248            // so better skip next IDLE and do another round of flag synchronization.
1249            self.new_mail = true;
1250        }
1251
1252        set_modseq(context, folder, highest_modseq)
1253            .await
1254            .with_context(|| format!("failed to set MODSEQ for folder {folder}"))?;
1255        if !updated_chat_ids.is_empty() {
1256            context.on_archived_chats_maybe_noticed();
1257        }
1258        for updated_chat_id in updated_chat_ids {
1259            context.emit_event(EventType::MsgsNoticed(updated_chat_id));
1260            chatlist_events::emit_chatlist_item_changed(context, updated_chat_id);
1261        }
1262
1263        Ok(())
1264    }
1265
1266    /// Gets the from, to and bcc addresses from all existing outgoing emails.
1267    pub async fn get_all_recipients(&mut self, context: &Context) -> Result<Vec<SingleInfo>> {
1268        let mut uids: Vec<_> = self
1269            .uid_search(get_imap_self_sent_search_command(context).await?)
1270            .await?
1271            .into_iter()
1272            .collect();
1273        uids.sort_unstable();
1274
1275        let mut result = Vec::new();
1276        for (_, uid_set) in build_sequence_sets(&uids)? {
1277            let mut list = self
1278                .uid_fetch(uid_set, "(UID BODY.PEEK[HEADER.FIELDS (FROM TO CC BCC)])")
1279                .await
1280                .context("IMAP Could not fetch")?;
1281
1282            while let Some(msg) = list.try_next().await? {
1283                match get_fetch_headers(&msg) {
1284                    Ok(headers) => {
1285                        if let Some(from) = mimeparser::get_from(&headers) {
1286                            if context.is_self_addr(&from.addr).await? {
1287                                result.extend(mimeparser::get_recipients(&headers));
1288                            }
1289                        }
1290                    }
1291                    Err(err) => {
1292                        warn!(context, "{}", err);
1293                        continue;
1294                    }
1295                };
1296            }
1297        }
1298        Ok(result)
1299    }
1300
1301    /// Fetches a list of messages by server UID.
1302    ///
1303    /// Returns the last UID fetched successfully and the info about each downloaded message.
1304    /// If the message is incorrect or there is a failure to write a message to the database,
1305    /// it is skipped and the error is logged.
1306    pub(crate) async fn fetch_many_msgs(
1307        &mut self,
1308        context: &Context,
1309        folder: &str,
1310        uidvalidity: u32,
1311        request_uids: Vec<u32>,
1312        uid_message_ids: &BTreeMap<u32, String>,
1313        fetch_partially: bool,
1314    ) -> Result<(Option<u32>, Vec<ReceivedMsg>)> {
1315        let mut last_uid = None;
1316        let mut received_msgs = Vec::new();
1317
1318        if request_uids.is_empty() {
1319            return Ok((last_uid, received_msgs));
1320        }
1321
1322        for (request_uids, set) in build_sequence_sets(&request_uids)? {
1323            info!(
1324                context,
1325                "Starting a {} FETCH of message set \"{}\".",
1326                if fetch_partially { "partial" } else { "full" },
1327                set
1328            );
1329            let mut fetch_responses = self
1330                .uid_fetch(
1331                    &set,
1332                    if fetch_partially {
1333                        BODY_PARTIAL
1334                    } else {
1335                        BODY_FULL
1336                    },
1337                )
1338                .await
1339                .with_context(|| {
1340                    format!("fetching messages {} from folder \"{}\"", &set, folder)
1341                })?;
1342
1343            // Map from UIDs to unprocessed FETCH results. We put unprocessed FETCH results here
1344            // when we want to process other messages first.
1345            let mut uid_msgs = HashMap::with_capacity(request_uids.len());
1346
1347            let mut count = 0;
1348            for &request_uid in &request_uids {
1349                // Check if FETCH response is already in `uid_msgs`.
1350                let mut fetch_response = uid_msgs.remove(&request_uid);
1351
1352                // Try to find a requested UID in returned FETCH responses.
1353                while fetch_response.is_none() {
1354                    let Some(next_fetch_response) = fetch_responses.next().await else {
1355                        // No more FETCH responses received from the server.
1356                        break;
1357                    };
1358
1359                    let next_fetch_response =
1360                        next_fetch_response.context("Failed to process IMAP FETCH result")?;
1361
1362                    if let Some(next_uid) = next_fetch_response.uid {
1363                        if next_uid == request_uid {
1364                            fetch_response = Some(next_fetch_response);
1365                        } else if !request_uids.contains(&next_uid) {
1366                            // (size of `request_uids` is bounded by IMAP command length limit,
1367                            // search in this vector is always fast)
1368
1369                            // Unwanted UIDs are possible because of unsolicited responses, e.g. if
1370                            // another client changes \Seen flag on a message after we do a prefetch but
1371                            // before fetch. It's not an error if we receive such unsolicited response.
1372                            info!(
1373                                context,
1374                                "Skipping not requested FETCH response for UID {}.", next_uid
1375                            );
1376                        } else if uid_msgs.insert(next_uid, next_fetch_response).is_some() {
1377                            warn!(context, "Got duplicated UID {}.", next_uid);
1378                        }
1379                    } else {
1380                        info!(context, "Skipping FETCH response without UID.");
1381                    }
1382                }
1383
1384                let fetch_response = match fetch_response {
1385                    Some(fetch) => fetch,
1386                    None => {
1387                        warn!(
1388                            context,
1389                            "Missed UID {} in the server response.", request_uid
1390                        );
1391                        continue;
1392                    }
1393                };
1394                count += 1;
1395
1396                let is_deleted = fetch_response.flags().any(|flag| flag == Flag::Deleted);
1397                let (body, partial) = if fetch_partially {
1398                    (fetch_response.header(), fetch_response.size) // `BODY.PEEK[HEADER]` goes to header() ...
1399                } else {
1400                    (fetch_response.body(), None) // ... while `BODY.PEEK[]` goes to body() - and includes header()
1401                };
1402
1403                if is_deleted {
1404                    info!(context, "Not processing deleted msg {}.", request_uid);
1405                    last_uid = Some(request_uid);
1406                    continue;
1407                }
1408
1409                let body = if let Some(body) = body {
1410                    body
1411                } else {
1412                    info!(
1413                        context,
1414                        "Not processing message {} without a BODY.", request_uid
1415                    );
1416                    last_uid = Some(request_uid);
1417                    continue;
1418                };
1419
1420                let is_seen = fetch_response.flags().any(|flag| flag == Flag::Seen);
1421
1422                let Some(rfc724_mid) = uid_message_ids.get(&request_uid) else {
1423                    error!(
1424                        context,
1425                        "No Message-ID corresponding to UID {} passed in uid_messsage_ids.",
1426                        request_uid
1427                    );
1428                    continue;
1429                };
1430
1431                info!(
1432                    context,
1433                    "Passing message UID {} to receive_imf().", request_uid
1434                );
1435                match receive_imf_inner(
1436                    context,
1437                    folder,
1438                    uidvalidity,
1439                    request_uid,
1440                    rfc724_mid,
1441                    body,
1442                    is_seen,
1443                    partial,
1444                )
1445                .await
1446                {
1447                    Ok(received_msg) => {
1448                        if let Some(m) = received_msg {
1449                            received_msgs.push(m);
1450                        }
1451                    }
1452                    Err(err) => {
1453                        warn!(context, "receive_imf error: {:#}.", err);
1454                    }
1455                };
1456                last_uid = Some(request_uid)
1457            }
1458
1459            // If we don't process the whole response, IMAP client is left in a broken state where
1460            // it will try to process the rest of response as the next response.
1461            while fetch_responses.next().await.is_some() {}
1462
1463            if count != request_uids.len() {
1464                warn!(
1465                    context,
1466                    "Failed to fetch all UIDs: got {}, requested {}, we requested the UIDs {:?}.",
1467                    count,
1468                    request_uids.len(),
1469                    request_uids,
1470                );
1471            } else {
1472                info!(
1473                    context,
1474                    "Successfully received {} UIDs.",
1475                    request_uids.len()
1476                );
1477            }
1478        }
1479
1480        Ok((last_uid, received_msgs))
1481    }
1482
1483    /// Retrieves server metadata if it is supported.
1484    ///
1485    /// We get [`/shared/comment`](https://www.rfc-editor.org/rfc/rfc5464#section-6.2.1)
1486    /// and [`/shared/admin`](https://www.rfc-editor.org/rfc/rfc5464#section-6.2.2)
1487    /// metadata.
1488    pub(crate) async fn fetch_metadata(&mut self, context: &Context) -> Result<()> {
1489        if !self.can_metadata() {
1490            return Ok(());
1491        }
1492
1493        let mut lock = context.metadata.write().await;
1494        if (*lock).is_some() {
1495            return Ok(());
1496        }
1497
1498        info!(
1499            context,
1500            "Server supports metadata, retrieving server comment and admin contact."
1501        );
1502
1503        let mut comment = None;
1504        let mut admin = None;
1505        let mut iroh_relay = None;
1506
1507        let mailbox = "";
1508        let options = "";
1509        let metadata = self
1510            .get_metadata(
1511                mailbox,
1512                options,
1513                "(/shared/comment /shared/admin /shared/vendor/deltachat/irohrelay)",
1514            )
1515            .await?;
1516        for m in metadata {
1517            match m.entry.as_ref() {
1518                "/shared/comment" => {
1519                    comment = m.value;
1520                }
1521                "/shared/admin" => {
1522                    admin = m.value;
1523                }
1524                "/shared/vendor/deltachat/irohrelay" => {
1525                    if let Some(value) = m.value {
1526                        if let Ok(url) = Url::parse(&value) {
1527                            iroh_relay = Some(url);
1528                        } else {
1529                            warn!(
1530                                context,
1531                                "Got invalid URL from iroh relay metadata: {:?}.", value
1532                            );
1533                        }
1534                    }
1535                }
1536                _ => {}
1537            }
1538        }
1539        *lock = Some(ServerMetadata {
1540            comment,
1541            admin,
1542            iroh_relay,
1543        });
1544        Ok(())
1545    }
1546
1547    /// Stores device token into /private/devicetoken IMAP METADATA of the Inbox.
1548    pub(crate) async fn register_token(&mut self, context: &Context) -> Result<()> {
1549        if context.push_subscribed.load(Ordering::Relaxed) {
1550            return Ok(());
1551        }
1552
1553        let Some(device_token) = context.push_subscriber.device_token().await else {
1554            return Ok(());
1555        };
1556
1557        if self.can_metadata() && self.can_push() {
1558            let old_encrypted_device_token =
1559                context.get_config(Config::EncryptedDeviceToken).await?;
1560
1561            // Whether we need to update encrypted device token.
1562            let device_token_changed = old_encrypted_device_token.is_none()
1563                || context.get_config(Config::DeviceToken).await?.as_ref() != Some(&device_token);
1564
1565            let new_encrypted_device_token;
1566            if device_token_changed {
1567                let encrypted_device_token = encrypt_device_token(&device_token)
1568                    .context("Failed to encrypt device token")?;
1569
1570                // We expect that the server supporting `XDELTAPUSH` capability
1571                // has non-synchronizing literals support as well:
1572                // <https://www.rfc-editor.org/rfc/rfc7888>.
1573                let encrypted_device_token_len = encrypted_device_token.len();
1574
1575                // Store device token saved on the server
1576                // to prevent storing duplicate tokens.
1577                // The server cannot deduplicate on its own
1578                // because encryption gives a different
1579                // result each time.
1580                context
1581                    .set_config_internal(Config::DeviceToken, Some(&device_token))
1582                    .await?;
1583                context
1584                    .set_config_internal(
1585                        Config::EncryptedDeviceToken,
1586                        Some(&encrypted_device_token),
1587                    )
1588                    .await?;
1589
1590                if encrypted_device_token_len <= 4096 {
1591                    new_encrypted_device_token = Some(encrypted_device_token);
1592                } else {
1593                    // If Apple or Google (FCM) gives us a very large token,
1594                    // do not even try to give it to IMAP servers.
1595                    //
1596                    // Limit of 4096 is arbitrarily selected
1597                    // to be the same as required by LITERAL- IMAP extension.
1598                    //
1599                    // Dovecot supports LITERAL+ and non-synchronizing literals
1600                    // of any length, but there is no reason for tokens
1601                    // to be that large even after OpenPGP encryption.
1602                    warn!(context, "Device token is too long for LITERAL-, ignoring.");
1603                    new_encrypted_device_token = None;
1604                }
1605            } else {
1606                new_encrypted_device_token = old_encrypted_device_token;
1607            }
1608
1609            // Store new encrypted device token on the server
1610            // even if it is the same as the old one.
1611            if let Some(encrypted_device_token) = new_encrypted_device_token {
1612                let folder = context
1613                    .get_config(Config::ConfiguredInboxFolder)
1614                    .await?
1615                    .context("INBOX is not configured")?;
1616
1617                self.run_command_and_check_ok(&format_setmetadata(
1618                    &folder,
1619                    &encrypted_device_token,
1620                ))
1621                .await
1622                .context("SETMETADATA command failed")?;
1623
1624                context.push_subscribed.store(true, Ordering::Relaxed);
1625            }
1626        } else if !context.push_subscriber.heartbeat_subscribed().await {
1627            let context = context.clone();
1628            // Subscribe for heartbeat notifications.
1629            tokio::spawn(async move { context.push_subscriber.subscribe(&context).await });
1630        }
1631
1632        Ok(())
1633    }
1634}
1635
1636fn format_setmetadata(folder: &str, device_token: &str) -> String {
1637    let device_token_len = device_token.len();
1638    format!(
1639        "SETMETADATA \"{folder}\" (/private/devicetoken {{{device_token_len}+}}\r\n{device_token})"
1640    )
1641}
1642
1643impl Session {
1644    /// Returns success if we successfully set the flag or we otherwise
1645    /// think add_flag should not be retried: Disconnection during setting
1646    /// the flag, or other imap-errors, returns Ok as well.
1647    ///
1648    /// Returning error means that the operation can be retried.
1649    async fn add_flag_finalized_with_set(&mut self, uid_set: &str, flag: &str) -> Result<()> {
1650        if flag == "\\Deleted" {
1651            self.selected_folder_needs_expunge = true;
1652        }
1653        let query = format!("+FLAGS ({flag})");
1654        let mut responses = self
1655            .uid_store(uid_set, &query)
1656            .await
1657            .with_context(|| format!("IMAP failed to store: ({uid_set}, {query})"))?;
1658        while let Some(_response) = responses.next().await {
1659            // Read all the responses
1660        }
1661        Ok(())
1662    }
1663
1664    /// Attempts to configure mvbox.
1665    ///
1666    /// Tries to find any folder examining `folders` in the order they go. If none is found, tries
1667    /// to create any folder in the same order. This method does not use LIST command to ensure that
1668    /// configuration works even if mailbox lookup is forbidden via Access Control List (see
1669    /// <https://datatracker.ietf.org/doc/html/rfc4314>).
1670    ///
1671    /// Returns first found or created folder name.
1672    async fn configure_mvbox<'a>(
1673        &mut self,
1674        context: &Context,
1675        folders: &[&'a str],
1676        create_mvbox: bool,
1677    ) -> Result<Option<&'a str>> {
1678        // Close currently selected folder if needed.
1679        // We are going to select folders using low-level EXAMINE operations below.
1680        self.maybe_close_folder(context).await?;
1681
1682        for folder in folders {
1683            info!(context, "Looking for MVBOX-folder \"{}\"...", &folder);
1684            let res = self.examine(&folder).await;
1685            if res.is_ok() {
1686                info!(
1687                    context,
1688                    "MVBOX-folder {:?} successfully selected, using it.", &folder
1689                );
1690                self.close().await?;
1691                // Before moving emails to the mvbox we need to remember its UIDVALIDITY, otherwise
1692                // emails moved before that wouldn't be fetched but considered "old" instead.
1693                let create = false;
1694                let folder_exists = self
1695                    .select_with_uidvalidity(context, folder, create)
1696                    .await?;
1697                ensure!(folder_exists, "No MVBOX folder {:?}??", &folder);
1698                return Ok(Some(folder));
1699            }
1700        }
1701
1702        if !create_mvbox {
1703            return Ok(None);
1704        }
1705        // Some servers require namespace-style folder names like "INBOX.DeltaChat", so we try all
1706        // the variants here.
1707        for folder in folders {
1708            match self
1709                .select_with_uidvalidity(context, folder, create_mvbox)
1710                .await
1711            {
1712                Ok(_) => {
1713                    info!(context, "MVBOX-folder {} created.", folder);
1714                    return Ok(Some(folder));
1715                }
1716                Err(err) => {
1717                    warn!(context, "Cannot create MVBOX-folder {:?}: {}", folder, err);
1718                }
1719            }
1720        }
1721        Ok(None)
1722    }
1723}
1724
1725impl Imap {
1726    pub(crate) async fn configure_folders(
1727        &mut self,
1728        context: &Context,
1729        session: &mut Session,
1730        create_mvbox: bool,
1731    ) -> Result<()> {
1732        let mut folders = session
1733            .list(Some(""), Some("*"))
1734            .await
1735            .context("list_folders failed")?;
1736        let mut delimiter = ".".to_string();
1737        let mut delimiter_is_default = true;
1738        let mut folder_configs = BTreeMap::new();
1739
1740        while let Some(folder) = folders.try_next().await? {
1741            info!(context, "Scanning folder: {:?}", folder);
1742
1743            // Update the delimiter iff there is a different one, but only once.
1744            if let Some(d) = folder.delimiter() {
1745                if delimiter_is_default && !d.is_empty() && delimiter != d {
1746                    delimiter = d.to_string();
1747                    delimiter_is_default = false;
1748                }
1749            }
1750
1751            let folder_meaning = get_folder_meaning_by_attrs(folder.attributes());
1752            let folder_name_meaning = get_folder_meaning_by_name(folder.name());
1753            if let Some(config) = folder_meaning.to_config() {
1754                // Always takes precedence
1755                folder_configs.insert(config, folder.name().to_string());
1756            } else if let Some(config) = folder_name_meaning.to_config() {
1757                // only set if none has been already set
1758                folder_configs
1759                    .entry(config)
1760                    .or_insert_with(|| folder.name().to_string());
1761            }
1762        }
1763        drop(folders);
1764
1765        info!(context, "Using \"{}\" as folder-delimiter.", delimiter);
1766
1767        let fallback_folder = format!("INBOX{delimiter}DeltaChat");
1768        let mvbox_folder = session
1769            .configure_mvbox(context, &["DeltaChat", &fallback_folder], create_mvbox)
1770            .await
1771            .context("failed to configure mvbox")?;
1772
1773        context
1774            .set_config_internal(Config::ConfiguredInboxFolder, Some("INBOX"))
1775            .await?;
1776        if let Some(mvbox_folder) = mvbox_folder {
1777            info!(context, "Setting MVBOX FOLDER TO {}", &mvbox_folder);
1778            context
1779                .set_config_internal(Config::ConfiguredMvboxFolder, Some(mvbox_folder))
1780                .await?;
1781        }
1782        for (config, name) in folder_configs {
1783            context.set_config_internal(config, Some(&name)).await?;
1784        }
1785        context
1786            .sql
1787            .set_raw_config_int(
1788                constants::DC_FOLDERS_CONFIGURED_KEY,
1789                constants::DC_FOLDERS_CONFIGURED_VERSION,
1790            )
1791            .await?;
1792
1793        info!(context, "FINISHED configuring IMAP-folders.");
1794        Ok(())
1795    }
1796}
1797
1798impl Session {
1799    /// Return whether the server sent an unsolicited EXISTS or FETCH response.
1800    ///
1801    /// Drains all responses from `session.unsolicited_responses` in the process.
1802    ///
1803    /// If this returns `true`, this means that new emails arrived
1804    /// or flags have been changed.
1805    /// In this case we may want to skip next IDLE and do a round
1806    /// of fetching new messages and synchronizing seen flags.
1807    fn drain_unsolicited_responses(&self, context: &Context) -> Result<bool> {
1808        use UnsolicitedResponse::*;
1809        use async_imap::imap_proto::Response;
1810        use async_imap::imap_proto::ResponseCode;
1811
1812        let folder = self.selected_folder.as_deref().unwrap_or_default();
1813        let mut should_refetch = false;
1814        while let Ok(response) = self.unsolicited_responses.try_recv() {
1815            match response {
1816                Exists(_) => {
1817                    info!(
1818                        context,
1819                        "Need to refetch {folder:?}, got unsolicited EXISTS {response:?}"
1820                    );
1821                    should_refetch = true;
1822                }
1823
1824                Expunge(_) | Recent(_) => {}
1825                Other(ref response_data) => {
1826                    match response_data.parsed() {
1827                        Response::Fetch { .. } => {
1828                            info!(
1829                                context,
1830                                "Need to refetch {folder:?}, got unsolicited FETCH {response:?}"
1831                            );
1832                            should_refetch = true;
1833                        }
1834
1835                        // We are not interested in the following responses and they are are
1836                        // sent quite frequently, so, we ignore them without logging them.
1837                        Response::Done {
1838                            code: Some(ResponseCode::CopyUid(_, _, _)),
1839                            ..
1840                        } => {}
1841
1842                        _ => {
1843                            info!(context, "{folder:?}: got unsolicited response {response:?}")
1844                        }
1845                    }
1846                }
1847                _ => {
1848                    info!(context, "{folder:?}: got unsolicited response {response:?}")
1849                }
1850            }
1851        }
1852        Ok(should_refetch)
1853    }
1854}
1855
1856async fn should_move_out_of_spam(
1857    context: &Context,
1858    headers: &[mailparse::MailHeader<'_>],
1859) -> Result<bool> {
1860    if headers.get_header_value(HeaderDef::ChatVersion).is_some() {
1861        // If this is a chat message (i.e. has a ChatVersion header), then this might be
1862        // a securejoin message. We can't find out at this point as we didn't prefetch
1863        // the SecureJoin header. So, we always move chat messages out of Spam.
1864        // Two possibilities to change this would be:
1865        // 1. Remove the `&& !context.is_spam_folder(folder).await?` check from
1866        // `fetch_new_messages()`, and then let `receive_imf()` check
1867        // if it's a spam message and should be hidden.
1868        // 2. Or add a flag to the ChatVersion header that this is a securejoin
1869        // request, and return `true` here only if the message has this flag.
1870        // `receive_imf()` can then check if the securejoin request is valid.
1871        return Ok(true);
1872    }
1873
1874    if let Some(msg) = get_prefetch_parent_message(context, headers).await? {
1875        if msg.chat_blocked != Blocked::Not {
1876            // Blocked or contact request message in the spam folder, leave it there.
1877            return Ok(false);
1878        }
1879    } else {
1880        let from = match mimeparser::get_from(headers) {
1881            Some(f) => f,
1882            None => return Ok(false),
1883        };
1884        // No chat found.
1885        let (from_id, blocked_contact, _origin) =
1886            match from_field_to_contact_id(context, &from, None, true, true)
1887                .await
1888                .context("from_field_to_contact_id")?
1889            {
1890                Some(res) => res,
1891                None => {
1892                    warn!(
1893                        context,
1894                        "Contact with From address {:?} cannot exist, not moving out of spam", from
1895                    );
1896                    return Ok(false);
1897                }
1898            };
1899        if blocked_contact {
1900            // Contact is blocked, leave the message in spam.
1901            return Ok(false);
1902        }
1903
1904        if let Some(chat_id_blocked) = ChatIdBlocked::lookup_by_contact(context, from_id).await? {
1905            if chat_id_blocked.blocked != Blocked::Not {
1906                return Ok(false);
1907            }
1908        } else if from_id != ContactId::SELF {
1909            // No chat with this contact found.
1910            return Ok(false);
1911        }
1912    }
1913
1914    Ok(true)
1915}
1916
1917/// Returns target folder for a message found in the Spam folder.
1918/// If this returns None, the message will not be moved out of the
1919/// Spam folder, and as `fetch_new_messages()` doesn't download
1920/// messages from the Spam folder, the message will be ignored.
1921async fn spam_target_folder_cfg(
1922    context: &Context,
1923    headers: &[mailparse::MailHeader<'_>],
1924) -> Result<Option<Config>> {
1925    if !should_move_out_of_spam(context, headers).await? {
1926        return Ok(None);
1927    }
1928
1929    if needs_move_to_mvbox(context, headers).await?
1930        // If OnlyFetchMvbox is set, we don't want to move the message to
1931        // the inbox or sentbox where we wouldn't fetch it again:
1932        || context.get_config_bool(Config::OnlyFetchMvbox).await?
1933    {
1934        Ok(Some(Config::ConfiguredMvboxFolder))
1935    } else {
1936        Ok(Some(Config::ConfiguredInboxFolder))
1937    }
1938}
1939
1940/// Returns `ConfiguredInboxFolder`, `ConfiguredMvboxFolder` or `ConfiguredSentboxFolder` if
1941/// the message needs to be moved from `folder`. Otherwise returns `None`.
1942pub async fn target_folder_cfg(
1943    context: &Context,
1944    folder: &str,
1945    folder_meaning: FolderMeaning,
1946    headers: &[mailparse::MailHeader<'_>],
1947) -> Result<Option<Config>> {
1948    if context.is_mvbox(folder).await? {
1949        return Ok(None);
1950    }
1951
1952    if folder_meaning == FolderMeaning::Spam {
1953        spam_target_folder_cfg(context, headers).await
1954    } else if needs_move_to_mvbox(context, headers).await? {
1955        Ok(Some(Config::ConfiguredMvboxFolder))
1956    } else {
1957        Ok(None)
1958    }
1959}
1960
1961pub async fn target_folder(
1962    context: &Context,
1963    folder: &str,
1964    folder_meaning: FolderMeaning,
1965    headers: &[mailparse::MailHeader<'_>],
1966) -> Result<String> {
1967    match target_folder_cfg(context, folder, folder_meaning, headers).await? {
1968        Some(config) => match context.get_config(config).await? {
1969            Some(target) => Ok(target),
1970            None => Ok(folder.to_string()),
1971        },
1972        None => Ok(folder.to_string()),
1973    }
1974}
1975
1976async fn needs_move_to_mvbox(
1977    context: &Context,
1978    headers: &[mailparse::MailHeader<'_>],
1979) -> Result<bool> {
1980    let has_chat_version = headers.get_header_value(HeaderDef::ChatVersion).is_some();
1981    if !context.get_config_bool(Config::IsChatmail).await?
1982        && has_chat_version
1983        && headers
1984            .get_header_value(HeaderDef::AutoSubmitted)
1985            .filter(|val| val.eq_ignore_ascii_case("auto-generated"))
1986            .is_some()
1987    {
1988        if let Some(from) = mimeparser::get_from(headers) {
1989            if context.is_self_addr(&from.addr).await? {
1990                return Ok(true);
1991            }
1992        }
1993    }
1994    if !context.get_config_bool(Config::MvboxMove).await? {
1995        return Ok(false);
1996    }
1997
1998    if headers
1999        .get_header_value(HeaderDef::AutocryptSetupMessage)
2000        .is_some()
2001    {
2002        // do not move setup messages;
2003        // there may be a non-delta device that wants to handle it
2004        return Ok(false);
2005    }
2006
2007    if has_chat_version {
2008        Ok(true)
2009    } else if let Some(parent) = get_prefetch_parent_message(context, headers).await? {
2010        match parent.is_dc_message {
2011            MessengerMessage::No => Ok(false),
2012            MessengerMessage::Yes | MessengerMessage::Reply => Ok(true),
2013        }
2014    } else {
2015        Ok(false)
2016    }
2017}
2018
2019/// Try to get the folder meaning by the name of the folder only used if the server does not support XLIST.
2020// TODO: lots languages missing - maybe there is a list somewhere on other MUAs?
2021// however, if we fail to find out the sent-folder,
2022// only watching this folder is not working. at least, this is no show stopper.
2023// CAVE: if possible, take care not to add a name here that is "sent" in one language
2024// but sth. different in others - a hard job.
2025fn get_folder_meaning_by_name(folder_name: &str) -> FolderMeaning {
2026    // source: <https://stackoverflow.com/questions/2185391/localized-gmail-imap-folders>
2027    const SENT_NAMES: &[&str] = &[
2028        "sent",
2029        "sentmail",
2030        "sent objects",
2031        "gesendet",
2032        "Sent Mail",
2033        "Sendte e-mails",
2034        "Enviados",
2035        "Messages envoyés",
2036        "Messages envoyes",
2037        "Posta inviata",
2038        "Verzonden berichten",
2039        "Wyslane",
2040        "E-mails enviados",
2041        "Correio enviado",
2042        "Enviada",
2043        "Enviado",
2044        "Gönderildi",
2045        "Inviati",
2046        "Odeslaná pošta",
2047        "Sendt",
2048        "Skickat",
2049        "Verzonden",
2050        "Wysłane",
2051        "Éléments envoyés",
2052        "Απεσταλμένα",
2053        "Отправленные",
2054        "寄件備份",
2055        "已发送邮件",
2056        "送信済み",
2057        "보낸편지함",
2058    ];
2059    const SPAM_NAMES: &[&str] = &[
2060        "spam",
2061        "junk",
2062        "Correio electrónico não solicitado",
2063        "Correo basura",
2064        "Lixo",
2065        "Nettsøppel",
2066        "Nevyžádaná pošta",
2067        "No solicitado",
2068        "Ongewenst",
2069        "Posta indesiderata",
2070        "Skräp",
2071        "Wiadomości-śmieci",
2072        "Önemsiz",
2073        "Ανεπιθύμητα",
2074        "Спам",
2075        "垃圾邮件",
2076        "垃圾郵件",
2077        "迷惑メール",
2078        "스팸",
2079    ];
2080    const DRAFT_NAMES: &[&str] = &[
2081        "Drafts",
2082        "Kladder",
2083        "Entw?rfe",
2084        "Borradores",
2085        "Brouillons",
2086        "Bozze",
2087        "Concepten",
2088        "Wersje robocze",
2089        "Rascunhos",
2090        "Entwürfe",
2091        "Koncepty",
2092        "Kopie robocze",
2093        "Taslaklar",
2094        "Utkast",
2095        "Πρόχειρα",
2096        "Черновики",
2097        "下書き",
2098        "草稿",
2099        "임시보관함",
2100    ];
2101    const TRASH_NAMES: &[&str] = &[
2102        "Trash",
2103        "Bin",
2104        "Caixote do lixo",
2105        "Cestino",
2106        "Corbeille",
2107        "Papelera",
2108        "Papierkorb",
2109        "Papirkurv",
2110        "Papperskorgen",
2111        "Prullenbak",
2112        "Rubujo",
2113        "Κάδος απορριμμάτων",
2114        "Корзина",
2115        "Кошик",
2116        "ゴミ箱",
2117        "垃圾桶",
2118        "已删除邮件",
2119        "휴지통",
2120    ];
2121    let lower = folder_name.to_lowercase();
2122
2123    if SENT_NAMES.iter().any(|s| s.to_lowercase() == lower) {
2124        FolderMeaning::Sent
2125    } else if SPAM_NAMES.iter().any(|s| s.to_lowercase() == lower) {
2126        FolderMeaning::Spam
2127    } else if DRAFT_NAMES.iter().any(|s| s.to_lowercase() == lower) {
2128        FolderMeaning::Drafts
2129    } else if TRASH_NAMES.iter().any(|s| s.to_lowercase() == lower) {
2130        FolderMeaning::Trash
2131    } else {
2132        FolderMeaning::Unknown
2133    }
2134}
2135
2136fn get_folder_meaning_by_attrs(folder_attrs: &[NameAttribute]) -> FolderMeaning {
2137    for attr in folder_attrs {
2138        match attr {
2139            NameAttribute::Trash => return FolderMeaning::Trash,
2140            NameAttribute::Sent => return FolderMeaning::Sent,
2141            NameAttribute::Junk => return FolderMeaning::Spam,
2142            NameAttribute::Drafts => return FolderMeaning::Drafts,
2143            NameAttribute::All | NameAttribute::Flagged => return FolderMeaning::Virtual,
2144            NameAttribute::Extension(label) => {
2145                match label.as_ref() {
2146                    "\\Spam" => return FolderMeaning::Spam,
2147                    "\\Important" => return FolderMeaning::Virtual,
2148                    _ => {}
2149                };
2150            }
2151            _ => {}
2152        }
2153    }
2154    FolderMeaning::Unknown
2155}
2156
2157pub(crate) fn get_folder_meaning(folder: &Name) -> FolderMeaning {
2158    match get_folder_meaning_by_attrs(folder.attributes()) {
2159        FolderMeaning::Unknown => get_folder_meaning_by_name(folder.name()),
2160        meaning => meaning,
2161    }
2162}
2163
2164/// Parses the headers from the FETCH result.
2165fn get_fetch_headers(prefetch_msg: &Fetch) -> Result<Vec<mailparse::MailHeader<'_>>> {
2166    match prefetch_msg.header() {
2167        Some(header_bytes) => {
2168            let (headers, _) = mailparse::parse_headers(header_bytes)?;
2169            Ok(headers)
2170        }
2171        None => Ok(Vec::new()),
2172    }
2173}
2174
2175pub(crate) fn prefetch_get_message_id(headers: &[mailparse::MailHeader]) -> Option<String> {
2176    headers
2177        .get_header_value(HeaderDef::XMicrosoftOriginalMessageId)
2178        .or_else(|| headers.get_header_value(HeaderDef::MessageId))
2179        .and_then(|msgid| mimeparser::parse_message_id(&msgid).ok())
2180}
2181
2182pub(crate) fn create_message_id() -> String {
2183    format!("{}{}", GENERATED_PREFIX, create_id())
2184}
2185
2186/// Returns chat by prefetched headers.
2187async fn prefetch_get_chat(
2188    context: &Context,
2189    headers: &[mailparse::MailHeader<'_>],
2190) -> Result<Option<chat::Chat>> {
2191    let parent = get_prefetch_parent_message(context, headers).await?;
2192    if let Some(parent) = &parent {
2193        return Ok(Some(
2194            chat::Chat::load_from_db(context, parent.get_chat_id()).await?,
2195        ));
2196    }
2197
2198    Ok(None)
2199}
2200
2201/// Determines whether the message should be downloaded based on prefetched headers.
2202pub(crate) async fn prefetch_should_download(
2203    context: &Context,
2204    headers: &[mailparse::MailHeader<'_>],
2205    message_id: &str,
2206    mut flags: impl Iterator<Item = Flag<'_>>,
2207) -> Result<bool> {
2208    if message::rfc724_mid_exists(context, message_id)
2209        .await?
2210        .is_some()
2211    {
2212        markseen_on_imap_table(context, message_id).await?;
2213        return Ok(false);
2214    }
2215
2216    // We do not know the Message-ID or the Message-ID is missing (in this case, we create one in
2217    // the further process).
2218
2219    if let Some(chat) = prefetch_get_chat(context, headers).await? {
2220        if chat.typ == Chattype::Group && !chat.id.is_special() {
2221            // This might be a group command, like removing a group member.
2222            // We really need to fetch this to avoid inconsistent group state.
2223            return Ok(true);
2224        }
2225    }
2226
2227    let maybe_ndn = if let Some(from) = headers.get_header_value(HeaderDef::From_) {
2228        let from = from.to_ascii_lowercase();
2229        from.contains("mailer-daemon") || from.contains("mail-daemon")
2230    } else {
2231        false
2232    };
2233
2234    // Autocrypt Setup Message should be shown even if it is from non-chat client.
2235    let is_autocrypt_setup_message = headers
2236        .get_header_value(HeaderDef::AutocryptSetupMessage)
2237        .is_some();
2238
2239    let from = match mimeparser::get_from(headers) {
2240        Some(f) => f,
2241        None => return Ok(false),
2242    };
2243    let (_from_id, blocked_contact, origin) =
2244        match from_field_to_contact_id(context, &from, None, true, true).await? {
2245            Some(res) => res,
2246            None => return Ok(false),
2247        };
2248    // prevent_rename=true as this might be a mailing list message and in this case it would be bad if we rename the contact.
2249    // (prevent_rename is the last argument of from_field_to_contact_id())
2250
2251    if flags.any(|f| f == Flag::Draft) {
2252        info!(context, "Ignoring draft message");
2253        return Ok(false);
2254    }
2255
2256    let is_chat_message = headers.get_header_value(HeaderDef::ChatVersion).is_some();
2257    let accepted_contact = origin.is_known();
2258    let is_reply_to_chat_message = get_prefetch_parent_message(context, headers)
2259        .await?
2260        .map(|parent| match parent.is_dc_message {
2261            MessengerMessage::No => false,
2262            MessengerMessage::Yes | MessengerMessage::Reply => true,
2263        })
2264        .unwrap_or_default();
2265
2266    let show_emails =
2267        ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await?).unwrap_or_default();
2268
2269    let show = is_autocrypt_setup_message
2270        || match show_emails {
2271            ShowEmails::Off => is_chat_message || is_reply_to_chat_message,
2272            ShowEmails::AcceptedContacts => {
2273                is_chat_message || is_reply_to_chat_message || accepted_contact
2274            }
2275            ShowEmails::All => true,
2276        };
2277
2278    let should_download = (show && !blocked_contact) || maybe_ndn;
2279    Ok(should_download)
2280}
2281
2282/// Returns whether a message is a duplicate (resent message).
2283pub(crate) fn is_dup_msg(is_chat_msg: bool, ts_sent: i64, ts_sent_old: i64) -> bool {
2284    // If the existing message has timestamp_sent == 0, that means we don't know its actual sent
2285    // timestamp, so don't delete the new message. E.g. outgoing messages have zero timestamp_sent
2286    // because they are stored to the db before sending. Also consider as duplicates only messages
2287    // with greater timestamp to avoid deleting both messages in a multi-device setting.
2288    is_chat_msg && ts_sent_old != 0 && ts_sent > ts_sent_old
2289}
2290
2291/// Marks messages in `msgs` table as seen, searching for them by UID.
2292///
2293/// Returns updated chat ID if any message was marked as seen.
2294async fn mark_seen_by_uid(
2295    context: &Context,
2296    folder: &str,
2297    uid_validity: u32,
2298    uid: u32,
2299) -> Result<Option<ChatId>> {
2300    if let Some((msg_id, chat_id)) = context
2301        .sql
2302        .query_row_optional(
2303            "SELECT id, chat_id FROM msgs
2304                 WHERE id > 9 AND rfc724_mid IN (
2305                   SELECT rfc724_mid FROM imap
2306                   WHERE folder=?1
2307                   AND uidvalidity=?2
2308                   AND uid=?3
2309                   LIMIT 1
2310                 )",
2311            (&folder, uid_validity, uid),
2312            |row| {
2313                let msg_id: MsgId = row.get(0)?;
2314                let chat_id: ChatId = row.get(1)?;
2315                Ok((msg_id, chat_id))
2316            },
2317        )
2318        .await
2319        .with_context(|| format!("failed to get msg and chat ID for IMAP message {folder}/{uid}"))?
2320    {
2321        let updated = context
2322            .sql
2323            .execute(
2324                "UPDATE msgs SET state=?1
2325                     WHERE (state=?2 OR state=?3)
2326                     AND id=?4",
2327                (
2328                    MessageState::InSeen,
2329                    MessageState::InFresh,
2330                    MessageState::InNoticed,
2331                    msg_id,
2332                ),
2333            )
2334            .await
2335            .with_context(|| format!("failed to update msg {msg_id} state"))?
2336            > 0;
2337
2338        if updated {
2339            msg_id
2340                .start_ephemeral_timer(context)
2341                .await
2342                .with_context(|| format!("failed to start ephemeral timer for message {msg_id}"))?;
2343            Ok(Some(chat_id))
2344        } else {
2345            // Message state has not changed.
2346            Ok(None)
2347        }
2348    } else {
2349        // There is no message is `msgs` table matching the given UID.
2350        Ok(None)
2351    }
2352}
2353
2354/// Schedule marking the message as Seen on IMAP by adding all known IMAP messages corresponding to
2355/// the given Message-ID to `imap_markseen` table.
2356pub(crate) async fn markseen_on_imap_table(context: &Context, message_id: &str) -> Result<()> {
2357    context
2358        .sql
2359        .execute(
2360            "INSERT OR IGNORE INTO imap_markseen (id)
2361             SELECT id FROM imap WHERE rfc724_mid=?",
2362            (message_id,),
2363        )
2364        .await?;
2365    context.scheduler.interrupt_inbox().await;
2366
2367    Ok(())
2368}
2369
2370/// uid_next is the next unique identifier value from the last time we fetched a folder
2371/// See <https://tools.ietf.org/html/rfc3501#section-2.3.1.1>
2372/// This function is used to update our uid_next after fetching messages.
2373pub(crate) async fn set_uid_next(context: &Context, folder: &str, uid_next: u32) -> Result<()> {
2374    context
2375        .sql
2376        .execute(
2377            "INSERT INTO imap_sync (folder, uid_next) VALUES (?,?)
2378                ON CONFLICT(folder) DO UPDATE SET uid_next=excluded.uid_next",
2379            (folder, uid_next),
2380        )
2381        .await?;
2382    Ok(())
2383}
2384
2385/// uid_next is the next unique identifier value from the last time we fetched a folder
2386/// See <https://tools.ietf.org/html/rfc3501#section-2.3.1.1>
2387/// This method returns the uid_next from the last time we fetched messages.
2388/// We can compare this to the current uid_next to find out whether there are new messages
2389/// and fetch from this value on to get all new messages.
2390async fn get_uid_next(context: &Context, folder: &str) -> Result<u32> {
2391    Ok(context
2392        .sql
2393        .query_get_value("SELECT uid_next FROM imap_sync WHERE folder=?;", (folder,))
2394        .await?
2395        .unwrap_or(0))
2396}
2397
2398pub(crate) async fn set_uidvalidity(
2399    context: &Context,
2400    folder: &str,
2401    uidvalidity: u32,
2402) -> Result<()> {
2403    context
2404        .sql
2405        .execute(
2406            "INSERT INTO imap_sync (folder, uidvalidity) VALUES (?,?)
2407                ON CONFLICT(folder) DO UPDATE SET uidvalidity=excluded.uidvalidity",
2408            (folder, uidvalidity),
2409        )
2410        .await?;
2411    Ok(())
2412}
2413
2414async fn get_uidvalidity(context: &Context, folder: &str) -> Result<u32> {
2415    Ok(context
2416        .sql
2417        .query_get_value(
2418            "SELECT uidvalidity FROM imap_sync WHERE folder=?;",
2419            (folder,),
2420        )
2421        .await?
2422        .unwrap_or(0))
2423}
2424
2425pub(crate) async fn set_modseq(context: &Context, folder: &str, modseq: u64) -> Result<()> {
2426    context
2427        .sql
2428        .execute(
2429            "INSERT INTO imap_sync (folder, modseq) VALUES (?,?)
2430                ON CONFLICT(folder) DO UPDATE SET modseq=excluded.modseq",
2431            (folder, modseq),
2432        )
2433        .await?;
2434    Ok(())
2435}
2436
2437async fn get_modseq(context: &Context, folder: &str) -> Result<u64> {
2438    Ok(context
2439        .sql
2440        .query_get_value("SELECT modseq FROM imap_sync WHERE folder=?;", (folder,))
2441        .await?
2442        .unwrap_or(0))
2443}
2444
2445/// Compute the imap search expression for all self-sent mails (for all self addresses)
2446pub(crate) async fn get_imap_self_sent_search_command(context: &Context) -> Result<String> {
2447    // See https://www.rfc-editor.org/rfc/rfc3501#section-6.4.4 for syntax of SEARCH and OR
2448    let mut search_command = format!("FROM \"{}\"", context.get_primary_self_addr().await?);
2449
2450    for item in context.get_secondary_self_addrs().await? {
2451        search_command = format!("OR ({search_command}) (FROM \"{item}\")");
2452    }
2453
2454    Ok(search_command)
2455}
2456
2457/// Deprecated, use get_uid_next() and get_uidvalidity()
2458pub async fn get_config_last_seen_uid(context: &Context, folder: &str) -> Result<(u32, u32)> {
2459    let key = format!("imap.mailbox.{folder}");
2460    if let Some(entry) = context.sql.get_raw_config(&key).await? {
2461        // the entry has the format `imap.mailbox.<folder>=<uidvalidity>:<lastseenuid>`
2462        let mut parts = entry.split(':');
2463        Ok((
2464            parts.next().unwrap_or_default().parse().unwrap_or(0),
2465            parts.next().unwrap_or_default().parse().unwrap_or(0),
2466        ))
2467    } else {
2468        Ok((0, 0))
2469    }
2470}
2471
2472/// Whether to ignore fetching messages from a folder.
2473///
2474/// This caters for the [`Config::OnlyFetchMvbox`] setting which means mails from folders
2475/// not explicitly watched should not be fetched.
2476async fn should_ignore_folder(
2477    context: &Context,
2478    folder: &str,
2479    folder_meaning: FolderMeaning,
2480) -> Result<bool> {
2481    if !context.get_config_bool(Config::OnlyFetchMvbox).await? {
2482        return Ok(false);
2483    }
2484    if context.is_sentbox(folder).await? {
2485        // Still respect the SentboxWatch setting.
2486        return Ok(!context.get_config_bool(Config::SentboxWatch).await?);
2487    }
2488    Ok(!(context.is_mvbox(folder).await? || folder_meaning == FolderMeaning::Spam))
2489}
2490
2491/// Builds a list of sequence/uid sets. The returned sets have each no more than around 1000
2492/// characters because according to <https://tools.ietf.org/html/rfc2683#section-3.2.1.5>
2493/// command lines should not be much more than 1000 chars (servers should allow at least 8000 chars)
2494fn build_sequence_sets(uids: &[u32]) -> Result<Vec<(Vec<u32>, String)>> {
2495    // first, try to find consecutive ranges:
2496    let mut ranges: Vec<UidRange> = vec![];
2497
2498    for &current in uids {
2499        if let Some(last) = ranges.last_mut() {
2500            if last.end + 1 == current {
2501                last.end = current;
2502                continue;
2503            }
2504        }
2505
2506        ranges.push(UidRange {
2507            start: current,
2508            end: current,
2509        });
2510    }
2511
2512    // Second, sort the uids into uid sets that are each below ~1000 characters
2513    let mut result = vec![];
2514    let (mut last_uids, mut last_str) = (Vec::new(), String::new());
2515    for range in ranges {
2516        last_uids.reserve((range.end - range.start + 1).try_into()?);
2517        (range.start..=range.end).for_each(|u| last_uids.push(u));
2518        if !last_str.is_empty() {
2519            last_str.push(',');
2520        }
2521        last_str.push_str(&range.to_string());
2522
2523        if last_str.len() > 990 {
2524            result.push((take(&mut last_uids), take(&mut last_str)));
2525        }
2526    }
2527    result.push((last_uids, last_str));
2528
2529    result.retain(|(_, s)| !s.is_empty());
2530    Ok(result)
2531}
2532
2533struct UidRange {
2534    start: u32,
2535    end: u32,
2536    // If start == end, then this range represents a single number
2537}
2538
2539impl std::fmt::Display for UidRange {
2540    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
2541        if self.start == self.end {
2542            write!(f, "{}", self.start)
2543        } else {
2544            write!(f, "{}:{}", self.start, self.end)
2545        }
2546    }
2547}
2548async fn add_all_recipients_as_contacts(
2549    context: &Context,
2550    session: &mut Session,
2551    folder: Config,
2552) -> Result<()> {
2553    let mailbox = if let Some(m) = context.get_config(folder).await? {
2554        m
2555    } else {
2556        info!(
2557            context,
2558            "Folder {} is not configured, skipping fetching contacts from it.", folder
2559        );
2560        return Ok(());
2561    };
2562    let create = false;
2563    let folder_exists = session
2564        .select_with_uidvalidity(context, &mailbox, create)
2565        .await
2566        .with_context(|| format!("could not select {mailbox}"))?;
2567    if !folder_exists {
2568        return Ok(());
2569    }
2570
2571    let recipients = session
2572        .get_all_recipients(context)
2573        .await
2574        .context("could not get recipients")?;
2575
2576    let mut any_modified = false;
2577    for recipient in recipients {
2578        let recipient_addr = match ContactAddress::new(&recipient.addr) {
2579            Err(err) => {
2580                warn!(
2581                    context,
2582                    "Could not add contact for recipient with address {:?}: {:#}",
2583                    recipient.addr,
2584                    err
2585                );
2586                continue;
2587            }
2588            Ok(recipient_addr) => recipient_addr,
2589        };
2590
2591        let (_, modified) = Contact::add_or_lookup(
2592            context,
2593            &recipient.display_name.unwrap_or_default(),
2594            &recipient_addr,
2595            Origin::OutgoingTo,
2596        )
2597        .await?;
2598        if modified != Modifier::None {
2599            any_modified = true;
2600        }
2601    }
2602    if any_modified {
2603        context.emit_event(EventType::ContactsChanged(None));
2604    }
2605
2606    Ok(())
2607}
2608
2609#[cfg(test)]
2610mod imap_tests;