deltachat/
imap.rs

1//! # IMAP handling module.
2//!
3//! uses [async-email/async-imap](https://github.com/async-email/async-imap)
4//! to implement connect, fetch, delete functionality with standard IMAP servers.
5
6use std::{
7    cmp::max,
8    cmp::min,
9    collections::{BTreeMap, BTreeSet, HashMap},
10    iter::Peekable,
11    mem::take,
12    sync::atomic::Ordering,
13    time::{Duration, UNIX_EPOCH},
14};
15
16use anyhow::{Context as _, Result, bail, ensure, format_err};
17use async_channel::{self, Receiver, Sender};
18use async_imap::types::{Fetch, Flag, Name, NameAttribute, UnsolicitedResponse};
19use deltachat_contact_tools::ContactAddress;
20use futures::{FutureExt as _, TryStreamExt};
21use futures_lite::FutureExt;
22use num_traits::FromPrimitive;
23use rand::Rng;
24use ratelimit::Ratelimit;
25use url::Url;
26
27use crate::chat::{self, ChatId, ChatIdBlocked};
28use crate::chatlist_events;
29use crate::config::Config;
30use crate::constants::{self, Blocked, Chattype, ShowEmails};
31use crate::contact::{Contact, ContactId, Modifier, Origin};
32use crate::context::Context;
33use crate::events::EventType;
34use crate::headerdef::{HeaderDef, HeaderDefMap};
35use crate::log::{LogExt, error, info, warn};
36use crate::login_param::{
37    ConfiguredLoginParam, ConfiguredServerLoginParam, prioritize_server_login_params,
38};
39use crate::message::{self, Message, MessageState, MessengerMessage, MsgId};
40use crate::mimeparser;
41use crate::net::proxy::ProxyConfig;
42use crate::net::session::SessionStream;
43use crate::oauth2::get_oauth2_access_token;
44use crate::push::encrypt_device_token;
45use crate::receive_imf::{
46    ReceivedMsg, from_field_to_contact_id, get_prefetch_parent_message, receive_imf_inner,
47};
48use crate::scheduler::connectivity::ConnectivityStore;
49use crate::stock_str;
50use crate::tools::{self, create_id, duration_to_str};
51
52pub(crate) mod capabilities;
53mod client;
54mod idle;
55pub mod scan_folders;
56pub mod select_folder;
57pub(crate) mod session;
58
59use client::{Client, determine_capabilities};
60use mailparse::SingleInfo;
61use session::Session;
62
63pub(crate) const GENERATED_PREFIX: &str = "GEN_";
64
65const RFC724MID_UID: &str = "(UID BODY.PEEK[HEADER.FIELDS (\
66                             MESSAGE-ID \
67                             X-MICROSOFT-ORIGINAL-MESSAGE-ID\
68                             )])";
69const BODY_FULL: &str = "(FLAGS BODY.PEEK[])";
70const BODY_PARTIAL: &str = "(FLAGS RFC822.SIZE BODY.PEEK[HEADER])";
71
72#[derive(Debug)]
73pub(crate) struct Imap {
74    pub(crate) idle_interrupt_receiver: Receiver<()>,
75
76    /// Email address.
77    addr: String,
78
79    /// Login parameters.
80    lp: Vec<ConfiguredServerLoginParam>,
81
82    /// Password.
83    password: String,
84
85    /// Proxy configuration.
86    proxy_config: Option<ProxyConfig>,
87
88    strict_tls: bool,
89
90    oauth2: bool,
91
92    authentication_failed_once: bool,
93
94    pub(crate) connectivity: ConnectivityStore,
95
96    conn_last_try: tools::Time,
97    conn_backoff_ms: u64,
98
99    /// Rate limit for successful IMAP connections.
100    ///
101    /// This rate limit prevents busy loop in case the server refuses logins
102    /// or in case connection gets dropped over and over due to IMAP bug,
103    /// e.g. the server returning invalid response to SELECT command
104    /// immediately after logging in or returning an error in response to LOGIN command
105    /// due to internal server error.
106    ratelimit: Ratelimit,
107}
108
109#[derive(Debug)]
110struct OAuth2 {
111    user: String,
112    access_token: String,
113}
114
115#[derive(Debug)]
116pub(crate) struct ServerMetadata {
117    /// IMAP METADATA `/shared/comment` as defined in
118    /// <https://www.rfc-editor.org/rfc/rfc5464#section-6.2.1>.
119    pub comment: Option<String>,
120
121    /// IMAP METADATA `/shared/admin` as defined in
122    /// <https://www.rfc-editor.org/rfc/rfc5464#section-6.2.2>.
123    pub admin: Option<String>,
124
125    pub iroh_relay: Option<Url>,
126}
127
128impl async_imap::Authenticator for OAuth2 {
129    type Response = String;
130
131    fn process(&mut self, _data: &[u8]) -> Self::Response {
132        format!(
133            "user={}\x01auth=Bearer {}\x01\x01",
134            self.user, self.access_token
135        )
136    }
137}
138
139#[derive(Debug, Display, PartialEq, Eq, Clone, Copy)]
140pub enum FolderMeaning {
141    Unknown,
142
143    /// Spam folder.
144    Spam,
145    Inbox,
146    Mvbox,
147    Sent,
148    Trash,
149    Drafts,
150
151    /// Virtual folders.
152    ///
153    /// On Gmail there are virtual folders marked as \\All, \\Important and \\Flagged.
154    /// Delta Chat ignores these folders because the same messages can be fetched
155    /// from the real folder and the result of moving and deleting messages via
156    /// virtual folder is unclear.
157    Virtual,
158}
159
160impl FolderMeaning {
161    pub fn to_config(self) -> Option<Config> {
162        match self {
163            FolderMeaning::Unknown => None,
164            FolderMeaning::Spam => None,
165            FolderMeaning::Inbox => Some(Config::ConfiguredInboxFolder),
166            FolderMeaning::Mvbox => Some(Config::ConfiguredMvboxFolder),
167            FolderMeaning::Sent => Some(Config::ConfiguredSentboxFolder),
168            FolderMeaning::Trash => Some(Config::ConfiguredTrashFolder),
169            FolderMeaning::Drafts => None,
170            FolderMeaning::Virtual => None,
171        }
172    }
173}
174
175struct UidGrouper<T: Iterator<Item = (i64, u32, String)>> {
176    inner: Peekable<T>,
177}
178
179impl<T, I> From<I> for UidGrouper<T>
180where
181    T: Iterator<Item = (i64, u32, String)>,
182    I: IntoIterator<IntoIter = T>,
183{
184    fn from(inner: I) -> Self {
185        Self {
186            inner: inner.into_iter().peekable(),
187        }
188    }
189}
190
191impl<T: Iterator<Item = (i64, u32, String)>> Iterator for UidGrouper<T> {
192    // Tuple of folder, row IDs, and UID range as a string.
193    type Item = (String, Vec<i64>, String);
194
195    fn next(&mut self) -> Option<Self::Item> {
196        let (_, _, folder) = self.inner.peek().cloned()?;
197
198        let mut uid_set = String::new();
199        let mut rowid_set = Vec::new();
200
201        while uid_set.len() < 1000 {
202            // Construct a new range.
203            if let Some((start_rowid, start_uid, _)) = self
204                .inner
205                .next_if(|(_, _, start_folder)| start_folder == &folder)
206            {
207                rowid_set.push(start_rowid);
208                let mut end_uid = start_uid;
209
210                while let Some((next_rowid, next_uid, _)) =
211                    self.inner.next_if(|(_, next_uid, next_folder)| {
212                        next_folder == &folder && (*next_uid == end_uid + 1 || *next_uid == end_uid)
213                    })
214                {
215                    end_uid = next_uid;
216                    rowid_set.push(next_rowid);
217                }
218
219                let uid_range = UidRange {
220                    start: start_uid,
221                    end: end_uid,
222                };
223                if !uid_set.is_empty() {
224                    uid_set.push(',');
225                }
226                uid_set.push_str(&uid_range.to_string());
227            } else {
228                break;
229            }
230        }
231
232        Some((folder, rowid_set, uid_set))
233    }
234}
235
236impl Imap {
237    /// Creates new disconnected IMAP client using the specific login parameters.
238    ///
239    /// `addr` is used to renew token if OAuth2 authentication is used.
240    pub fn new(
241        lp: Vec<ConfiguredServerLoginParam>,
242        password: String,
243        proxy_config: Option<ProxyConfig>,
244        addr: &str,
245        strict_tls: bool,
246        oauth2: bool,
247        idle_interrupt_receiver: Receiver<()>,
248    ) -> Self {
249        Imap {
250            idle_interrupt_receiver,
251            addr: addr.to_string(),
252            lp,
253            password,
254            proxy_config,
255            strict_tls,
256            oauth2,
257            authentication_failed_once: false,
258            connectivity: Default::default(),
259            conn_last_try: UNIX_EPOCH,
260            conn_backoff_ms: 0,
261            // 1 connection per minute + a burst of 2.
262            ratelimit: Ratelimit::new(Duration::new(120, 0), 2.0),
263        }
264    }
265
266    /// Creates new disconnected IMAP client using configured parameters.
267    pub async fn new_configured(
268        context: &Context,
269        idle_interrupt_receiver: Receiver<()>,
270    ) -> Result<Self> {
271        let param = ConfiguredLoginParam::load(context)
272            .await?
273            .context("Not configured")?;
274        let proxy_config = ProxyConfig::load(context).await?;
275        let strict_tls = param.strict_tls(proxy_config.is_some());
276        let imap = Self::new(
277            param.imap.clone(),
278            param.imap_password.clone(),
279            proxy_config,
280            &param.addr,
281            strict_tls,
282            param.oauth2,
283            idle_interrupt_receiver,
284        );
285        Ok(imap)
286    }
287
288    /// Connects to IMAP server and returns a new IMAP session.
289    ///
290    /// Calling this function is not enough to perform IMAP operations. Use [`Imap::prepare`]
291    /// instead if you are going to actually use connection rather than trying connection
292    /// parameters.
293    pub(crate) async fn connect(
294        &mut self,
295        context: &Context,
296        configuring: bool,
297    ) -> Result<Session> {
298        let now = tools::Time::now();
299        let until_can_send = max(
300            min(self.conn_last_try, now)
301                .checked_add(Duration::from_millis(self.conn_backoff_ms))
302                .unwrap_or(now),
303            now,
304        )
305        .duration_since(now)?;
306        let ratelimit_duration = max(until_can_send, self.ratelimit.until_can_send());
307        if !ratelimit_duration.is_zero() {
308            warn!(
309                context,
310                "IMAP got rate limited, waiting for {} until can connect.",
311                duration_to_str(ratelimit_duration),
312            );
313            let interrupted = async {
314                tokio::time::sleep(ratelimit_duration).await;
315                false
316            }
317            .race(self.idle_interrupt_receiver.recv().map(|_| true))
318            .await;
319            if interrupted {
320                info!(
321                    context,
322                    "Connecting to IMAP without waiting for ratelimit due to interrupt."
323                );
324            }
325        }
326
327        info!(context, "Connecting to IMAP server.");
328        self.connectivity.set_connecting(context).await;
329
330        self.conn_last_try = tools::Time::now();
331        const BACKOFF_MIN_MS: u64 = 2000;
332        const BACKOFF_MAX_MS: u64 = 80_000;
333        self.conn_backoff_ms = min(self.conn_backoff_ms, BACKOFF_MAX_MS / 2);
334        self.conn_backoff_ms = self.conn_backoff_ms.saturating_add(
335            rand::thread_rng().gen_range((self.conn_backoff_ms / 2)..=self.conn_backoff_ms),
336        );
337        self.conn_backoff_ms = max(BACKOFF_MIN_MS, self.conn_backoff_ms);
338
339        let login_params = prioritize_server_login_params(&context.sql, &self.lp, "imap").await?;
340        let mut first_error = None;
341        for lp in login_params {
342            info!(context, "IMAP trying to connect to {}.", &lp.connection);
343            let connection_candidate = lp.connection.clone();
344            let client = match Client::connect(
345                context,
346                self.proxy_config.clone(),
347                self.strict_tls,
348                connection_candidate,
349            )
350            .await
351            .context("IMAP failed to connect")
352            {
353                Ok(client) => client,
354                Err(err) => {
355                    warn!(context, "{err:#}.");
356                    first_error.get_or_insert(err);
357                    continue;
358                }
359            };
360
361            self.conn_backoff_ms = BACKOFF_MIN_MS;
362            self.ratelimit.send();
363
364            let imap_user: &str = lp.user.as_ref();
365            let imap_pw: &str = &self.password;
366
367            let login_res = if self.oauth2 {
368                info!(context, "Logging into IMAP server with OAuth 2.");
369                let addr: &str = self.addr.as_ref();
370
371                let token = get_oauth2_access_token(context, addr, imap_pw, true)
372                    .await?
373                    .context("IMAP could not get OAUTH token")?;
374                let auth = OAuth2 {
375                    user: imap_user.into(),
376                    access_token: token,
377                };
378                client.authenticate("XOAUTH2", auth).await
379            } else {
380                info!(context, "Logging into IMAP server with LOGIN.");
381                client.login(imap_user, imap_pw).await
382            };
383
384            match login_res {
385                Ok(mut session) => {
386                    let capabilities = determine_capabilities(&mut session).await?;
387
388                    let session = if capabilities.can_compress {
389                        info!(context, "Enabling IMAP compression.");
390                        let compressed_session = session
391                            .compress(|s| {
392                                let session_stream: Box<dyn SessionStream> = Box::new(s);
393                                session_stream
394                            })
395                            .await
396                            .context("Failed to enable IMAP compression")?;
397                        Session::new(compressed_session, capabilities)
398                    } else {
399                        Session::new(session, capabilities)
400                    };
401
402                    // Store server ID in the context to display in account info.
403                    let mut lock = context.server_id.write().await;
404                    lock.clone_from(&session.capabilities.server_id);
405
406                    self.authentication_failed_once = false;
407                    context.emit_event(EventType::ImapConnected(format!(
408                        "IMAP-LOGIN as {}",
409                        lp.user
410                    )));
411                    self.connectivity.set_preparing(context).await;
412                    info!(context, "Successfully logged into IMAP server.");
413                    return Ok(session);
414                }
415
416                Err(err) => {
417                    let imap_user = lp.user.to_owned();
418                    let message = stock_str::cannot_login(context, &imap_user).await;
419
420                    warn!(context, "IMAP failed to login: {err:#}.");
421                    first_error.get_or_insert(format_err!("{message} ({err:#})"));
422
423                    // If it looks like the password is wrong, send a notification:
424                    let _lock = context.wrong_pw_warning_mutex.lock().await;
425                    if err.to_string().to_lowercase().contains("authentication") {
426                        if self.authentication_failed_once
427                            && !configuring
428                            && context.get_config_bool(Config::NotifyAboutWrongPw).await?
429                        {
430                            let mut msg = Message::new_text(message);
431                            if let Err(e) = chat::add_device_msg_with_importance(
432                                context,
433                                None,
434                                Some(&mut msg),
435                                true,
436                            )
437                            .await
438                            {
439                                warn!(context, "Failed to add device message: {e:#}.");
440                            } else {
441                                context
442                                    .set_config_internal(Config::NotifyAboutWrongPw, None)
443                                    .await
444                                    .log_err(context)
445                                    .ok();
446                            }
447                        } else {
448                            self.authentication_failed_once = true;
449                        }
450                    } else {
451                        self.authentication_failed_once = false;
452                    }
453                }
454            }
455        }
456
457        Err(first_error.unwrap_or_else(|| format_err!("No IMAP connection candidates provided")))
458    }
459
460    /// Prepare a new IMAP session.
461    ///
462    /// This creates a new IMAP connection and ensures
463    /// that folders are created and IMAP capabilities are determined.
464    pub(crate) async fn prepare(&mut self, context: &Context) -> Result<Session> {
465        let configuring = false;
466        let mut session = match self.connect(context, configuring).await {
467            Ok(session) => session,
468            Err(err) => {
469                self.connectivity.set_err(context, &err).await;
470                return Err(err);
471            }
472        };
473
474        let folders_configured = context
475            .sql
476            .get_raw_config_int(constants::DC_FOLDERS_CONFIGURED_KEY)
477            .await?;
478        if folders_configured.unwrap_or_default() < constants::DC_FOLDERS_CONFIGURED_VERSION {
479            let is_chatmail = match context.get_config_bool(Config::FixIsChatmail).await? {
480                false => session.is_chatmail(),
481                true => context.get_config_bool(Config::IsChatmail).await?,
482            };
483            let create_mvbox = !is_chatmail || context.get_config_bool(Config::MvboxMove).await?;
484            self.configure_folders(context, &mut session, create_mvbox)
485                .await?;
486        }
487
488        Ok(session)
489    }
490
491    /// FETCH-MOVE-DELETE iteration.
492    ///
493    /// Prefetches headers and downloads new message from the folder, moves messages away from the
494    /// folder and deletes messages in the folder.
495    pub async fn fetch_move_delete(
496        &mut self,
497        context: &Context,
498        session: &mut Session,
499        watch_folder: &str,
500        folder_meaning: FolderMeaning,
501    ) -> Result<()> {
502        if !context.sql.is_open().await {
503            // probably shutdown
504            bail!("IMAP operation attempted while it is torn down");
505        }
506
507        let msgs_fetched = self
508            .fetch_new_messages(context, session, watch_folder, folder_meaning)
509            .await
510            .context("fetch_new_messages")?;
511        if msgs_fetched && context.get_config_delete_device_after().await?.is_some() {
512            // New messages were fetched and shall be deleted later, restart ephemeral loop.
513            // Note that the `Config::DeleteDeviceAfter` timer starts as soon as the messages are
514            // fetched while the per-chat ephemeral timers start as soon as the messages are marked
515            // as noticed.
516            context.scheduler.interrupt_ephemeral_task().await;
517        }
518
519        session
520            .move_delete_messages(context, watch_folder)
521            .await
522            .context("move_delete_messages")?;
523
524        Ok(())
525    }
526
527    /// Fetches new messages.
528    ///
529    /// Returns true if at least one message was fetched.
530    pub(crate) async fn fetch_new_messages(
531        &mut self,
532        context: &Context,
533        session: &mut Session,
534        folder: &str,
535        folder_meaning: FolderMeaning,
536    ) -> Result<bool> {
537        if should_ignore_folder(context, folder, folder_meaning).await? {
538            info!(context, "Not fetching from {folder:?}.");
539            session.new_mail = false;
540            return Ok(false);
541        }
542
543        let create = false;
544        let folder_exists = session
545            .select_with_uidvalidity(context, folder, create)
546            .await
547            .with_context(|| format!("Failed to select folder {folder:?}"))?;
548        if !folder_exists {
549            return Ok(false);
550        }
551
552        if !session.new_mail {
553            info!(context, "No new emails in folder {folder:?}.");
554            return Ok(false);
555        }
556        session.new_mail = false;
557
558        let uid_validity = get_uidvalidity(context, folder).await?;
559        let old_uid_next = get_uid_next(context, folder).await?;
560
561        let msgs = session.prefetch(old_uid_next).await.context("prefetch")?;
562        let read_cnt = msgs.len();
563
564        let download_limit = context.download_limit().await?;
565        let mut uids_fetch = Vec::<(u32, bool /* partially? */)>::with_capacity(msgs.len() + 1);
566        let mut uid_message_ids = BTreeMap::new();
567        let mut largest_uid_skipped = None;
568        let delete_target = context.get_delete_msgs_target().await?;
569
570        // Store the info about IMAP messages in the database.
571        for (uid, ref fetch_response) in msgs {
572            let headers = match get_fetch_headers(fetch_response) {
573                Ok(headers) => headers,
574                Err(err) => {
575                    warn!(context, "Failed to parse FETCH headers: {err:#}.");
576                    continue;
577                }
578            };
579
580            let message_id = prefetch_get_message_id(&headers);
581
582            // Determine the target folder where the message should be moved to.
583            //
584            // If we have seen the message on the IMAP server before, do not move it.
585            // This is required to avoid infinite MOVE loop on IMAP servers
586            // that alias `DeltaChat` folder to other names.
587            // For example, some Dovecot servers alias `DeltaChat` folder to `INBOX.DeltaChat`.
588            // In this case Delta Chat configured with `DeltaChat` as the destination folder
589            // would detect messages in the `INBOX.DeltaChat` folder
590            // and try to move them to the `DeltaChat` folder.
591            // Such move to the same folder results in the messages
592            // getting a new UID, so the messages will be detected as new
593            // in the `INBOX.DeltaChat` folder again.
594            let _target;
595            let target = if let Some(message_id) = &message_id {
596                let msg_info =
597                    message::rfc724_mid_exists_ex(context, message_id, "deleted=1").await?;
598                let delete = if let Some((_, _, true)) = msg_info {
599                    info!(context, "Deleting locally deleted message {message_id}.");
600                    true
601                } else if let Some((_, ts_sent_old, _)) = msg_info {
602                    let is_chat_msg = headers.get_header_value(HeaderDef::ChatVersion).is_some();
603                    let ts_sent = headers
604                        .get_header_value(HeaderDef::Date)
605                        .and_then(|v| mailparse::dateparse(&v).ok())
606                        .unwrap_or_default();
607                    let is_dup = is_dup_msg(is_chat_msg, ts_sent, ts_sent_old);
608                    if is_dup {
609                        info!(context, "Deleting duplicate message {message_id}.");
610                    }
611                    is_dup
612                } else {
613                    false
614                };
615                if delete {
616                    &delete_target
617                } else if context
618                    .sql
619                    .exists(
620                        "SELECT COUNT (*) FROM imap WHERE rfc724_mid=?",
621                        (message_id,),
622                    )
623                    .await?
624                {
625                    info!(
626                        context,
627                        "Not moving the message {} that we have seen before.", &message_id
628                    );
629                    folder
630                } else {
631                    _target = target_folder(context, folder, folder_meaning, &headers).await?;
632                    &_target
633                }
634            } else {
635                // Do not move the messages without Message-ID.
636                // We cannot reliably determine if we have seen them before,
637                // so it is safer not to move them.
638                warn!(
639                    context,
640                    "Not moving the message that does not have a Message-ID."
641                );
642                folder
643            };
644
645            // Generate a fake Message-ID to identify the message in the database
646            // if the message has no real Message-ID.
647            let message_id = message_id.unwrap_or_else(create_message_id);
648
649            context
650                .sql
651                .execute(
652                    "INSERT INTO imap (rfc724_mid, folder, uid, uidvalidity, target)
653                       VALUES         (?1,         ?2,     ?3,  ?4,          ?5)
654                       ON CONFLICT(folder, uid, uidvalidity)
655                       DO UPDATE SET rfc724_mid=excluded.rfc724_mid,
656                                     target=excluded.target",
657                    (&message_id, &folder, uid, uid_validity, target),
658                )
659                .await?;
660
661            // Download only the messages which have reached their target folder if there are
662            // multiple devices. This prevents race conditions in multidevice case, where one
663            // device tries to download the message while another device moves the message at the
664            // same time. Even in single device case it is possible to fail downloading the first
665            // message, move it to the movebox and then download the second message before
666            // downloading the first one, if downloading from inbox before moving is allowed.
667            if folder == target
668                // Never download messages directly from the spam folder.
669                // If the sender is known, the message will be moved to the Inbox or Mvbox
670                // and then we download the message from there.
671                // Also see `spam_target_folder_cfg()`.
672                && folder_meaning != FolderMeaning::Spam
673                && prefetch_should_download(
674                    context,
675                    &headers,
676                    &message_id,
677                    fetch_response.flags(),
678                )
679                .await.context("prefetch_should_download")?
680            {
681                match download_limit {
682                    Some(download_limit) => uids_fetch.push((
683                        uid,
684                        fetch_response.size.unwrap_or_default() > download_limit,
685                    )),
686                    None => uids_fetch.push((uid, false)),
687                }
688                uid_message_ids.insert(uid, message_id);
689            } else {
690                largest_uid_skipped = Some(uid);
691            }
692        }
693
694        if !uids_fetch.is_empty() {
695            self.connectivity.set_working(context).await;
696        }
697
698        let (sender, receiver) = async_channel::unbounded();
699
700        let mut received_msgs = Vec::with_capacity(uids_fetch.len());
701        let mailbox_uid_next = session
702            .selected_mailbox
703            .as_ref()
704            .with_context(|| format!("Expected {folder:?} to be selected"))?
705            .uid_next
706            .unwrap_or_default();
707
708        let update_uids_future = async {
709            let mut largest_uid_fetched: u32 = 0;
710
711            while let Ok((uid, received_msg_opt)) = receiver.recv().await {
712                largest_uid_fetched = max(largest_uid_fetched, uid);
713                if let Some(received_msg) = received_msg_opt {
714                    received_msgs.push(received_msg)
715                }
716            }
717
718            largest_uid_fetched
719        };
720
721        let actually_download_messages_future = async move {
722            let mut uids_fetch_in_batch = Vec::with_capacity(max(uids_fetch.len(), 1));
723            let mut fetch_partially = false;
724            uids_fetch.push((0, !uids_fetch.last().unwrap_or(&(0, false)).1));
725            for (uid, fp) in uids_fetch {
726                if fp != fetch_partially {
727                    session
728                        .fetch_many_msgs(
729                            context,
730                            folder,
731                            uid_validity,
732                            uids_fetch_in_batch.split_off(0),
733                            &uid_message_ids,
734                            fetch_partially,
735                            sender.clone(),
736                        )
737                        .await
738                        .context("fetch_many_msgs")?;
739                    fetch_partially = fp;
740                }
741                uids_fetch_in_batch.push(uid);
742            }
743
744            anyhow::Ok(())
745        };
746
747        let (largest_uid_fetched, fetch_res) =
748            tokio::join!(update_uids_future, actually_download_messages_future);
749
750        // Advance uid_next to the largest fetched UID plus 1.
751        //
752        // This may be larger than `mailbox_uid_next`
753        // if the message has arrived after selecting mailbox
754        // and determining its UIDNEXT and before prefetch.
755        let mut new_uid_next = largest_uid_fetched + 1;
756        if fetch_res.is_ok() {
757            // If we have successfully fetched all messages we planned during prefetch,
758            // then we have covered at least the range between old UIDNEXT
759            // and UIDNEXT of the mailbox at the time of selecting it.
760            new_uid_next = max(new_uid_next, mailbox_uid_next);
761
762            new_uid_next = max(new_uid_next, largest_uid_skipped.unwrap_or(0) + 1);
763        }
764        if new_uid_next > old_uid_next {
765            set_uid_next(context, folder, new_uid_next).await?;
766        }
767
768        info!(context, "{} mails read from \"{}\".", read_cnt, folder);
769
770        if !received_msgs.is_empty() {
771            context.emit_event(EventType::IncomingMsgBunch);
772        }
773
774        chat::mark_old_messages_as_noticed(context, received_msgs).await?;
775
776        // Now fail if fetching failed, so we will
777        // establish a new session if this one is broken.
778        fetch_res?;
779
780        Ok(read_cnt > 0)
781    }
782
783    /// Read the recipients from old emails sent by the user and add them as contacts.
784    /// This way, we can already offer them some email addresses they can write to.
785    ///
786    /// Then, Fetch the last messages DC_FETCH_EXISTING_MSGS_COUNT emails from the server
787    /// and show them in the chat list.
788    pub(crate) async fn fetch_existing_msgs(
789        &mut self,
790        context: &Context,
791        session: &mut Session,
792    ) -> Result<()> {
793        add_all_recipients_as_contacts(context, session, Config::ConfiguredSentboxFolder)
794            .await
795            .context("failed to get recipients from the sentbox")?;
796        add_all_recipients_as_contacts(context, session, Config::ConfiguredMvboxFolder)
797            .await
798            .context("failed to get recipients from the movebox")?;
799        add_all_recipients_as_contacts(context, session, Config::ConfiguredInboxFolder)
800            .await
801            .context("failed to get recipients from the inbox")?;
802
803        info!(context, "Done fetching existing messages.");
804        Ok(())
805    }
806}
807
808impl Session {
809    /// Synchronizes UIDs for all folders.
810    pub(crate) async fn resync_folders(&mut self, context: &Context) -> Result<()> {
811        let all_folders = self
812            .list_folders()
813            .await
814            .context("listing folders for resync")?;
815        for folder in all_folders {
816            let folder_meaning = get_folder_meaning(&folder);
817            if folder_meaning != FolderMeaning::Virtual {
818                self.resync_folder_uids(context, folder.name(), folder_meaning)
819                    .await?;
820            }
821        }
822        Ok(())
823    }
824
825    /// Synchronizes UIDs in the database with UIDs on the server.
826    ///
827    /// It is assumed that no operations are taking place on the same
828    /// folder at the moment. Make sure to run it in the same
829    /// thread/task as other network operations on this folder to
830    /// avoid race conditions.
831    pub(crate) async fn resync_folder_uids(
832        &mut self,
833        context: &Context,
834        folder: &str,
835        folder_meaning: FolderMeaning,
836    ) -> Result<()> {
837        let uid_validity;
838        // Collect pairs of UID and Message-ID.
839        let mut msgs = BTreeMap::new();
840
841        let create = false;
842        let folder_exists = self
843            .select_with_uidvalidity(context, folder, create)
844            .await?;
845        if folder_exists {
846            let mut list = self
847                .uid_fetch("1:*", RFC724MID_UID)
848                .await
849                .with_context(|| format!("Can't resync folder {folder}"))?;
850            while let Some(fetch) = list.try_next().await? {
851                let headers = match get_fetch_headers(&fetch) {
852                    Ok(headers) => headers,
853                    Err(err) => {
854                        warn!(context, "Failed to parse FETCH headers: {}", err);
855                        continue;
856                    }
857                };
858                let message_id = prefetch_get_message_id(&headers);
859
860                if let (Some(uid), Some(rfc724_mid)) = (fetch.uid, message_id) {
861                    msgs.insert(
862                        uid,
863                        (
864                            rfc724_mid,
865                            target_folder(context, folder, folder_meaning, &headers).await?,
866                        ),
867                    );
868                }
869            }
870
871            info!(
872                context,
873                "resync_folder_uids: Collected {} message IDs in {folder}.",
874                msgs.len(),
875            );
876
877            uid_validity = get_uidvalidity(context, folder).await?;
878        } else {
879            warn!(context, "resync_folder_uids: No folder {folder}.");
880            uid_validity = 0;
881        }
882
883        // Write collected UIDs to SQLite database.
884        context
885            .sql
886            .transaction(move |transaction| {
887                transaction.execute("DELETE FROM imap WHERE folder=?", (folder,))?;
888                for (uid, (rfc724_mid, target)) in &msgs {
889                    // This may detect previously undetected moved
890                    // messages, so we update server_folder too.
891                    transaction.execute(
892                        "INSERT INTO imap (rfc724_mid, folder, uid, uidvalidity, target)
893                         VALUES           (?1,         ?2,     ?3,  ?4,          ?5)
894                         ON CONFLICT(folder, uid, uidvalidity)
895                         DO UPDATE SET rfc724_mid=excluded.rfc724_mid,
896                                       target=excluded.target",
897                        (rfc724_mid, folder, uid, uid_validity, target),
898                    )?;
899                }
900                Ok(())
901            })
902            .await?;
903        Ok(())
904    }
905
906    /// Deletes batch of messages identified by their UID from the currently
907    /// selected folder.
908    async fn delete_message_batch(
909        &mut self,
910        context: &Context,
911        uid_set: &str,
912        row_ids: Vec<i64>,
913    ) -> Result<()> {
914        // mark the message for deletion
915        self.add_flag_finalized_with_set(uid_set, "\\Deleted")
916            .await?;
917        context
918            .sql
919            .transaction(|transaction| {
920                let mut stmt = transaction.prepare("DELETE FROM imap WHERE id = ?")?;
921                for row_id in row_ids {
922                    stmt.execute((row_id,))?;
923                }
924                Ok(())
925            })
926            .await
927            .context("Cannot remove deleted messages from imap table")?;
928
929        context.emit_event(EventType::ImapMessageDeleted(format!(
930            "IMAP messages {uid_set} marked as deleted"
931        )));
932        Ok(())
933    }
934
935    /// Moves batch of messages identified by their UID from the currently
936    /// selected folder to the target folder.
937    async fn move_message_batch(
938        &mut self,
939        context: &Context,
940        set: &str,
941        row_ids: Vec<i64>,
942        target: &str,
943    ) -> Result<()> {
944        if self.can_move() {
945            match self.uid_mv(set, &target).await {
946                Ok(()) => {
947                    // Messages are moved or don't exist, IMAP returns OK response in both cases.
948                    context
949                        .sql
950                        .transaction(|transaction| {
951                            let mut stmt = transaction.prepare("DELETE FROM imap WHERE id = ?")?;
952                            for row_id in row_ids {
953                                stmt.execute((row_id,))?;
954                            }
955                            Ok(())
956                        })
957                        .await
958                        .context("Cannot delete moved messages from imap table")?;
959                    context.emit_event(EventType::ImapMessageMoved(format!(
960                        "IMAP messages {set} moved to {target}"
961                    )));
962                    return Ok(());
963                }
964                Err(err) => {
965                    if context.should_delete_to_trash().await? {
966                        error!(
967                            context,
968                            "Cannot move messages {} to {}, no fallback to COPY/DELETE because \
969                            delete_to_trash is set. Error: {:#}",
970                            set,
971                            target,
972                            err,
973                        );
974                        return Err(err.into());
975                    }
976                    warn!(
977                        context,
978                        "Cannot move messages, fallback to COPY/DELETE {} to {}: {}",
979                        set,
980                        target,
981                        err
982                    );
983                }
984            }
985        }
986
987        // Server does not support MOVE or MOVE failed.
988        // Copy messages to the destination folder if needed and mark records for deletion.
989        let copy = !context.is_trash(target).await?;
990        if copy {
991            info!(
992                context,
993                "Server does not support MOVE, fallback to COPY/DELETE {} to {}", set, target
994            );
995            self.uid_copy(&set, &target).await?;
996        } else {
997            error!(
998                context,
999                "Server does not support MOVE, fallback to DELETE {} to {}", set, target,
1000            );
1001        }
1002        context
1003            .sql
1004            .transaction(|transaction| {
1005                let mut stmt = transaction.prepare("UPDATE imap SET target='' WHERE id = ?")?;
1006                for row_id in row_ids {
1007                    stmt.execute((row_id,))?;
1008                }
1009                Ok(())
1010            })
1011            .await
1012            .context("Cannot plan deletion of messages")?;
1013        if copy {
1014            context.emit_event(EventType::ImapMessageMoved(format!(
1015                "IMAP messages {set} copied to {target}"
1016            )));
1017        }
1018        Ok(())
1019    }
1020
1021    /// Moves and deletes messages as planned in the `imap` table.
1022    ///
1023    /// This is the only place where messages are moved or deleted on the IMAP server.
1024    async fn move_delete_messages(&mut self, context: &Context, folder: &str) -> Result<()> {
1025        let rows = context
1026            .sql
1027            .query_map(
1028                "SELECT id, uid, target FROM imap
1029        WHERE folder = ?
1030        AND target != folder
1031        ORDER BY target, uid",
1032                (folder,),
1033                |row| {
1034                    let rowid: i64 = row.get(0)?;
1035                    let uid: u32 = row.get(1)?;
1036                    let target: String = row.get(2)?;
1037                    Ok((rowid, uid, target))
1038                },
1039                |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
1040            )
1041            .await?;
1042
1043        for (target, rowid_set, uid_set) in UidGrouper::from(rows) {
1044            // Select folder inside the loop to avoid selecting it if there are no pending
1045            // MOVE/DELETE operations. This does not result in multiple SELECT commands
1046            // being sent because `select_folder()` does nothing if the folder is already
1047            // selected.
1048            let create = false;
1049            let folder_exists = self
1050                .select_with_uidvalidity(context, folder, create)
1051                .await?;
1052            ensure!(folder_exists, "No folder {folder}");
1053
1054            // Empty target folder name means messages should be deleted.
1055            if target.is_empty() {
1056                self.delete_message_batch(context, &uid_set, rowid_set)
1057                    .await
1058                    .with_context(|| format!("cannot delete batch of messages {:?}", &uid_set))?;
1059            } else {
1060                self.move_message_batch(context, &uid_set, rowid_set, &target)
1061                    .await
1062                    .with_context(|| {
1063                        format!(
1064                            "cannot move batch of messages {:?} to folder {:?}",
1065                            &uid_set, target
1066                        )
1067                    })?;
1068            }
1069        }
1070
1071        // Expunge folder if needed, e.g. if some jobs have
1072        // deleted messages on the server.
1073        if let Err(err) = self.maybe_close_folder(context).await {
1074            warn!(context, "Failed to close folder: {err:#}.");
1075        }
1076
1077        Ok(())
1078    }
1079
1080    /// Uploads sync messages from the `imap_send` table with `\Seen` flag set.
1081    pub(crate) async fn send_sync_msgs(&mut self, context: &Context, folder: &str) -> Result<()> {
1082        context.send_sync_msg().await?;
1083        while let Some((id, mime, msg_id, attempts)) = context
1084            .sql
1085            .query_row_optional(
1086                "SELECT id, mime, msg_id, attempts FROM imap_send ORDER BY id LIMIT 1",
1087                (),
1088                |row| {
1089                    let id: i64 = row.get(0)?;
1090                    let mime: String = row.get(1)?;
1091                    let msg_id: MsgId = row.get(2)?;
1092                    let attempts: i64 = row.get(3)?;
1093                    Ok((id, mime, msg_id, attempts))
1094                },
1095            )
1096            .await
1097            .context("Failed to SELECT from imap_send")?
1098        {
1099            let res = self
1100                .append(folder, Some("(\\Seen)"), None, mime)
1101                .await
1102                .with_context(|| format!("IMAP APPEND to {folder} failed for {msg_id}"))
1103                .log_err(context);
1104            if res.is_ok() {
1105                msg_id.set_delivered(context).await?;
1106            }
1107            const MAX_ATTEMPTS: i64 = 2;
1108            if res.is_ok() || attempts >= MAX_ATTEMPTS - 1 {
1109                context
1110                    .sql
1111                    .execute("DELETE FROM imap_send WHERE id=?", (id,))
1112                    .await
1113                    .context("Failed to delete from imap_send")?;
1114            } else {
1115                context
1116                    .sql
1117                    .execute("UPDATE imap_send SET attempts=attempts+1 WHERE id=?", (id,))
1118                    .await
1119                    .context("Failed to update imap_send.attempts")?;
1120                res?;
1121            }
1122        }
1123        Ok(())
1124    }
1125
1126    /// Stores pending `\Seen` flags for messages in `imap_markseen` table.
1127    pub(crate) async fn store_seen_flags_on_imap(&mut self, context: &Context) -> Result<()> {
1128        let rows = context
1129            .sql
1130            .query_map(
1131                "SELECT imap.id, uid, folder FROM imap, imap_markseen
1132                 WHERE imap.id = imap_markseen.id AND target = folder
1133                 ORDER BY folder, uid",
1134                [],
1135                |row| {
1136                    let rowid: i64 = row.get(0)?;
1137                    let uid: u32 = row.get(1)?;
1138                    let folder: String = row.get(2)?;
1139                    Ok((rowid, uid, folder))
1140                },
1141                |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
1142            )
1143            .await?;
1144
1145        for (folder, rowid_set, uid_set) in UidGrouper::from(rows) {
1146            let create = false;
1147            let folder_exists = match self.select_with_uidvalidity(context, &folder, create).await {
1148                Err(err) => {
1149                    warn!(
1150                        context,
1151                        "store_seen_flags_on_imap: Failed to select {folder}, will retry later: {err:#}."
1152                    );
1153                    continue;
1154                }
1155                Ok(folder_exists) => folder_exists,
1156            };
1157            if !folder_exists {
1158                warn!(context, "store_seen_flags_on_imap: No folder {folder}.");
1159            } else if let Err(err) = self.add_flag_finalized_with_set(&uid_set, "\\Seen").await {
1160                warn!(
1161                    context,
1162                    "Cannot mark messages {uid_set} in {folder} as seen, will retry later: {err:#}."
1163                );
1164                continue;
1165            } else {
1166                info!(
1167                    context,
1168                    "Marked messages {} in folder {} as seen.", uid_set, folder
1169                );
1170            }
1171            context
1172                .sql
1173                .transaction(|transaction| {
1174                    let mut stmt = transaction.prepare("DELETE FROM imap_markseen WHERE id = ?")?;
1175                    for rowid in rowid_set {
1176                        stmt.execute((rowid,))?;
1177                    }
1178                    Ok(())
1179                })
1180                .await
1181                .context("Cannot remove messages marked as seen from imap_markseen table")?;
1182        }
1183
1184        Ok(())
1185    }
1186
1187    /// Synchronizes `\Seen` flags using `CONDSTORE` extension.
1188    pub(crate) async fn sync_seen_flags(&mut self, context: &Context, folder: &str) -> Result<()> {
1189        if !self.can_condstore() {
1190            info!(
1191                context,
1192                "Server does not support CONDSTORE, skipping flag synchronization."
1193            );
1194            return Ok(());
1195        }
1196
1197        let create = false;
1198        let folder_exists = self
1199            .select_with_uidvalidity(context, folder, create)
1200            .await
1201            .context("Failed to select folder")?;
1202        if !folder_exists {
1203            return Ok(());
1204        }
1205
1206        let mailbox = self
1207            .selected_mailbox
1208            .as_ref()
1209            .with_context(|| format!("No mailbox selected, folder: {folder}"))?;
1210
1211        // Check if the mailbox supports MODSEQ.
1212        // We are not interested in actual value of HIGHESTMODSEQ.
1213        if mailbox.highest_modseq.is_none() {
1214            info!(
1215                context,
1216                "Mailbox {} does not support mod-sequences, skipping flag synchronization.", folder
1217            );
1218            return Ok(());
1219        }
1220
1221        let mut updated_chat_ids = BTreeSet::new();
1222        let uid_validity = get_uidvalidity(context, folder)
1223            .await
1224            .with_context(|| format!("failed to get UID validity for folder {folder}"))?;
1225        let mut highest_modseq = get_modseq(context, folder)
1226            .await
1227            .with_context(|| format!("failed to get MODSEQ for folder {folder}"))?;
1228        let mut list = self
1229            .uid_fetch("1:*", format!("(FLAGS) (CHANGEDSINCE {highest_modseq})"))
1230            .await
1231            .context("failed to fetch flags")?;
1232
1233        let mut got_unsolicited_fetch = false;
1234
1235        while let Some(fetch) = list
1236            .try_next()
1237            .await
1238            .context("failed to get FETCH result")?
1239        {
1240            let uid = if let Some(uid) = fetch.uid {
1241                uid
1242            } else {
1243                info!(context, "FETCH result contains no UID, skipping");
1244                got_unsolicited_fetch = true;
1245                continue;
1246            };
1247            let is_seen = fetch.flags().any(|flag| flag == Flag::Seen);
1248            if is_seen {
1249                if let Some(chat_id) = mark_seen_by_uid(context, folder, uid_validity, uid)
1250                    .await
1251                    .with_context(|| {
1252                        format!("failed to update seen status for msg {folder}/{uid}")
1253                    })?
1254                {
1255                    updated_chat_ids.insert(chat_id);
1256                }
1257            }
1258
1259            if let Some(modseq) = fetch.modseq {
1260                if modseq > highest_modseq {
1261                    highest_modseq = modseq;
1262                }
1263            } else {
1264                warn!(context, "FETCH result contains no MODSEQ");
1265            }
1266        }
1267        drop(list);
1268
1269        if got_unsolicited_fetch {
1270            // We got unsolicited FETCH, which means some flags
1271            // have been modified while our request was in progress.
1272            // We may or may not have these new flags as a part of the response,
1273            // so better skip next IDLE and do another round of flag synchronization.
1274            self.new_mail = true;
1275        }
1276
1277        set_modseq(context, folder, highest_modseq)
1278            .await
1279            .with_context(|| format!("failed to set MODSEQ for folder {folder}"))?;
1280        if !updated_chat_ids.is_empty() {
1281            context.on_archived_chats_maybe_noticed();
1282        }
1283        for updated_chat_id in updated_chat_ids {
1284            context.emit_event(EventType::MsgsNoticed(updated_chat_id));
1285            chatlist_events::emit_chatlist_item_changed(context, updated_chat_id);
1286        }
1287
1288        Ok(())
1289    }
1290
1291    /// Gets the from, to and bcc addresses from all existing outgoing emails.
1292    pub async fn get_all_recipients(&mut self, context: &Context) -> Result<Vec<SingleInfo>> {
1293        let mut uids: Vec<_> = self
1294            .uid_search(get_imap_self_sent_search_command(context).await?)
1295            .await?
1296            .into_iter()
1297            .collect();
1298        uids.sort_unstable();
1299
1300        let mut result = Vec::new();
1301        for (_, uid_set) in build_sequence_sets(&uids)? {
1302            let mut list = self
1303                .uid_fetch(uid_set, "(UID BODY.PEEK[HEADER.FIELDS (FROM TO CC BCC)])")
1304                .await
1305                .context("IMAP Could not fetch")?;
1306
1307            while let Some(msg) = list.try_next().await? {
1308                match get_fetch_headers(&msg) {
1309                    Ok(headers) => {
1310                        if let Some(from) = mimeparser::get_from(&headers) {
1311                            if context.is_self_addr(&from.addr).await? {
1312                                result.extend(mimeparser::get_recipients(&headers));
1313                            }
1314                        }
1315                    }
1316                    Err(err) => {
1317                        warn!(context, "{}", err);
1318                        continue;
1319                    }
1320                };
1321            }
1322        }
1323        Ok(result)
1324    }
1325
1326    /// Fetches a list of messages by server UID.
1327    ///
1328    /// Sends pairs of UID and info about each downloaded message to the provided channel.
1329    /// Received message info is optional because UID may be ignored
1330    /// if the message has a `\Deleted` flag.
1331    ///
1332    /// The channel is used to return the results because the function may fail
1333    /// due to network errors before it finishes fetching all the messages.
1334    /// In this case caller still may want to process all the results
1335    /// received over the channel and persist last seen UID in the database
1336    /// before bubbling up the failure.
1337    ///
1338    /// If the message is incorrect or there is a failure to write a message to the database,
1339    /// it is skipped and the error is logged.
1340    #[expect(clippy::too_many_arguments)]
1341    pub(crate) async fn fetch_many_msgs(
1342        &mut self,
1343        context: &Context,
1344        folder: &str,
1345        uidvalidity: u32,
1346        request_uids: Vec<u32>,
1347        uid_message_ids: &BTreeMap<u32, String>,
1348        fetch_partially: bool,
1349        received_msgs_channel: Sender<(u32, Option<ReceivedMsg>)>,
1350    ) -> Result<()> {
1351        if request_uids.is_empty() {
1352            return Ok(());
1353        }
1354
1355        for (request_uids, set) in build_sequence_sets(&request_uids)? {
1356            info!(
1357                context,
1358                "Starting a {} FETCH of message set \"{}\".",
1359                if fetch_partially { "partial" } else { "full" },
1360                set
1361            );
1362            let mut fetch_responses = self
1363                .uid_fetch(
1364                    &set,
1365                    if fetch_partially {
1366                        BODY_PARTIAL
1367                    } else {
1368                        BODY_FULL
1369                    },
1370                )
1371                .await
1372                .with_context(|| {
1373                    format!("fetching messages {} from folder \"{}\"", &set, folder)
1374                })?;
1375
1376            // Map from UIDs to unprocessed FETCH results. We put unprocessed FETCH results here
1377            // when we want to process other messages first.
1378            let mut uid_msgs = HashMap::with_capacity(request_uids.len());
1379
1380            let mut count = 0;
1381            for &request_uid in &request_uids {
1382                // Check if FETCH response is already in `uid_msgs`.
1383                let mut fetch_response = uid_msgs.remove(&request_uid);
1384
1385                // Try to find a requested UID in returned FETCH responses.
1386                while fetch_response.is_none() {
1387                    let Some(next_fetch_response) = fetch_responses
1388                        .try_next()
1389                        .await
1390                        .context("Failed to process IMAP FETCH result")?
1391                    else {
1392                        // No more FETCH responses received from the server.
1393                        break;
1394                    };
1395
1396                    if let Some(next_uid) = next_fetch_response.uid {
1397                        if next_uid == request_uid {
1398                            fetch_response = Some(next_fetch_response);
1399                        } else if !request_uids.contains(&next_uid) {
1400                            // (size of `request_uids` is bounded by IMAP command length limit,
1401                            // search in this vector is always fast)
1402
1403                            // Unwanted UIDs are possible because of unsolicited responses, e.g. if
1404                            // another client changes \Seen flag on a message after we do a prefetch but
1405                            // before fetch. It's not an error if we receive such unsolicited response.
1406                            info!(
1407                                context,
1408                                "Skipping not requested FETCH response for UID {}.", next_uid
1409                            );
1410                        } else if uid_msgs.insert(next_uid, next_fetch_response).is_some() {
1411                            warn!(context, "Got duplicated UID {}.", next_uid);
1412                        }
1413                    } else {
1414                        info!(context, "Skipping FETCH response without UID.");
1415                    }
1416                }
1417
1418                let fetch_response = match fetch_response {
1419                    Some(fetch) => fetch,
1420                    None => {
1421                        warn!(
1422                            context,
1423                            "Missed UID {} in the server response.", request_uid
1424                        );
1425                        continue;
1426                    }
1427                };
1428                count += 1;
1429
1430                let is_deleted = fetch_response.flags().any(|flag| flag == Flag::Deleted);
1431                let (body, partial) = if fetch_partially {
1432                    (fetch_response.header(), fetch_response.size) // `BODY.PEEK[HEADER]` goes to header() ...
1433                } else {
1434                    (fetch_response.body(), None) // ... while `BODY.PEEK[]` goes to body() - and includes header()
1435                };
1436
1437                if is_deleted {
1438                    info!(context, "Not processing deleted msg {}.", request_uid);
1439                    received_msgs_channel.send((request_uid, None)).await?;
1440                    continue;
1441                }
1442
1443                let body = if let Some(body) = body {
1444                    body
1445                } else {
1446                    info!(
1447                        context,
1448                        "Not processing message {} without a BODY.", request_uid
1449                    );
1450                    received_msgs_channel.send((request_uid, None)).await?;
1451                    continue;
1452                };
1453
1454                let is_seen = fetch_response.flags().any(|flag| flag == Flag::Seen);
1455
1456                let Some(rfc724_mid) = uid_message_ids.get(&request_uid) else {
1457                    error!(
1458                        context,
1459                        "No Message-ID corresponding to UID {} passed in uid_messsage_ids.",
1460                        request_uid
1461                    );
1462                    continue;
1463                };
1464
1465                info!(
1466                    context,
1467                    "Passing message UID {} to receive_imf().", request_uid
1468                );
1469                match receive_imf_inner(
1470                    context,
1471                    folder,
1472                    uidvalidity,
1473                    request_uid,
1474                    rfc724_mid,
1475                    body,
1476                    is_seen,
1477                    partial,
1478                )
1479                .await
1480                {
1481                    Ok(received_msg) => {
1482                        received_msgs_channel
1483                            .send((request_uid, received_msg))
1484                            .await?;
1485                    }
1486                    Err(err) => {
1487                        warn!(context, "receive_imf error: {:#}.", err);
1488                        received_msgs_channel.send((request_uid, None)).await?;
1489                    }
1490                };
1491            }
1492
1493            // If we don't process the whole response, IMAP client is left in a broken state where
1494            // it will try to process the rest of response as the next response.
1495            //
1496            // Make sure to not ignore the errors, because
1497            // if connection times out, it will return
1498            // infinite stream of `Some(Err(_))` results.
1499            while fetch_responses
1500                .try_next()
1501                .await
1502                .context("Failed to drain FETCH responses")?
1503                .is_some()
1504            {}
1505
1506            if count != request_uids.len() {
1507                warn!(
1508                    context,
1509                    "Failed to fetch all UIDs: got {}, requested {}, we requested the UIDs {:?}.",
1510                    count,
1511                    request_uids.len(),
1512                    request_uids,
1513                );
1514            } else {
1515                info!(
1516                    context,
1517                    "Successfully received {} UIDs.",
1518                    request_uids.len()
1519                );
1520            }
1521        }
1522
1523        Ok(())
1524    }
1525
1526    /// Retrieves server metadata if it is supported.
1527    ///
1528    /// We get [`/shared/comment`](https://www.rfc-editor.org/rfc/rfc5464#section-6.2.1)
1529    /// and [`/shared/admin`](https://www.rfc-editor.org/rfc/rfc5464#section-6.2.2)
1530    /// metadata.
1531    pub(crate) async fn fetch_metadata(&mut self, context: &Context) -> Result<()> {
1532        if !self.can_metadata() {
1533            return Ok(());
1534        }
1535
1536        let mut lock = context.metadata.write().await;
1537        if (*lock).is_some() {
1538            return Ok(());
1539        }
1540
1541        info!(
1542            context,
1543            "Server supports metadata, retrieving server comment and admin contact."
1544        );
1545
1546        let mut comment = None;
1547        let mut admin = None;
1548        let mut iroh_relay = None;
1549
1550        let mailbox = "";
1551        let options = "";
1552        let metadata = self
1553            .get_metadata(
1554                mailbox,
1555                options,
1556                "(/shared/comment /shared/admin /shared/vendor/deltachat/irohrelay)",
1557            )
1558            .await?;
1559        for m in metadata {
1560            match m.entry.as_ref() {
1561                "/shared/comment" => {
1562                    comment = m.value;
1563                }
1564                "/shared/admin" => {
1565                    admin = m.value;
1566                }
1567                "/shared/vendor/deltachat/irohrelay" => {
1568                    if let Some(value) = m.value {
1569                        if let Ok(url) = Url::parse(&value) {
1570                            iroh_relay = Some(url);
1571                        } else {
1572                            warn!(
1573                                context,
1574                                "Got invalid URL from iroh relay metadata: {:?}.", value
1575                            );
1576                        }
1577                    }
1578                }
1579                _ => {}
1580            }
1581        }
1582        *lock = Some(ServerMetadata {
1583            comment,
1584            admin,
1585            iroh_relay,
1586        });
1587        Ok(())
1588    }
1589
1590    /// Stores device token into /private/devicetoken IMAP METADATA of the Inbox.
1591    pub(crate) async fn register_token(&mut self, context: &Context) -> Result<()> {
1592        if context.push_subscribed.load(Ordering::Relaxed) {
1593            return Ok(());
1594        }
1595
1596        let Some(device_token) = context.push_subscriber.device_token().await else {
1597            return Ok(());
1598        };
1599
1600        if self.can_metadata() && self.can_push() {
1601            let old_encrypted_device_token =
1602                context.get_config(Config::EncryptedDeviceToken).await?;
1603
1604            // Whether we need to update encrypted device token.
1605            let device_token_changed = old_encrypted_device_token.is_none()
1606                || context.get_config(Config::DeviceToken).await?.as_ref() != Some(&device_token);
1607
1608            let new_encrypted_device_token;
1609            if device_token_changed {
1610                let encrypted_device_token = encrypt_device_token(&device_token)
1611                    .context("Failed to encrypt device token")?;
1612
1613                // We expect that the server supporting `XDELTAPUSH` capability
1614                // has non-synchronizing literals support as well:
1615                // <https://www.rfc-editor.org/rfc/rfc7888>.
1616                let encrypted_device_token_len = encrypted_device_token.len();
1617
1618                // Store device token saved on the server
1619                // to prevent storing duplicate tokens.
1620                // The server cannot deduplicate on its own
1621                // because encryption gives a different
1622                // result each time.
1623                context
1624                    .set_config_internal(Config::DeviceToken, Some(&device_token))
1625                    .await?;
1626                context
1627                    .set_config_internal(
1628                        Config::EncryptedDeviceToken,
1629                        Some(&encrypted_device_token),
1630                    )
1631                    .await?;
1632
1633                if encrypted_device_token_len <= 4096 {
1634                    new_encrypted_device_token = Some(encrypted_device_token);
1635                } else {
1636                    // If Apple or Google (FCM) gives us a very large token,
1637                    // do not even try to give it to IMAP servers.
1638                    //
1639                    // Limit of 4096 is arbitrarily selected
1640                    // to be the same as required by LITERAL- IMAP extension.
1641                    //
1642                    // Dovecot supports LITERAL+ and non-synchronizing literals
1643                    // of any length, but there is no reason for tokens
1644                    // to be that large even after OpenPGP encryption.
1645                    warn!(context, "Device token is too long for LITERAL-, ignoring.");
1646                    new_encrypted_device_token = None;
1647                }
1648            } else {
1649                new_encrypted_device_token = old_encrypted_device_token;
1650            }
1651
1652            // Store new encrypted device token on the server
1653            // even if it is the same as the old one.
1654            if let Some(encrypted_device_token) = new_encrypted_device_token {
1655                let folder = context
1656                    .get_config(Config::ConfiguredInboxFolder)
1657                    .await?
1658                    .context("INBOX is not configured")?;
1659
1660                self.run_command_and_check_ok(&format_setmetadata(
1661                    &folder,
1662                    &encrypted_device_token,
1663                ))
1664                .await
1665                .context("SETMETADATA command failed")?;
1666
1667                context.push_subscribed.store(true, Ordering::Relaxed);
1668            }
1669        } else if !context.push_subscriber.heartbeat_subscribed().await {
1670            let context = context.clone();
1671            // Subscribe for heartbeat notifications.
1672            tokio::spawn(async move { context.push_subscriber.subscribe(&context).await });
1673        }
1674
1675        Ok(())
1676    }
1677}
1678
1679fn format_setmetadata(folder: &str, device_token: &str) -> String {
1680    let device_token_len = device_token.len();
1681    format!(
1682        "SETMETADATA \"{folder}\" (/private/devicetoken {{{device_token_len}+}}\r\n{device_token})"
1683    )
1684}
1685
1686impl Session {
1687    /// Returns success if we successfully set the flag or we otherwise
1688    /// think add_flag should not be retried: Disconnection during setting
1689    /// the flag, or other imap-errors, returns Ok as well.
1690    ///
1691    /// Returning error means that the operation can be retried.
1692    async fn add_flag_finalized_with_set(&mut self, uid_set: &str, flag: &str) -> Result<()> {
1693        if flag == "\\Deleted" {
1694            self.selected_folder_needs_expunge = true;
1695        }
1696        let query = format!("+FLAGS ({flag})");
1697        let mut responses = self
1698            .uid_store(uid_set, &query)
1699            .await
1700            .with_context(|| format!("IMAP failed to store: ({uid_set}, {query})"))?;
1701        while let Some(_response) = responses.try_next().await? {
1702            // Read all the responses
1703        }
1704        Ok(())
1705    }
1706
1707    /// Attempts to configure mvbox.
1708    ///
1709    /// Tries to find any folder examining `folders` in the order they go. If none is found, tries
1710    /// to create any folder in the same order. This method does not use LIST command to ensure that
1711    /// configuration works even if mailbox lookup is forbidden via Access Control List (see
1712    /// <https://datatracker.ietf.org/doc/html/rfc4314>).
1713    ///
1714    /// Returns first found or created folder name.
1715    async fn configure_mvbox<'a>(
1716        &mut self,
1717        context: &Context,
1718        folders: &[&'a str],
1719        create_mvbox: bool,
1720    ) -> Result<Option<&'a str>> {
1721        // Close currently selected folder if needed.
1722        // We are going to select folders using low-level EXAMINE operations below.
1723        self.maybe_close_folder(context).await?;
1724
1725        for folder in folders {
1726            info!(context, "Looking for MVBOX-folder \"{}\"...", &folder);
1727            let res = self.examine(&folder).await;
1728            if res.is_ok() {
1729                info!(
1730                    context,
1731                    "MVBOX-folder {:?} successfully selected, using it.", &folder
1732                );
1733                self.close().await?;
1734                // Before moving emails to the mvbox we need to remember its UIDVALIDITY, otherwise
1735                // emails moved before that wouldn't be fetched but considered "old" instead.
1736                let create = false;
1737                let folder_exists = self
1738                    .select_with_uidvalidity(context, folder, create)
1739                    .await?;
1740                ensure!(folder_exists, "No MVBOX folder {:?}??", &folder);
1741                return Ok(Some(folder));
1742            }
1743        }
1744
1745        if !create_mvbox {
1746            return Ok(None);
1747        }
1748        // Some servers require namespace-style folder names like "INBOX.DeltaChat", so we try all
1749        // the variants here.
1750        for folder in folders {
1751            match self
1752                .select_with_uidvalidity(context, folder, create_mvbox)
1753                .await
1754            {
1755                Ok(_) => {
1756                    info!(context, "MVBOX-folder {} created.", folder);
1757                    return Ok(Some(folder));
1758                }
1759                Err(err) => {
1760                    warn!(context, "Cannot create MVBOX-folder {:?}: {}", folder, err);
1761                }
1762            }
1763        }
1764        Ok(None)
1765    }
1766}
1767
1768impl Imap {
1769    pub(crate) async fn configure_folders(
1770        &mut self,
1771        context: &Context,
1772        session: &mut Session,
1773        create_mvbox: bool,
1774    ) -> Result<()> {
1775        let mut folders = session
1776            .list(Some(""), Some("*"))
1777            .await
1778            .context("list_folders failed")?;
1779        let mut delimiter = ".".to_string();
1780        let mut delimiter_is_default = true;
1781        let mut folder_configs = BTreeMap::new();
1782
1783        while let Some(folder) = folders.try_next().await? {
1784            info!(context, "Scanning folder: {:?}", folder);
1785
1786            // Update the delimiter iff there is a different one, but only once.
1787            if let Some(d) = folder.delimiter() {
1788                if delimiter_is_default && !d.is_empty() && delimiter != d {
1789                    delimiter = d.to_string();
1790                    delimiter_is_default = false;
1791                }
1792            }
1793
1794            let folder_meaning = get_folder_meaning_by_attrs(folder.attributes());
1795            let folder_name_meaning = get_folder_meaning_by_name(folder.name());
1796            if let Some(config) = folder_meaning.to_config() {
1797                // Always takes precedence
1798                folder_configs.insert(config, folder.name().to_string());
1799            } else if let Some(config) = folder_name_meaning.to_config() {
1800                // only set if none has been already set
1801                folder_configs
1802                    .entry(config)
1803                    .or_insert_with(|| folder.name().to_string());
1804            }
1805        }
1806        drop(folders);
1807
1808        info!(context, "Using \"{}\" as folder-delimiter.", delimiter);
1809
1810        let fallback_folder = format!("INBOX{delimiter}DeltaChat");
1811        let mvbox_folder = session
1812            .configure_mvbox(context, &["DeltaChat", &fallback_folder], create_mvbox)
1813            .await
1814            .context("failed to configure mvbox")?;
1815
1816        context
1817            .set_config_internal(Config::ConfiguredInboxFolder, Some("INBOX"))
1818            .await?;
1819        if let Some(mvbox_folder) = mvbox_folder {
1820            info!(context, "Setting MVBOX FOLDER TO {}", &mvbox_folder);
1821            context
1822                .set_config_internal(Config::ConfiguredMvboxFolder, Some(mvbox_folder))
1823                .await?;
1824        }
1825        for (config, name) in folder_configs {
1826            context.set_config_internal(config, Some(&name)).await?;
1827        }
1828        context
1829            .sql
1830            .set_raw_config_int(
1831                constants::DC_FOLDERS_CONFIGURED_KEY,
1832                constants::DC_FOLDERS_CONFIGURED_VERSION,
1833            )
1834            .await?;
1835
1836        info!(context, "FINISHED configuring IMAP-folders.");
1837        Ok(())
1838    }
1839}
1840
1841impl Session {
1842    /// Return whether the server sent an unsolicited EXISTS or FETCH response.
1843    ///
1844    /// Drains all responses from `session.unsolicited_responses` in the process.
1845    ///
1846    /// If this returns `true`, this means that new emails arrived
1847    /// or flags have been changed.
1848    /// In this case we may want to skip next IDLE and do a round
1849    /// of fetching new messages and synchronizing seen flags.
1850    fn drain_unsolicited_responses(&self, context: &Context) -> Result<bool> {
1851        use UnsolicitedResponse::*;
1852        use async_imap::imap_proto::Response;
1853        use async_imap::imap_proto::ResponseCode;
1854
1855        let folder = self.selected_folder.as_deref().unwrap_or_default();
1856        let mut should_refetch = false;
1857        while let Ok(response) = self.unsolicited_responses.try_recv() {
1858            match response {
1859                Exists(_) => {
1860                    info!(
1861                        context,
1862                        "Need to refetch {folder:?}, got unsolicited EXISTS {response:?}"
1863                    );
1864                    should_refetch = true;
1865                }
1866
1867                Expunge(_) | Recent(_) => {}
1868                Other(ref response_data) => {
1869                    match response_data.parsed() {
1870                        Response::Fetch { .. } => {
1871                            info!(
1872                                context,
1873                                "Need to refetch {folder:?}, got unsolicited FETCH {response:?}"
1874                            );
1875                            should_refetch = true;
1876                        }
1877
1878                        // We are not interested in the following responses and they are are
1879                        // sent quite frequently, so, we ignore them without logging them.
1880                        Response::Done {
1881                            code: Some(ResponseCode::CopyUid(_, _, _)),
1882                            ..
1883                        } => {}
1884
1885                        _ => {
1886                            info!(context, "{folder:?}: got unsolicited response {response:?}")
1887                        }
1888                    }
1889                }
1890                _ => {
1891                    info!(context, "{folder:?}: got unsolicited response {response:?}")
1892                }
1893            }
1894        }
1895        Ok(should_refetch)
1896    }
1897}
1898
1899async fn should_move_out_of_spam(
1900    context: &Context,
1901    headers: &[mailparse::MailHeader<'_>],
1902) -> Result<bool> {
1903    if headers.get_header_value(HeaderDef::ChatVersion).is_some() {
1904        // If this is a chat message (i.e. has a ChatVersion header), then this might be
1905        // a securejoin message. We can't find out at this point as we didn't prefetch
1906        // the SecureJoin header. So, we always move chat messages out of Spam.
1907        // Two possibilities to change this would be:
1908        // 1. Remove the `&& !context.is_spam_folder(folder).await?` check from
1909        // `fetch_new_messages()`, and then let `receive_imf()` check
1910        // if it's a spam message and should be hidden.
1911        // 2. Or add a flag to the ChatVersion header that this is a securejoin
1912        // request, and return `true` here only if the message has this flag.
1913        // `receive_imf()` can then check if the securejoin request is valid.
1914        return Ok(true);
1915    }
1916
1917    if let Some(msg) = get_prefetch_parent_message(context, headers).await? {
1918        if msg.chat_blocked != Blocked::Not {
1919            // Blocked or contact request message in the spam folder, leave it there.
1920            return Ok(false);
1921        }
1922    } else {
1923        let from = match mimeparser::get_from(headers) {
1924            Some(f) => f,
1925            None => return Ok(false),
1926        };
1927        // No chat found.
1928        let (from_id, blocked_contact, _origin) =
1929            match from_field_to_contact_id(context, &from, None, true, true)
1930                .await
1931                .context("from_field_to_contact_id")?
1932            {
1933                Some(res) => res,
1934                None => {
1935                    warn!(
1936                        context,
1937                        "Contact with From address {:?} cannot exist, not moving out of spam", from
1938                    );
1939                    return Ok(false);
1940                }
1941            };
1942        if blocked_contact {
1943            // Contact is blocked, leave the message in spam.
1944            return Ok(false);
1945        }
1946
1947        if let Some(chat_id_blocked) = ChatIdBlocked::lookup_by_contact(context, from_id).await? {
1948            if chat_id_blocked.blocked != Blocked::Not {
1949                return Ok(false);
1950            }
1951        } else if from_id != ContactId::SELF {
1952            // No chat with this contact found.
1953            return Ok(false);
1954        }
1955    }
1956
1957    Ok(true)
1958}
1959
1960/// Returns target folder for a message found in the Spam folder.
1961/// If this returns None, the message will not be moved out of the
1962/// Spam folder, and as `fetch_new_messages()` doesn't download
1963/// messages from the Spam folder, the message will be ignored.
1964async fn spam_target_folder_cfg(
1965    context: &Context,
1966    headers: &[mailparse::MailHeader<'_>],
1967) -> Result<Option<Config>> {
1968    if !should_move_out_of_spam(context, headers).await? {
1969        return Ok(None);
1970    }
1971
1972    if needs_move_to_mvbox(context, headers).await?
1973        // If OnlyFetchMvbox is set, we don't want to move the message to
1974        // the inbox or sentbox where we wouldn't fetch it again:
1975        || context.get_config_bool(Config::OnlyFetchMvbox).await?
1976    {
1977        Ok(Some(Config::ConfiguredMvboxFolder))
1978    } else {
1979        Ok(Some(Config::ConfiguredInboxFolder))
1980    }
1981}
1982
1983/// Returns `ConfiguredInboxFolder`, `ConfiguredMvboxFolder` or `ConfiguredSentboxFolder` if
1984/// the message needs to be moved from `folder`. Otherwise returns `None`.
1985pub async fn target_folder_cfg(
1986    context: &Context,
1987    folder: &str,
1988    folder_meaning: FolderMeaning,
1989    headers: &[mailparse::MailHeader<'_>],
1990) -> Result<Option<Config>> {
1991    if context.is_mvbox(folder).await? {
1992        return Ok(None);
1993    }
1994
1995    if folder_meaning == FolderMeaning::Spam {
1996        spam_target_folder_cfg(context, headers).await
1997    } else if needs_move_to_mvbox(context, headers).await? {
1998        Ok(Some(Config::ConfiguredMvboxFolder))
1999    } else {
2000        Ok(None)
2001    }
2002}
2003
2004pub async fn target_folder(
2005    context: &Context,
2006    folder: &str,
2007    folder_meaning: FolderMeaning,
2008    headers: &[mailparse::MailHeader<'_>],
2009) -> Result<String> {
2010    match target_folder_cfg(context, folder, folder_meaning, headers).await? {
2011        Some(config) => match context.get_config(config).await? {
2012            Some(target) => Ok(target),
2013            None => Ok(folder.to_string()),
2014        },
2015        None => Ok(folder.to_string()),
2016    }
2017}
2018
2019async fn needs_move_to_mvbox(
2020    context: &Context,
2021    headers: &[mailparse::MailHeader<'_>],
2022) -> Result<bool> {
2023    let has_chat_version = headers.get_header_value(HeaderDef::ChatVersion).is_some();
2024    if !context.get_config_bool(Config::IsChatmail).await?
2025        && has_chat_version
2026        && headers
2027            .get_header_value(HeaderDef::AutoSubmitted)
2028            .filter(|val| val.eq_ignore_ascii_case("auto-generated"))
2029            .is_some()
2030    {
2031        if let Some(from) = mimeparser::get_from(headers) {
2032            if context.is_self_addr(&from.addr).await? {
2033                return Ok(true);
2034            }
2035        }
2036    }
2037    if !context.get_config_bool(Config::MvboxMove).await? {
2038        return Ok(false);
2039    }
2040
2041    if headers
2042        .get_header_value(HeaderDef::AutocryptSetupMessage)
2043        .is_some()
2044    {
2045        // do not move setup messages;
2046        // there may be a non-delta device that wants to handle it
2047        return Ok(false);
2048    }
2049
2050    if has_chat_version {
2051        Ok(true)
2052    } else if let Some(parent) = get_prefetch_parent_message(context, headers).await? {
2053        match parent.is_dc_message {
2054            MessengerMessage::No => Ok(false),
2055            MessengerMessage::Yes | MessengerMessage::Reply => Ok(true),
2056        }
2057    } else {
2058        Ok(false)
2059    }
2060}
2061
2062/// Try to get the folder meaning by the name of the folder only used if the server does not support XLIST.
2063// TODO: lots languages missing - maybe there is a list somewhere on other MUAs?
2064// however, if we fail to find out the sent-folder,
2065// only watching this folder is not working. at least, this is no show stopper.
2066// CAVE: if possible, take care not to add a name here that is "sent" in one language
2067// but sth. different in others - a hard job.
2068fn get_folder_meaning_by_name(folder_name: &str) -> FolderMeaning {
2069    // source: <https://stackoverflow.com/questions/2185391/localized-gmail-imap-folders>
2070    const SENT_NAMES: &[&str] = &[
2071        "sent",
2072        "sentmail",
2073        "sent objects",
2074        "gesendet",
2075        "Sent Mail",
2076        "Sendte e-mails",
2077        "Enviados",
2078        "Messages envoyés",
2079        "Messages envoyes",
2080        "Posta inviata",
2081        "Verzonden berichten",
2082        "Wyslane",
2083        "E-mails enviados",
2084        "Correio enviado",
2085        "Enviada",
2086        "Enviado",
2087        "Gönderildi",
2088        "Inviati",
2089        "Odeslaná pošta",
2090        "Sendt",
2091        "Skickat",
2092        "Verzonden",
2093        "Wysłane",
2094        "Éléments envoyés",
2095        "Απεσταλμένα",
2096        "Отправленные",
2097        "寄件備份",
2098        "已发送邮件",
2099        "送信済み",
2100        "보낸편지함",
2101    ];
2102    const SPAM_NAMES: &[&str] = &[
2103        "spam",
2104        "junk",
2105        "Correio electrónico não solicitado",
2106        "Correo basura",
2107        "Lixo",
2108        "Nettsøppel",
2109        "Nevyžádaná pošta",
2110        "No solicitado",
2111        "Ongewenst",
2112        "Posta indesiderata",
2113        "Skräp",
2114        "Wiadomości-śmieci",
2115        "Önemsiz",
2116        "Ανεπιθύμητα",
2117        "Спам",
2118        "垃圾邮件",
2119        "垃圾郵件",
2120        "迷惑メール",
2121        "스팸",
2122    ];
2123    const DRAFT_NAMES: &[&str] = &[
2124        "Drafts",
2125        "Kladder",
2126        "Entw?rfe",
2127        "Borradores",
2128        "Brouillons",
2129        "Bozze",
2130        "Concepten",
2131        "Wersje robocze",
2132        "Rascunhos",
2133        "Entwürfe",
2134        "Koncepty",
2135        "Kopie robocze",
2136        "Taslaklar",
2137        "Utkast",
2138        "Πρόχειρα",
2139        "Черновики",
2140        "下書き",
2141        "草稿",
2142        "임시보관함",
2143    ];
2144    const TRASH_NAMES: &[&str] = &[
2145        "Trash",
2146        "Bin",
2147        "Caixote do lixo",
2148        "Cestino",
2149        "Corbeille",
2150        "Papelera",
2151        "Papierkorb",
2152        "Papirkurv",
2153        "Papperskorgen",
2154        "Prullenbak",
2155        "Rubujo",
2156        "Κάδος απορριμμάτων",
2157        "Корзина",
2158        "Кошик",
2159        "ゴミ箱",
2160        "垃圾桶",
2161        "已删除邮件",
2162        "휴지통",
2163    ];
2164    let lower = folder_name.to_lowercase();
2165
2166    if SENT_NAMES.iter().any(|s| s.to_lowercase() == lower) {
2167        FolderMeaning::Sent
2168    } else if SPAM_NAMES.iter().any(|s| s.to_lowercase() == lower) {
2169        FolderMeaning::Spam
2170    } else if DRAFT_NAMES.iter().any(|s| s.to_lowercase() == lower) {
2171        FolderMeaning::Drafts
2172    } else if TRASH_NAMES.iter().any(|s| s.to_lowercase() == lower) {
2173        FolderMeaning::Trash
2174    } else {
2175        FolderMeaning::Unknown
2176    }
2177}
2178
2179fn get_folder_meaning_by_attrs(folder_attrs: &[NameAttribute]) -> FolderMeaning {
2180    for attr in folder_attrs {
2181        match attr {
2182            NameAttribute::Trash => return FolderMeaning::Trash,
2183            NameAttribute::Sent => return FolderMeaning::Sent,
2184            NameAttribute::Junk => return FolderMeaning::Spam,
2185            NameAttribute::Drafts => return FolderMeaning::Drafts,
2186            NameAttribute::All | NameAttribute::Flagged => return FolderMeaning::Virtual,
2187            NameAttribute::Extension(label) => {
2188                match label.as_ref() {
2189                    "\\Spam" => return FolderMeaning::Spam,
2190                    "\\Important" => return FolderMeaning::Virtual,
2191                    _ => {}
2192                };
2193            }
2194            _ => {}
2195        }
2196    }
2197    FolderMeaning::Unknown
2198}
2199
2200pub(crate) fn get_folder_meaning(folder: &Name) -> FolderMeaning {
2201    match get_folder_meaning_by_attrs(folder.attributes()) {
2202        FolderMeaning::Unknown => get_folder_meaning_by_name(folder.name()),
2203        meaning => meaning,
2204    }
2205}
2206
2207/// Parses the headers from the FETCH result.
2208fn get_fetch_headers(prefetch_msg: &Fetch) -> Result<Vec<mailparse::MailHeader<'_>>> {
2209    match prefetch_msg.header() {
2210        Some(header_bytes) => {
2211            let (headers, _) = mailparse::parse_headers(header_bytes)?;
2212            Ok(headers)
2213        }
2214        None => Ok(Vec::new()),
2215    }
2216}
2217
2218pub(crate) fn prefetch_get_message_id(headers: &[mailparse::MailHeader]) -> Option<String> {
2219    headers
2220        .get_header_value(HeaderDef::XMicrosoftOriginalMessageId)
2221        .or_else(|| headers.get_header_value(HeaderDef::MessageId))
2222        .and_then(|msgid| mimeparser::parse_message_id(&msgid).ok())
2223}
2224
2225pub(crate) fn create_message_id() -> String {
2226    format!("{}{}", GENERATED_PREFIX, create_id())
2227}
2228
2229/// Returns chat by prefetched headers.
2230async fn prefetch_get_chat(
2231    context: &Context,
2232    headers: &[mailparse::MailHeader<'_>],
2233) -> Result<Option<chat::Chat>> {
2234    let parent = get_prefetch_parent_message(context, headers).await?;
2235    if let Some(parent) = &parent {
2236        return Ok(Some(
2237            chat::Chat::load_from_db(context, parent.get_chat_id()).await?,
2238        ));
2239    }
2240
2241    Ok(None)
2242}
2243
2244/// Determines whether the message should be downloaded based on prefetched headers.
2245pub(crate) async fn prefetch_should_download(
2246    context: &Context,
2247    headers: &[mailparse::MailHeader<'_>],
2248    message_id: &str,
2249    mut flags: impl Iterator<Item = Flag<'_>>,
2250) -> Result<bool> {
2251    if message::rfc724_mid_exists(context, message_id)
2252        .await?
2253        .is_some()
2254    {
2255        markseen_on_imap_table(context, message_id).await?;
2256        return Ok(false);
2257    }
2258
2259    // We do not know the Message-ID or the Message-ID is missing (in this case, we create one in
2260    // the further process).
2261
2262    if let Some(chat) = prefetch_get_chat(context, headers).await? {
2263        if chat.typ == Chattype::Group && !chat.id.is_special() {
2264            // This might be a group command, like removing a group member.
2265            // We really need to fetch this to avoid inconsistent group state.
2266            return Ok(true);
2267        }
2268    }
2269
2270    let maybe_ndn = if let Some(from) = headers.get_header_value(HeaderDef::From_) {
2271        let from = from.to_ascii_lowercase();
2272        from.contains("mailer-daemon") || from.contains("mail-daemon")
2273    } else {
2274        false
2275    };
2276
2277    // Autocrypt Setup Message should be shown even if it is from non-chat client.
2278    let is_autocrypt_setup_message = headers
2279        .get_header_value(HeaderDef::AutocryptSetupMessage)
2280        .is_some();
2281
2282    let from = match mimeparser::get_from(headers) {
2283        Some(f) => f,
2284        None => return Ok(false),
2285    };
2286    let (_from_id, blocked_contact, origin) =
2287        match from_field_to_contact_id(context, &from, None, true, true).await? {
2288            Some(res) => res,
2289            None => return Ok(false),
2290        };
2291    // prevent_rename=true as this might be a mailing list message and in this case it would be bad if we rename the contact.
2292    // (prevent_rename is the last argument of from_field_to_contact_id())
2293
2294    if flags.any(|f| f == Flag::Draft) {
2295        info!(context, "Ignoring draft message");
2296        return Ok(false);
2297    }
2298
2299    let is_chat_message = headers.get_header_value(HeaderDef::ChatVersion).is_some();
2300    let accepted_contact = origin.is_known();
2301    let is_reply_to_chat_message = get_prefetch_parent_message(context, headers)
2302        .await?
2303        .map(|parent| match parent.is_dc_message {
2304            MessengerMessage::No => false,
2305            MessengerMessage::Yes | MessengerMessage::Reply => true,
2306        })
2307        .unwrap_or_default();
2308
2309    let show_emails =
2310        ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await?).unwrap_or_default();
2311
2312    let show = is_autocrypt_setup_message
2313        || match show_emails {
2314            ShowEmails::Off => is_chat_message || is_reply_to_chat_message,
2315            ShowEmails::AcceptedContacts => {
2316                is_chat_message || is_reply_to_chat_message || accepted_contact
2317            }
2318            ShowEmails::All => true,
2319        };
2320
2321    let should_download = (show && !blocked_contact) || maybe_ndn;
2322    Ok(should_download)
2323}
2324
2325/// Returns whether a message is a duplicate (resent message).
2326pub(crate) fn is_dup_msg(is_chat_msg: bool, ts_sent: i64, ts_sent_old: i64) -> bool {
2327    // If the existing message has timestamp_sent == 0, that means we don't know its actual sent
2328    // timestamp, so don't delete the new message. E.g. outgoing messages have zero timestamp_sent
2329    // because they are stored to the db before sending. Also consider as duplicates only messages
2330    // with greater timestamp to avoid deleting both messages in a multi-device setting.
2331    is_chat_msg && ts_sent_old != 0 && ts_sent > ts_sent_old
2332}
2333
2334/// Marks messages in `msgs` table as seen, searching for them by UID.
2335///
2336/// Returns updated chat ID if any message was marked as seen.
2337async fn mark_seen_by_uid(
2338    context: &Context,
2339    folder: &str,
2340    uid_validity: u32,
2341    uid: u32,
2342) -> Result<Option<ChatId>> {
2343    if let Some((msg_id, chat_id)) = context
2344        .sql
2345        .query_row_optional(
2346            "SELECT id, chat_id FROM msgs
2347                 WHERE id > 9 AND rfc724_mid IN (
2348                   SELECT rfc724_mid FROM imap
2349                   WHERE folder=?1
2350                   AND uidvalidity=?2
2351                   AND uid=?3
2352                   LIMIT 1
2353                 )",
2354            (&folder, uid_validity, uid),
2355            |row| {
2356                let msg_id: MsgId = row.get(0)?;
2357                let chat_id: ChatId = row.get(1)?;
2358                Ok((msg_id, chat_id))
2359            },
2360        )
2361        .await
2362        .with_context(|| format!("failed to get msg and chat ID for IMAP message {folder}/{uid}"))?
2363    {
2364        let updated = context
2365            .sql
2366            .execute(
2367                "UPDATE msgs SET state=?1
2368                     WHERE (state=?2 OR state=?3)
2369                     AND id=?4",
2370                (
2371                    MessageState::InSeen,
2372                    MessageState::InFresh,
2373                    MessageState::InNoticed,
2374                    msg_id,
2375                ),
2376            )
2377            .await
2378            .with_context(|| format!("failed to update msg {msg_id} state"))?
2379            > 0;
2380
2381        if updated {
2382            msg_id
2383                .start_ephemeral_timer(context)
2384                .await
2385                .with_context(|| format!("failed to start ephemeral timer for message {msg_id}"))?;
2386            Ok(Some(chat_id))
2387        } else {
2388            // Message state has not changed.
2389            Ok(None)
2390        }
2391    } else {
2392        // There is no message is `msgs` table matching the given UID.
2393        Ok(None)
2394    }
2395}
2396
2397/// Schedule marking the message as Seen on IMAP by adding all known IMAP messages corresponding to
2398/// the given Message-ID to `imap_markseen` table.
2399pub(crate) async fn markseen_on_imap_table(context: &Context, message_id: &str) -> Result<()> {
2400    context
2401        .sql
2402        .execute(
2403            "INSERT OR IGNORE INTO imap_markseen (id)
2404             SELECT id FROM imap WHERE rfc724_mid=?",
2405            (message_id,),
2406        )
2407        .await?;
2408    context.scheduler.interrupt_inbox().await;
2409
2410    Ok(())
2411}
2412
2413/// uid_next is the next unique identifier value from the last time we fetched a folder
2414/// See <https://tools.ietf.org/html/rfc3501#section-2.3.1.1>
2415/// This function is used to update our uid_next after fetching messages.
2416pub(crate) async fn set_uid_next(context: &Context, folder: &str, uid_next: u32) -> Result<()> {
2417    context
2418        .sql
2419        .execute(
2420            "INSERT INTO imap_sync (folder, uid_next) VALUES (?,?)
2421                ON CONFLICT(folder) DO UPDATE SET uid_next=excluded.uid_next",
2422            (folder, uid_next),
2423        )
2424        .await?;
2425    Ok(())
2426}
2427
2428/// uid_next is the next unique identifier value from the last time we fetched a folder
2429/// See <https://tools.ietf.org/html/rfc3501#section-2.3.1.1>
2430/// This method returns the uid_next from the last time we fetched messages.
2431/// We can compare this to the current uid_next to find out whether there are new messages
2432/// and fetch from this value on to get all new messages.
2433async fn get_uid_next(context: &Context, folder: &str) -> Result<u32> {
2434    Ok(context
2435        .sql
2436        .query_get_value("SELECT uid_next FROM imap_sync WHERE folder=?;", (folder,))
2437        .await?
2438        .unwrap_or(0))
2439}
2440
2441pub(crate) async fn set_uidvalidity(
2442    context: &Context,
2443    folder: &str,
2444    uidvalidity: u32,
2445) -> Result<()> {
2446    context
2447        .sql
2448        .execute(
2449            "INSERT INTO imap_sync (folder, uidvalidity) VALUES (?,?)
2450                ON CONFLICT(folder) DO UPDATE SET uidvalidity=excluded.uidvalidity",
2451            (folder, uidvalidity),
2452        )
2453        .await?;
2454    Ok(())
2455}
2456
2457async fn get_uidvalidity(context: &Context, folder: &str) -> Result<u32> {
2458    Ok(context
2459        .sql
2460        .query_get_value(
2461            "SELECT uidvalidity FROM imap_sync WHERE folder=?;",
2462            (folder,),
2463        )
2464        .await?
2465        .unwrap_or(0))
2466}
2467
2468pub(crate) async fn set_modseq(context: &Context, folder: &str, modseq: u64) -> Result<()> {
2469    context
2470        .sql
2471        .execute(
2472            "INSERT INTO imap_sync (folder, modseq) VALUES (?,?)
2473                ON CONFLICT(folder) DO UPDATE SET modseq=excluded.modseq",
2474            (folder, modseq),
2475        )
2476        .await?;
2477    Ok(())
2478}
2479
2480async fn get_modseq(context: &Context, folder: &str) -> Result<u64> {
2481    Ok(context
2482        .sql
2483        .query_get_value("SELECT modseq FROM imap_sync WHERE folder=?;", (folder,))
2484        .await?
2485        .unwrap_or(0))
2486}
2487
2488/// Compute the imap search expression for all self-sent mails (for all self addresses)
2489pub(crate) async fn get_imap_self_sent_search_command(context: &Context) -> Result<String> {
2490    // See https://www.rfc-editor.org/rfc/rfc3501#section-6.4.4 for syntax of SEARCH and OR
2491    let mut search_command = format!("FROM \"{}\"", context.get_primary_self_addr().await?);
2492
2493    for item in context.get_secondary_self_addrs().await? {
2494        search_command = format!("OR ({search_command}) (FROM \"{item}\")");
2495    }
2496
2497    Ok(search_command)
2498}
2499
2500/// Deprecated, use get_uid_next() and get_uidvalidity()
2501pub async fn get_config_last_seen_uid(context: &Context, folder: &str) -> Result<(u32, u32)> {
2502    let key = format!("imap.mailbox.{folder}");
2503    if let Some(entry) = context.sql.get_raw_config(&key).await? {
2504        // the entry has the format `imap.mailbox.<folder>=<uidvalidity>:<lastseenuid>`
2505        let mut parts = entry.split(':');
2506        Ok((
2507            parts.next().unwrap_or_default().parse().unwrap_or(0),
2508            parts.next().unwrap_or_default().parse().unwrap_or(0),
2509        ))
2510    } else {
2511        Ok((0, 0))
2512    }
2513}
2514
2515/// Whether to ignore fetching messages from a folder.
2516///
2517/// This caters for the [`Config::OnlyFetchMvbox`] setting which means mails from folders
2518/// not explicitly watched should not be fetched.
2519async fn should_ignore_folder(
2520    context: &Context,
2521    folder: &str,
2522    folder_meaning: FolderMeaning,
2523) -> Result<bool> {
2524    if !context.get_config_bool(Config::OnlyFetchMvbox).await? {
2525        return Ok(false);
2526    }
2527    if context.is_sentbox(folder).await? {
2528        // Still respect the SentboxWatch setting.
2529        return Ok(!context.get_config_bool(Config::SentboxWatch).await?);
2530    }
2531    Ok(!(context.is_mvbox(folder).await? || folder_meaning == FolderMeaning::Spam))
2532}
2533
2534/// Builds a list of sequence/uid sets. The returned sets have each no more than around 1000
2535/// characters because according to <https://tools.ietf.org/html/rfc2683#section-3.2.1.5>
2536/// command lines should not be much more than 1000 chars (servers should allow at least 8000 chars)
2537fn build_sequence_sets(uids: &[u32]) -> Result<Vec<(Vec<u32>, String)>> {
2538    // first, try to find consecutive ranges:
2539    let mut ranges: Vec<UidRange> = vec![];
2540
2541    for &current in uids {
2542        if let Some(last) = ranges.last_mut() {
2543            if last.end + 1 == current {
2544                last.end = current;
2545                continue;
2546            }
2547        }
2548
2549        ranges.push(UidRange {
2550            start: current,
2551            end: current,
2552        });
2553    }
2554
2555    // Second, sort the uids into uid sets that are each below ~1000 characters
2556    let mut result = vec![];
2557    let (mut last_uids, mut last_str) = (Vec::new(), String::new());
2558    for range in ranges {
2559        last_uids.reserve((range.end - range.start + 1).try_into()?);
2560        (range.start..=range.end).for_each(|u| last_uids.push(u));
2561        if !last_str.is_empty() {
2562            last_str.push(',');
2563        }
2564        last_str.push_str(&range.to_string());
2565
2566        if last_str.len() > 990 {
2567            result.push((take(&mut last_uids), take(&mut last_str)));
2568        }
2569    }
2570    result.push((last_uids, last_str));
2571
2572    result.retain(|(_, s)| !s.is_empty());
2573    Ok(result)
2574}
2575
2576struct UidRange {
2577    start: u32,
2578    end: u32,
2579    // If start == end, then this range represents a single number
2580}
2581
2582impl std::fmt::Display for UidRange {
2583    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
2584        if self.start == self.end {
2585            write!(f, "{}", self.start)
2586        } else {
2587            write!(f, "{}:{}", self.start, self.end)
2588        }
2589    }
2590}
2591async fn add_all_recipients_as_contacts(
2592    context: &Context,
2593    session: &mut Session,
2594    folder: Config,
2595) -> Result<()> {
2596    let mailbox = if let Some(m) = context.get_config(folder).await? {
2597        m
2598    } else {
2599        info!(
2600            context,
2601            "Folder {} is not configured, skipping fetching contacts from it.", folder
2602        );
2603        return Ok(());
2604    };
2605    let create = false;
2606    let folder_exists = session
2607        .select_with_uidvalidity(context, &mailbox, create)
2608        .await
2609        .with_context(|| format!("could not select {mailbox}"))?;
2610    if !folder_exists {
2611        return Ok(());
2612    }
2613
2614    let recipients = session
2615        .get_all_recipients(context)
2616        .await
2617        .context("could not get recipients")?;
2618
2619    let mut any_modified = false;
2620    for recipient in recipients {
2621        let recipient_addr = match ContactAddress::new(&recipient.addr) {
2622            Err(err) => {
2623                warn!(
2624                    context,
2625                    "Could not add contact for recipient with address {:?}: {:#}",
2626                    recipient.addr,
2627                    err
2628                );
2629                continue;
2630            }
2631            Ok(recipient_addr) => recipient_addr,
2632        };
2633
2634        let (_, modified) = Contact::add_or_lookup(
2635            context,
2636            &recipient.display_name.unwrap_or_default(),
2637            &recipient_addr,
2638            Origin::OutgoingTo,
2639        )
2640        .await?;
2641        if modified != Modifier::None {
2642            any_modified = true;
2643        }
2644    }
2645    if any_modified {
2646        context.emit_event(EventType::ContactsChanged(None));
2647    }
2648
2649    Ok(())
2650}
2651
2652#[cfg(test)]
2653mod imap_tests;