deltachat/
imap.rs

1//! # IMAP handling module.
2//!
3//! uses [async-email/async-imap](https://github.com/async-email/async-imap)
4//! to implement connect, fetch, delete functionality with standard IMAP servers.
5
6use std::{
7    cmp::max,
8    cmp::min,
9    collections::{BTreeMap, BTreeSet, HashMap},
10    iter::Peekable,
11    mem::take,
12    sync::atomic::Ordering,
13    time::{Duration, UNIX_EPOCH},
14};
15
16use anyhow::{Context as _, Result, bail, ensure, format_err};
17use async_channel::{self, Receiver, Sender};
18use async_imap::types::{Fetch, Flag, Name, NameAttribute, UnsolicitedResponse};
19use deltachat_contact_tools::ContactAddress;
20use futures::{FutureExt as _, TryStreamExt};
21use futures_lite::FutureExt;
22use num_traits::FromPrimitive;
23use rand::Rng;
24use ratelimit::Ratelimit;
25use url::Url;
26
27use crate::chat::{self, ChatId, ChatIdBlocked};
28use crate::chatlist_events;
29use crate::config::Config;
30use crate::constants::{self, Blocked, Chattype, ShowEmails};
31use crate::contact::{Contact, ContactId, Modifier, Origin};
32use crate::context::Context;
33use crate::events::EventType;
34use crate::headerdef::{HeaderDef, HeaderDefMap};
35use crate::log::{LogExt, error, info, warn};
36use crate::login_param::{
37    ConfiguredLoginParam, ConfiguredServerLoginParam, prioritize_server_login_params,
38};
39use crate::message::{self, Message, MessageState, MessengerMessage, MsgId};
40use crate::mimeparser;
41use crate::net::proxy::ProxyConfig;
42use crate::net::session::SessionStream;
43use crate::oauth2::get_oauth2_access_token;
44use crate::push::encrypt_device_token;
45use crate::receive_imf::{
46    ReceivedMsg, from_field_to_contact_id, get_prefetch_parent_message, receive_imf_inner,
47};
48use crate::scheduler::connectivity::ConnectivityStore;
49use crate::stock_str;
50use crate::tools::{self, create_id, duration_to_str};
51
52pub(crate) mod capabilities;
53mod client;
54mod idle;
55pub mod scan_folders;
56pub mod select_folder;
57pub(crate) mod session;
58
59use client::{Client, determine_capabilities};
60use mailparse::SingleInfo;
61use session::Session;
62
63pub(crate) const GENERATED_PREFIX: &str = "GEN_";
64
65const RFC724MID_UID: &str = "(UID BODY.PEEK[HEADER.FIELDS (\
66                             MESSAGE-ID \
67                             X-MICROSOFT-ORIGINAL-MESSAGE-ID\
68                             )])";
69const BODY_FULL: &str = "(FLAGS BODY.PEEK[])";
70const BODY_PARTIAL: &str = "(FLAGS RFC822.SIZE BODY.PEEK[HEADER])";
71
72#[derive(Debug)]
73pub(crate) struct Imap {
74    pub(crate) idle_interrupt_receiver: Receiver<()>,
75
76    /// Email address.
77    addr: String,
78
79    /// Login parameters.
80    lp: Vec<ConfiguredServerLoginParam>,
81
82    /// Password.
83    password: String,
84
85    /// Proxy configuration.
86    proxy_config: Option<ProxyConfig>,
87
88    strict_tls: bool,
89
90    oauth2: bool,
91
92    authentication_failed_once: bool,
93
94    pub(crate) connectivity: ConnectivityStore,
95
96    conn_last_try: tools::Time,
97    conn_backoff_ms: u64,
98
99    /// Rate limit for successful IMAP connections.
100    ///
101    /// This rate limit prevents busy loop in case the server refuses logins
102    /// or in case connection gets dropped over and over due to IMAP bug,
103    /// e.g. the server returning invalid response to SELECT command
104    /// immediately after logging in or returning an error in response to LOGIN command
105    /// due to internal server error.
106    ratelimit: Ratelimit,
107}
108
109#[derive(Debug)]
110struct OAuth2 {
111    user: String,
112    access_token: String,
113}
114
115#[derive(Debug)]
116pub(crate) struct ServerMetadata {
117    /// IMAP METADATA `/shared/comment` as defined in
118    /// <https://www.rfc-editor.org/rfc/rfc5464#section-6.2.1>.
119    pub comment: Option<String>,
120
121    /// IMAP METADATA `/shared/admin` as defined in
122    /// <https://www.rfc-editor.org/rfc/rfc5464#section-6.2.2>.
123    pub admin: Option<String>,
124
125    pub iroh_relay: Option<Url>,
126}
127
128impl async_imap::Authenticator for OAuth2 {
129    type Response = String;
130
131    fn process(&mut self, _data: &[u8]) -> Self::Response {
132        format!(
133            "user={}\x01auth=Bearer {}\x01\x01",
134            self.user, self.access_token
135        )
136    }
137}
138
139#[derive(Debug, Display, PartialEq, Eq, Clone, Copy)]
140pub enum FolderMeaning {
141    Unknown,
142
143    /// Spam folder.
144    Spam,
145    Inbox,
146    Mvbox,
147    Sent,
148    Trash,
149
150    /// Virtual folders.
151    ///
152    /// On Gmail there are virtual folders marked as \\All, \\Important and \\Flagged.
153    /// Delta Chat ignores these folders because the same messages can be fetched
154    /// from the real folder and the result of moving and deleting messages via
155    /// virtual folder is unclear.
156    Virtual,
157}
158
159impl FolderMeaning {
160    pub fn to_config(self) -> Option<Config> {
161        match self {
162            FolderMeaning::Unknown => None,
163            FolderMeaning::Spam => None,
164            FolderMeaning::Inbox => Some(Config::ConfiguredInboxFolder),
165            FolderMeaning::Mvbox => Some(Config::ConfiguredMvboxFolder),
166            FolderMeaning::Sent => Some(Config::ConfiguredSentboxFolder),
167            FolderMeaning::Trash => Some(Config::ConfiguredTrashFolder),
168            FolderMeaning::Virtual => None,
169        }
170    }
171}
172
173struct UidGrouper<T: Iterator<Item = (i64, u32, String)>> {
174    inner: Peekable<T>,
175}
176
177impl<T, I> From<I> for UidGrouper<T>
178where
179    T: Iterator<Item = (i64, u32, String)>,
180    I: IntoIterator<IntoIter = T>,
181{
182    fn from(inner: I) -> Self {
183        Self {
184            inner: inner.into_iter().peekable(),
185        }
186    }
187}
188
189impl<T: Iterator<Item = (i64, u32, String)>> Iterator for UidGrouper<T> {
190    // Tuple of folder, row IDs, and UID range as a string.
191    type Item = (String, Vec<i64>, String);
192
193    fn next(&mut self) -> Option<Self::Item> {
194        let (_, _, folder) = self.inner.peek().cloned()?;
195
196        let mut uid_set = String::new();
197        let mut rowid_set = Vec::new();
198
199        while uid_set.len() < 1000 {
200            // Construct a new range.
201            if let Some((start_rowid, start_uid, _)) = self
202                .inner
203                .next_if(|(_, _, start_folder)| start_folder == &folder)
204            {
205                rowid_set.push(start_rowid);
206                let mut end_uid = start_uid;
207
208                while let Some((next_rowid, next_uid, _)) =
209                    self.inner.next_if(|(_, next_uid, next_folder)| {
210                        next_folder == &folder && (*next_uid == end_uid + 1 || *next_uid == end_uid)
211                    })
212                {
213                    end_uid = next_uid;
214                    rowid_set.push(next_rowid);
215                }
216
217                let uid_range = UidRange {
218                    start: start_uid,
219                    end: end_uid,
220                };
221                if !uid_set.is_empty() {
222                    uid_set.push(',');
223                }
224                uid_set.push_str(&uid_range.to_string());
225            } else {
226                break;
227            }
228        }
229
230        Some((folder, rowid_set, uid_set))
231    }
232}
233
234impl Imap {
235    /// Creates new disconnected IMAP client using the specific login parameters.
236    ///
237    /// `addr` is used to renew token if OAuth2 authentication is used.
238    pub fn new(
239        lp: Vec<ConfiguredServerLoginParam>,
240        password: String,
241        proxy_config: Option<ProxyConfig>,
242        addr: &str,
243        strict_tls: bool,
244        oauth2: bool,
245        idle_interrupt_receiver: Receiver<()>,
246    ) -> Self {
247        Imap {
248            idle_interrupt_receiver,
249            addr: addr.to_string(),
250            lp,
251            password,
252            proxy_config,
253            strict_tls,
254            oauth2,
255            authentication_failed_once: false,
256            connectivity: Default::default(),
257            conn_last_try: UNIX_EPOCH,
258            conn_backoff_ms: 0,
259            // 1 connection per minute + a burst of 2.
260            ratelimit: Ratelimit::new(Duration::new(120, 0), 2.0),
261        }
262    }
263
264    /// Creates new disconnected IMAP client using configured parameters.
265    pub async fn new_configured(
266        context: &Context,
267        idle_interrupt_receiver: Receiver<()>,
268    ) -> Result<Self> {
269        let param = ConfiguredLoginParam::load(context)
270            .await?
271            .context("Not configured")?;
272        let proxy_config = ProxyConfig::load(context).await?;
273        let strict_tls = param.strict_tls(proxy_config.is_some());
274        let imap = Self::new(
275            param.imap.clone(),
276            param.imap_password.clone(),
277            proxy_config,
278            &param.addr,
279            strict_tls,
280            param.oauth2,
281            idle_interrupt_receiver,
282        );
283        Ok(imap)
284    }
285
286    /// Connects to IMAP server and returns a new IMAP session.
287    ///
288    /// Calling this function is not enough to perform IMAP operations. Use [`Imap::prepare`]
289    /// instead if you are going to actually use connection rather than trying connection
290    /// parameters.
291    pub(crate) async fn connect(
292        &mut self,
293        context: &Context,
294        configuring: bool,
295    ) -> Result<Session> {
296        let now = tools::Time::now();
297        let until_can_send = max(
298            min(self.conn_last_try, now)
299                .checked_add(Duration::from_millis(self.conn_backoff_ms))
300                .unwrap_or(now),
301            now,
302        )
303        .duration_since(now)?;
304        let ratelimit_duration = max(until_can_send, self.ratelimit.until_can_send());
305        if !ratelimit_duration.is_zero() {
306            warn!(
307                context,
308                "IMAP got rate limited, waiting for {} until can connect.",
309                duration_to_str(ratelimit_duration),
310            );
311            let interrupted = async {
312                tokio::time::sleep(ratelimit_duration).await;
313                false
314            }
315            .race(self.idle_interrupt_receiver.recv().map(|_| true))
316            .await;
317            if interrupted {
318                info!(
319                    context,
320                    "Connecting to IMAP without waiting for ratelimit due to interrupt."
321                );
322            }
323        }
324
325        info!(context, "Connecting to IMAP server.");
326        self.connectivity.set_connecting(context);
327
328        self.conn_last_try = tools::Time::now();
329        const BACKOFF_MIN_MS: u64 = 2000;
330        const BACKOFF_MAX_MS: u64 = 80_000;
331        self.conn_backoff_ms = min(self.conn_backoff_ms, BACKOFF_MAX_MS / 2);
332        self.conn_backoff_ms = self.conn_backoff_ms.saturating_add(
333            rand::thread_rng().gen_range((self.conn_backoff_ms / 2)..=self.conn_backoff_ms),
334        );
335        self.conn_backoff_ms = max(BACKOFF_MIN_MS, self.conn_backoff_ms);
336
337        let login_params = prioritize_server_login_params(&context.sql, &self.lp, "imap").await?;
338        let mut first_error = None;
339        for lp in login_params {
340            info!(context, "IMAP trying to connect to {}.", &lp.connection);
341            let connection_candidate = lp.connection.clone();
342            let client = match Client::connect(
343                context,
344                self.proxy_config.clone(),
345                self.strict_tls,
346                connection_candidate,
347            )
348            .await
349            .context("IMAP failed to connect")
350            {
351                Ok(client) => client,
352                Err(err) => {
353                    warn!(context, "{err:#}.");
354                    first_error.get_or_insert(err);
355                    continue;
356                }
357            };
358
359            self.conn_backoff_ms = BACKOFF_MIN_MS;
360            self.ratelimit.send();
361
362            let imap_user: &str = lp.user.as_ref();
363            let imap_pw: &str = &self.password;
364
365            let login_res = if self.oauth2 {
366                info!(context, "Logging into IMAP server with OAuth 2.");
367                let addr: &str = self.addr.as_ref();
368
369                let token = get_oauth2_access_token(context, addr, imap_pw, true)
370                    .await?
371                    .context("IMAP could not get OAUTH token")?;
372                let auth = OAuth2 {
373                    user: imap_user.into(),
374                    access_token: token,
375                };
376                client.authenticate("XOAUTH2", auth).await
377            } else {
378                info!(context, "Logging into IMAP server with LOGIN.");
379                client.login(imap_user, imap_pw).await
380            };
381
382            match login_res {
383                Ok(mut session) => {
384                    let capabilities = determine_capabilities(&mut session).await?;
385
386                    let session = if capabilities.can_compress {
387                        info!(context, "Enabling IMAP compression.");
388                        let compressed_session = session
389                            .compress(|s| {
390                                let session_stream: Box<dyn SessionStream> = Box::new(s);
391                                session_stream
392                            })
393                            .await
394                            .context("Failed to enable IMAP compression")?;
395                        Session::new(compressed_session, capabilities)
396                    } else {
397                        Session::new(session, capabilities)
398                    };
399
400                    // Store server ID in the context to display in account info.
401                    let mut lock = context.server_id.write().await;
402                    lock.clone_from(&session.capabilities.server_id);
403
404                    self.authentication_failed_once = false;
405                    context.emit_event(EventType::ImapConnected(format!(
406                        "IMAP-LOGIN as {}",
407                        lp.user
408                    )));
409                    self.connectivity.set_preparing(context);
410                    info!(context, "Successfully logged into IMAP server.");
411                    return Ok(session);
412                }
413
414                Err(err) => {
415                    let imap_user = lp.user.to_owned();
416                    let message = stock_str::cannot_login(context, &imap_user).await;
417
418                    warn!(context, "IMAP failed to login: {err:#}.");
419                    first_error.get_or_insert(format_err!("{message} ({err:#})"));
420
421                    // If it looks like the password is wrong, send a notification:
422                    let _lock = context.wrong_pw_warning_mutex.lock().await;
423                    if err.to_string().to_lowercase().contains("authentication") {
424                        if self.authentication_failed_once
425                            && !configuring
426                            && context.get_config_bool(Config::NotifyAboutWrongPw).await?
427                        {
428                            let mut msg = Message::new_text(message);
429                            if let Err(e) = chat::add_device_msg_with_importance(
430                                context,
431                                None,
432                                Some(&mut msg),
433                                true,
434                            )
435                            .await
436                            {
437                                warn!(context, "Failed to add device message: {e:#}.");
438                            } else {
439                                context
440                                    .set_config_internal(Config::NotifyAboutWrongPw, None)
441                                    .await
442                                    .log_err(context)
443                                    .ok();
444                            }
445                        } else {
446                            self.authentication_failed_once = true;
447                        }
448                    } else {
449                        self.authentication_failed_once = false;
450                    }
451                }
452            }
453        }
454
455        Err(first_error.unwrap_or_else(|| format_err!("No IMAP connection candidates provided")))
456    }
457
458    /// Prepare a new IMAP session.
459    ///
460    /// This creates a new IMAP connection and ensures
461    /// that folders are created and IMAP capabilities are determined.
462    pub(crate) async fn prepare(&mut self, context: &Context) -> Result<Session> {
463        let configuring = false;
464        let mut session = match self.connect(context, configuring).await {
465            Ok(session) => session,
466            Err(err) => {
467                self.connectivity.set_err(context, &err);
468                return Err(err);
469            }
470        };
471
472        let folders_configured = context
473            .sql
474            .get_raw_config_int(constants::DC_FOLDERS_CONFIGURED_KEY)
475            .await?;
476        if folders_configured.unwrap_or_default() < constants::DC_FOLDERS_CONFIGURED_VERSION {
477            let is_chatmail = match context.get_config_bool(Config::FixIsChatmail).await? {
478                false => session.is_chatmail(),
479                true => context.get_config_bool(Config::IsChatmail).await?,
480            };
481            let create_mvbox = !is_chatmail || context.get_config_bool(Config::MvboxMove).await?;
482            self.configure_folders(context, &mut session, create_mvbox)
483                .await?;
484        }
485
486        Ok(session)
487    }
488
489    /// FETCH-MOVE-DELETE iteration.
490    ///
491    /// Prefetches headers and downloads new message from the folder, moves messages away from the
492    /// folder and deletes messages in the folder.
493    pub async fn fetch_move_delete(
494        &mut self,
495        context: &Context,
496        session: &mut Session,
497        watch_folder: &str,
498        folder_meaning: FolderMeaning,
499    ) -> Result<()> {
500        if !context.sql.is_open().await {
501            // probably shutdown
502            bail!("IMAP operation attempted while it is torn down");
503        }
504
505        let msgs_fetched = self
506            .fetch_new_messages(context, session, watch_folder, folder_meaning)
507            .await
508            .context("fetch_new_messages")?;
509        if msgs_fetched && context.get_config_delete_device_after().await?.is_some() {
510            // New messages were fetched and shall be deleted later, restart ephemeral loop.
511            // Note that the `Config::DeleteDeviceAfter` timer starts as soon as the messages are
512            // fetched while the per-chat ephemeral timers start as soon as the messages are marked
513            // as noticed.
514            context.scheduler.interrupt_ephemeral_task().await;
515        }
516
517        session
518            .move_delete_messages(context, watch_folder)
519            .await
520            .context("move_delete_messages")?;
521
522        Ok(())
523    }
524
525    /// Fetches new messages.
526    ///
527    /// Returns true if at least one message was fetched.
528    pub(crate) async fn fetch_new_messages(
529        &mut self,
530        context: &Context,
531        session: &mut Session,
532        folder: &str,
533        folder_meaning: FolderMeaning,
534    ) -> Result<bool> {
535        if should_ignore_folder(context, folder, folder_meaning).await? {
536            info!(context, "Not fetching from {folder:?}.");
537            session.new_mail = false;
538            return Ok(false);
539        }
540
541        let create = false;
542        let folder_exists = session
543            .select_with_uidvalidity(context, folder, create)
544            .await
545            .with_context(|| format!("Failed to select folder {folder:?}"))?;
546        if !folder_exists {
547            return Ok(false);
548        }
549
550        if !session.new_mail {
551            info!(context, "No new emails in folder {folder:?}.");
552            return Ok(false);
553        }
554        session.new_mail = false;
555
556        let uid_validity = get_uidvalidity(context, folder).await?;
557        let old_uid_next = get_uid_next(context, folder).await?;
558
559        let msgs = session.prefetch(old_uid_next).await.context("prefetch")?;
560        let read_cnt = msgs.len();
561
562        let download_limit = context.download_limit().await?;
563        let mut uids_fetch = Vec::<(u32, bool /* partially? */)>::with_capacity(msgs.len() + 1);
564        let mut uid_message_ids = BTreeMap::new();
565        let mut largest_uid_skipped = None;
566        let delete_target = context.get_delete_msgs_target().await?;
567
568        // Store the info about IMAP messages in the database.
569        for (uid, ref fetch_response) in msgs {
570            let headers = match get_fetch_headers(fetch_response) {
571                Ok(headers) => headers,
572                Err(err) => {
573                    warn!(context, "Failed to parse FETCH headers: {err:#}.");
574                    continue;
575                }
576            };
577
578            let message_id = prefetch_get_message_id(&headers);
579
580            // Determine the target folder where the message should be moved to.
581            //
582            // If we have seen the message on the IMAP server before, do not move it.
583            // This is required to avoid infinite MOVE loop on IMAP servers
584            // that alias `DeltaChat` folder to other names.
585            // For example, some Dovecot servers alias `DeltaChat` folder to `INBOX.DeltaChat`.
586            // In this case Delta Chat configured with `DeltaChat` as the destination folder
587            // would detect messages in the `INBOX.DeltaChat` folder
588            // and try to move them to the `DeltaChat` folder.
589            // Such move to the same folder results in the messages
590            // getting a new UID, so the messages will be detected as new
591            // in the `INBOX.DeltaChat` folder again.
592            let _target;
593            let target = if let Some(message_id) = &message_id {
594                let msg_info =
595                    message::rfc724_mid_exists_ex(context, message_id, "deleted=1").await?;
596                let delete = if let Some((_, _, true)) = msg_info {
597                    info!(context, "Deleting locally deleted message {message_id}.");
598                    true
599                } else if let Some((_, ts_sent_old, _)) = msg_info {
600                    let is_chat_msg = headers.get_header_value(HeaderDef::ChatVersion).is_some();
601                    let ts_sent = headers
602                        .get_header_value(HeaderDef::Date)
603                        .and_then(|v| mailparse::dateparse(&v).ok())
604                        .unwrap_or_default();
605                    let is_dup = is_dup_msg(is_chat_msg, ts_sent, ts_sent_old);
606                    if is_dup {
607                        info!(context, "Deleting duplicate message {message_id}.");
608                    }
609                    is_dup
610                } else {
611                    false
612                };
613                if delete {
614                    &delete_target
615                } else if context
616                    .sql
617                    .exists(
618                        "SELECT COUNT (*) FROM imap WHERE rfc724_mid=?",
619                        (message_id,),
620                    )
621                    .await?
622                {
623                    info!(
624                        context,
625                        "Not moving the message {} that we have seen before.", &message_id
626                    );
627                    folder
628                } else {
629                    _target = target_folder(context, folder, folder_meaning, &headers).await?;
630                    &_target
631                }
632            } else {
633                // Do not move the messages without Message-ID.
634                // We cannot reliably determine if we have seen them before,
635                // so it is safer not to move them.
636                warn!(
637                    context,
638                    "Not moving the message that does not have a Message-ID."
639                );
640                folder
641            };
642
643            // Generate a fake Message-ID to identify the message in the database
644            // if the message has no real Message-ID.
645            let message_id = message_id.unwrap_or_else(create_message_id);
646
647            context
648                .sql
649                .execute(
650                    "INSERT INTO imap (rfc724_mid, folder, uid, uidvalidity, target)
651                       VALUES         (?1,         ?2,     ?3,  ?4,          ?5)
652                       ON CONFLICT(folder, uid, uidvalidity)
653                       DO UPDATE SET rfc724_mid=excluded.rfc724_mid,
654                                     target=excluded.target",
655                    (&message_id, &folder, uid, uid_validity, target),
656                )
657                .await?;
658
659            // Download only the messages which have reached their target folder if there are
660            // multiple devices. This prevents race conditions in multidevice case, where one
661            // device tries to download the message while another device moves the message at the
662            // same time. Even in single device case it is possible to fail downloading the first
663            // message, move it to the movebox and then download the second message before
664            // downloading the first one, if downloading from inbox before moving is allowed.
665            if folder == target
666                // Never download messages directly from the spam folder.
667                // If the sender is known, the message will be moved to the Inbox or Mvbox
668                // and then we download the message from there.
669                // Also see `spam_target_folder_cfg()`.
670                && folder_meaning != FolderMeaning::Spam
671                && prefetch_should_download(
672                    context,
673                    &headers,
674                    &message_id,
675                    fetch_response.flags(),
676                )
677                .await.context("prefetch_should_download")?
678            {
679                match download_limit {
680                    Some(download_limit) => uids_fetch.push((
681                        uid,
682                        fetch_response.size.unwrap_or_default() > download_limit,
683                    )),
684                    None => uids_fetch.push((uid, false)),
685                }
686                uid_message_ids.insert(uid, message_id);
687            } else {
688                largest_uid_skipped = Some(uid);
689            }
690        }
691
692        if !uids_fetch.is_empty() {
693            self.connectivity.set_working(context);
694        }
695
696        let (sender, receiver) = async_channel::unbounded();
697
698        let mut received_msgs = Vec::with_capacity(uids_fetch.len());
699        let mailbox_uid_next = session
700            .selected_mailbox
701            .as_ref()
702            .with_context(|| format!("Expected {folder:?} to be selected"))?
703            .uid_next
704            .unwrap_or_default();
705
706        let update_uids_future = async {
707            let mut largest_uid_fetched: u32 = 0;
708
709            while let Ok((uid, received_msg_opt)) = receiver.recv().await {
710                largest_uid_fetched = max(largest_uid_fetched, uid);
711                if let Some(received_msg) = received_msg_opt {
712                    received_msgs.push(received_msg)
713                }
714            }
715
716            largest_uid_fetched
717        };
718
719        let actually_download_messages_future = async move {
720            let mut uids_fetch_in_batch = Vec::with_capacity(max(uids_fetch.len(), 1));
721            let mut fetch_partially = false;
722            uids_fetch.push((0, !uids_fetch.last().unwrap_or(&(0, false)).1));
723            for (uid, fp) in uids_fetch {
724                if fp != fetch_partially {
725                    session
726                        .fetch_many_msgs(
727                            context,
728                            folder,
729                            uid_validity,
730                            uids_fetch_in_batch.split_off(0),
731                            &uid_message_ids,
732                            fetch_partially,
733                            sender.clone(),
734                        )
735                        .await
736                        .context("fetch_many_msgs")?;
737                    fetch_partially = fp;
738                }
739                uids_fetch_in_batch.push(uid);
740            }
741
742            anyhow::Ok(())
743        };
744
745        let (largest_uid_fetched, fetch_res) =
746            tokio::join!(update_uids_future, actually_download_messages_future);
747
748        // Advance uid_next to the largest fetched UID plus 1.
749        //
750        // This may be larger than `mailbox_uid_next`
751        // if the message has arrived after selecting mailbox
752        // and determining its UIDNEXT and before prefetch.
753        let mut new_uid_next = largest_uid_fetched + 1;
754        if fetch_res.is_ok() {
755            // If we have successfully fetched all messages we planned during prefetch,
756            // then we have covered at least the range between old UIDNEXT
757            // and UIDNEXT of the mailbox at the time of selecting it.
758            new_uid_next = max(new_uid_next, mailbox_uid_next);
759
760            new_uid_next = max(new_uid_next, largest_uid_skipped.unwrap_or(0) + 1);
761        }
762        if new_uid_next > old_uid_next {
763            set_uid_next(context, folder, new_uid_next).await?;
764        }
765
766        info!(context, "{} mails read from \"{}\".", read_cnt, folder);
767
768        if !received_msgs.is_empty() {
769            context.emit_event(EventType::IncomingMsgBunch);
770        }
771
772        chat::mark_old_messages_as_noticed(context, received_msgs).await?;
773
774        // Now fail if fetching failed, so we will
775        // establish a new session if this one is broken.
776        fetch_res?;
777
778        Ok(read_cnt > 0)
779    }
780
781    /// Read the recipients from old emails sent by the user and add them as contacts.
782    /// This way, we can already offer them some email addresses they can write to.
783    ///
784    /// Then, Fetch the last messages DC_FETCH_EXISTING_MSGS_COUNT emails from the server
785    /// and show them in the chat list.
786    pub(crate) async fn fetch_existing_msgs(
787        &mut self,
788        context: &Context,
789        session: &mut Session,
790    ) -> Result<()> {
791        add_all_recipients_as_contacts(context, session, Config::ConfiguredSentboxFolder)
792            .await
793            .context("failed to get recipients from the sentbox")?;
794        add_all_recipients_as_contacts(context, session, Config::ConfiguredMvboxFolder)
795            .await
796            .context("failed to get recipients from the movebox")?;
797        add_all_recipients_as_contacts(context, session, Config::ConfiguredInboxFolder)
798            .await
799            .context("failed to get recipients from the inbox")?;
800
801        info!(context, "Done fetching existing messages.");
802        Ok(())
803    }
804}
805
806impl Session {
807    /// Synchronizes UIDs for all folders.
808    pub(crate) async fn resync_folders(&mut self, context: &Context) -> Result<()> {
809        let all_folders = self
810            .list_folders()
811            .await
812            .context("listing folders for resync")?;
813        for folder in all_folders {
814            let folder_meaning = get_folder_meaning(&folder);
815            if !matches!(
816                folder_meaning,
817                FolderMeaning::Virtual | FolderMeaning::Unknown
818            ) {
819                self.resync_folder_uids(context, folder.name(), folder_meaning)
820                    .await?;
821            }
822        }
823        Ok(())
824    }
825
826    /// Synchronizes UIDs in the database with UIDs on the server.
827    ///
828    /// It is assumed that no operations are taking place on the same
829    /// folder at the moment. Make sure to run it in the same
830    /// thread/task as other network operations on this folder to
831    /// avoid race conditions.
832    pub(crate) async fn resync_folder_uids(
833        &mut self,
834        context: &Context,
835        folder: &str,
836        folder_meaning: FolderMeaning,
837    ) -> Result<()> {
838        let uid_validity;
839        // Collect pairs of UID and Message-ID.
840        let mut msgs = BTreeMap::new();
841
842        let create = false;
843        let folder_exists = self
844            .select_with_uidvalidity(context, folder, create)
845            .await?;
846        if folder_exists {
847            let mut list = self
848                .uid_fetch("1:*", RFC724MID_UID)
849                .await
850                .with_context(|| format!("Can't resync folder {folder}"))?;
851            while let Some(fetch) = list.try_next().await? {
852                let headers = match get_fetch_headers(&fetch) {
853                    Ok(headers) => headers,
854                    Err(err) => {
855                        warn!(context, "Failed to parse FETCH headers: {}", err);
856                        continue;
857                    }
858                };
859                let message_id = prefetch_get_message_id(&headers);
860
861                if let (Some(uid), Some(rfc724_mid)) = (fetch.uid, message_id) {
862                    msgs.insert(
863                        uid,
864                        (
865                            rfc724_mid,
866                            target_folder(context, folder, folder_meaning, &headers).await?,
867                        ),
868                    );
869                }
870            }
871
872            info!(
873                context,
874                "resync_folder_uids: Collected {} message IDs in {folder}.",
875                msgs.len(),
876            );
877
878            uid_validity = get_uidvalidity(context, folder).await?;
879        } else {
880            warn!(context, "resync_folder_uids: No folder {folder}.");
881            uid_validity = 0;
882        }
883
884        // Write collected UIDs to SQLite database.
885        context
886            .sql
887            .transaction(move |transaction| {
888                transaction.execute("DELETE FROM imap WHERE folder=?", (folder,))?;
889                for (uid, (rfc724_mid, target)) in &msgs {
890                    // This may detect previously undetected moved
891                    // messages, so we update server_folder too.
892                    transaction.execute(
893                        "INSERT INTO imap (rfc724_mid, folder, uid, uidvalidity, target)
894                         VALUES           (?1,         ?2,     ?3,  ?4,          ?5)
895                         ON CONFLICT(folder, uid, uidvalidity)
896                         DO UPDATE SET rfc724_mid=excluded.rfc724_mid,
897                                       target=excluded.target",
898                        (rfc724_mid, folder, uid, uid_validity, target),
899                    )?;
900                }
901                Ok(())
902            })
903            .await?;
904        Ok(())
905    }
906
907    /// Deletes batch of messages identified by their UID from the currently
908    /// selected folder.
909    async fn delete_message_batch(
910        &mut self,
911        context: &Context,
912        uid_set: &str,
913        row_ids: Vec<i64>,
914    ) -> Result<()> {
915        // mark the message for deletion
916        self.add_flag_finalized_with_set(uid_set, "\\Deleted")
917            .await?;
918        context
919            .sql
920            .transaction(|transaction| {
921                let mut stmt = transaction.prepare("DELETE FROM imap WHERE id = ?")?;
922                for row_id in row_ids {
923                    stmt.execute((row_id,))?;
924                }
925                Ok(())
926            })
927            .await
928            .context("Cannot remove deleted messages from imap table")?;
929
930        context.emit_event(EventType::ImapMessageDeleted(format!(
931            "IMAP messages {uid_set} marked as deleted"
932        )));
933        Ok(())
934    }
935
936    /// Moves batch of messages identified by their UID from the currently
937    /// selected folder to the target folder.
938    async fn move_message_batch(
939        &mut self,
940        context: &Context,
941        set: &str,
942        row_ids: Vec<i64>,
943        target: &str,
944    ) -> Result<()> {
945        if self.can_move() {
946            match self.uid_mv(set, &target).await {
947                Ok(()) => {
948                    // Messages are moved or don't exist, IMAP returns OK response in both cases.
949                    context
950                        .sql
951                        .transaction(|transaction| {
952                            let mut stmt = transaction.prepare("DELETE FROM imap WHERE id = ?")?;
953                            for row_id in row_ids {
954                                stmt.execute((row_id,))?;
955                            }
956                            Ok(())
957                        })
958                        .await
959                        .context("Cannot delete moved messages from imap table")?;
960                    context.emit_event(EventType::ImapMessageMoved(format!(
961                        "IMAP messages {set} moved to {target}"
962                    )));
963                    return Ok(());
964                }
965                Err(err) => {
966                    if context.should_delete_to_trash().await? {
967                        error!(
968                            context,
969                            "Cannot move messages {} to {}, no fallback to COPY/DELETE because \
970                            delete_to_trash is set. Error: {:#}",
971                            set,
972                            target,
973                            err,
974                        );
975                        return Err(err.into());
976                    }
977                    warn!(
978                        context,
979                        "Cannot move messages, fallback to COPY/DELETE {} to {}: {}",
980                        set,
981                        target,
982                        err
983                    );
984                }
985            }
986        }
987
988        // Server does not support MOVE or MOVE failed.
989        // Copy messages to the destination folder if needed and mark records for deletion.
990        let copy = !context.is_trash(target).await?;
991        if copy {
992            info!(
993                context,
994                "Server does not support MOVE, fallback to COPY/DELETE {} to {}", set, target
995            );
996            self.uid_copy(&set, &target).await?;
997        } else {
998            error!(
999                context,
1000                "Server does not support MOVE, fallback to DELETE {} to {}", set, target,
1001            );
1002        }
1003        context
1004            .sql
1005            .transaction(|transaction| {
1006                let mut stmt = transaction.prepare("UPDATE imap SET target='' WHERE id = ?")?;
1007                for row_id in row_ids {
1008                    stmt.execute((row_id,))?;
1009                }
1010                Ok(())
1011            })
1012            .await
1013            .context("Cannot plan deletion of messages")?;
1014        if copy {
1015            context.emit_event(EventType::ImapMessageMoved(format!(
1016                "IMAP messages {set} copied to {target}"
1017            )));
1018        }
1019        Ok(())
1020    }
1021
1022    /// Moves and deletes messages as planned in the `imap` table.
1023    ///
1024    /// This is the only place where messages are moved or deleted on the IMAP server.
1025    async fn move_delete_messages(&mut self, context: &Context, folder: &str) -> Result<()> {
1026        let rows = context
1027            .sql
1028            .query_map(
1029                "SELECT id, uid, target FROM imap
1030        WHERE folder = ?
1031        AND target != folder
1032        ORDER BY target, uid",
1033                (folder,),
1034                |row| {
1035                    let rowid: i64 = row.get(0)?;
1036                    let uid: u32 = row.get(1)?;
1037                    let target: String = row.get(2)?;
1038                    Ok((rowid, uid, target))
1039                },
1040                |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
1041            )
1042            .await?;
1043
1044        for (target, rowid_set, uid_set) in UidGrouper::from(rows) {
1045            // Select folder inside the loop to avoid selecting it if there are no pending
1046            // MOVE/DELETE operations. This does not result in multiple SELECT commands
1047            // being sent because `select_folder()` does nothing if the folder is already
1048            // selected.
1049            let create = false;
1050            let folder_exists = self
1051                .select_with_uidvalidity(context, folder, create)
1052                .await?;
1053            ensure!(folder_exists, "No folder {folder}");
1054
1055            // Empty target folder name means messages should be deleted.
1056            if target.is_empty() {
1057                self.delete_message_batch(context, &uid_set, rowid_set)
1058                    .await
1059                    .with_context(|| format!("cannot delete batch of messages {:?}", &uid_set))?;
1060            } else {
1061                self.move_message_batch(context, &uid_set, rowid_set, &target)
1062                    .await
1063                    .with_context(|| {
1064                        format!(
1065                            "cannot move batch of messages {:?} to folder {:?}",
1066                            &uid_set, target
1067                        )
1068                    })?;
1069            }
1070        }
1071
1072        // Expunge folder if needed, e.g. if some jobs have
1073        // deleted messages on the server.
1074        if let Err(err) = self.maybe_close_folder(context).await {
1075            warn!(context, "Failed to close folder: {err:#}.");
1076        }
1077
1078        Ok(())
1079    }
1080
1081    /// Uploads sync messages from the `imap_send` table with `\Seen` flag set.
1082    pub(crate) async fn send_sync_msgs(&mut self, context: &Context, folder: &str) -> Result<()> {
1083        context.send_sync_msg().await?;
1084        while let Some((id, mime, msg_id, attempts)) = context
1085            .sql
1086            .query_row_optional(
1087                "SELECT id, mime, msg_id, attempts FROM imap_send ORDER BY id LIMIT 1",
1088                (),
1089                |row| {
1090                    let id: i64 = row.get(0)?;
1091                    let mime: String = row.get(1)?;
1092                    let msg_id: MsgId = row.get(2)?;
1093                    let attempts: i64 = row.get(3)?;
1094                    Ok((id, mime, msg_id, attempts))
1095                },
1096            )
1097            .await
1098            .context("Failed to SELECT from imap_send")?
1099        {
1100            let res = self
1101                .append(folder, Some("(\\Seen)"), None, mime)
1102                .await
1103                .with_context(|| format!("IMAP APPEND to {folder} failed for {msg_id}"))
1104                .log_err(context);
1105            if res.is_ok() {
1106                msg_id.set_delivered(context).await?;
1107            }
1108            const MAX_ATTEMPTS: i64 = 2;
1109            if res.is_ok() || attempts >= MAX_ATTEMPTS - 1 {
1110                context
1111                    .sql
1112                    .execute("DELETE FROM imap_send WHERE id=?", (id,))
1113                    .await
1114                    .context("Failed to delete from imap_send")?;
1115            } else {
1116                context
1117                    .sql
1118                    .execute("UPDATE imap_send SET attempts=attempts+1 WHERE id=?", (id,))
1119                    .await
1120                    .context("Failed to update imap_send.attempts")?;
1121                res?;
1122            }
1123        }
1124        Ok(())
1125    }
1126
1127    /// Stores pending `\Seen` flags for messages in `imap_markseen` table.
1128    pub(crate) async fn store_seen_flags_on_imap(&mut self, context: &Context) -> Result<()> {
1129        let rows = context
1130            .sql
1131            .query_map(
1132                "SELECT imap.id, uid, folder FROM imap, imap_markseen
1133                 WHERE imap.id = imap_markseen.id AND target = folder
1134                 ORDER BY folder, uid",
1135                [],
1136                |row| {
1137                    let rowid: i64 = row.get(0)?;
1138                    let uid: u32 = row.get(1)?;
1139                    let folder: String = row.get(2)?;
1140                    Ok((rowid, uid, folder))
1141                },
1142                |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
1143            )
1144            .await?;
1145
1146        for (folder, rowid_set, uid_set) in UidGrouper::from(rows) {
1147            let create = false;
1148            let folder_exists = match self.select_with_uidvalidity(context, &folder, create).await {
1149                Err(err) => {
1150                    warn!(
1151                        context,
1152                        "store_seen_flags_on_imap: Failed to select {folder}, will retry later: {err:#}."
1153                    );
1154                    continue;
1155                }
1156                Ok(folder_exists) => folder_exists,
1157            };
1158            if !folder_exists {
1159                warn!(context, "store_seen_flags_on_imap: No folder {folder}.");
1160            } else if let Err(err) = self.add_flag_finalized_with_set(&uid_set, "\\Seen").await {
1161                warn!(
1162                    context,
1163                    "Cannot mark messages {uid_set} in {folder} as seen, will retry later: {err:#}."
1164                );
1165                continue;
1166            } else {
1167                info!(
1168                    context,
1169                    "Marked messages {} in folder {} as seen.", uid_set, folder
1170                );
1171            }
1172            context
1173                .sql
1174                .transaction(|transaction| {
1175                    let mut stmt = transaction.prepare("DELETE FROM imap_markseen WHERE id = ?")?;
1176                    for rowid in rowid_set {
1177                        stmt.execute((rowid,))?;
1178                    }
1179                    Ok(())
1180                })
1181                .await
1182                .context("Cannot remove messages marked as seen from imap_markseen table")?;
1183        }
1184
1185        Ok(())
1186    }
1187
1188    /// Synchronizes `\Seen` flags using `CONDSTORE` extension.
1189    pub(crate) async fn sync_seen_flags(&mut self, context: &Context, folder: &str) -> Result<()> {
1190        if !self.can_condstore() {
1191            info!(
1192                context,
1193                "Server does not support CONDSTORE, skipping flag synchronization."
1194            );
1195            return Ok(());
1196        }
1197
1198        let create = false;
1199        let folder_exists = self
1200            .select_with_uidvalidity(context, folder, create)
1201            .await
1202            .context("Failed to select folder")?;
1203        if !folder_exists {
1204            return Ok(());
1205        }
1206
1207        let mailbox = self
1208            .selected_mailbox
1209            .as_ref()
1210            .with_context(|| format!("No mailbox selected, folder: {folder}"))?;
1211
1212        // Check if the mailbox supports MODSEQ.
1213        // We are not interested in actual value of HIGHESTMODSEQ.
1214        if mailbox.highest_modseq.is_none() {
1215            info!(
1216                context,
1217                "Mailbox {} does not support mod-sequences, skipping flag synchronization.", folder
1218            );
1219            return Ok(());
1220        }
1221
1222        let mut updated_chat_ids = BTreeSet::new();
1223        let uid_validity = get_uidvalidity(context, folder)
1224            .await
1225            .with_context(|| format!("failed to get UID validity for folder {folder}"))?;
1226        let mut highest_modseq = get_modseq(context, folder)
1227            .await
1228            .with_context(|| format!("failed to get MODSEQ for folder {folder}"))?;
1229        let mut list = self
1230            .uid_fetch("1:*", format!("(FLAGS) (CHANGEDSINCE {highest_modseq})"))
1231            .await
1232            .context("failed to fetch flags")?;
1233
1234        let mut got_unsolicited_fetch = false;
1235
1236        while let Some(fetch) = list
1237            .try_next()
1238            .await
1239            .context("failed to get FETCH result")?
1240        {
1241            let uid = if let Some(uid) = fetch.uid {
1242                uid
1243            } else {
1244                info!(context, "FETCH result contains no UID, skipping");
1245                got_unsolicited_fetch = true;
1246                continue;
1247            };
1248            let is_seen = fetch.flags().any(|flag| flag == Flag::Seen);
1249            if is_seen {
1250                if let Some(chat_id) = mark_seen_by_uid(context, folder, uid_validity, uid)
1251                    .await
1252                    .with_context(|| {
1253                        format!("failed to update seen status for msg {folder}/{uid}")
1254                    })?
1255                {
1256                    updated_chat_ids.insert(chat_id);
1257                }
1258            }
1259
1260            if let Some(modseq) = fetch.modseq {
1261                if modseq > highest_modseq {
1262                    highest_modseq = modseq;
1263                }
1264            } else {
1265                warn!(context, "FETCH result contains no MODSEQ");
1266            }
1267        }
1268        drop(list);
1269
1270        if got_unsolicited_fetch {
1271            // We got unsolicited FETCH, which means some flags
1272            // have been modified while our request was in progress.
1273            // We may or may not have these new flags as a part of the response,
1274            // so better skip next IDLE and do another round of flag synchronization.
1275            self.new_mail = true;
1276        }
1277
1278        set_modseq(context, folder, highest_modseq)
1279            .await
1280            .with_context(|| format!("failed to set MODSEQ for folder {folder}"))?;
1281        if !updated_chat_ids.is_empty() {
1282            context.on_archived_chats_maybe_noticed();
1283        }
1284        for updated_chat_id in updated_chat_ids {
1285            context.emit_event(EventType::MsgsNoticed(updated_chat_id));
1286            chatlist_events::emit_chatlist_item_changed(context, updated_chat_id);
1287        }
1288
1289        Ok(())
1290    }
1291
1292    /// Gets the from, to and bcc addresses from all existing outgoing emails.
1293    pub async fn get_all_recipients(&mut self, context: &Context) -> Result<Vec<SingleInfo>> {
1294        let mut uids: Vec<_> = self
1295            .uid_search(get_imap_self_sent_search_command(context).await?)
1296            .await?
1297            .into_iter()
1298            .collect();
1299        uids.sort_unstable();
1300
1301        let mut result = Vec::new();
1302        for (_, uid_set) in build_sequence_sets(&uids)? {
1303            let mut list = self
1304                .uid_fetch(uid_set, "(UID BODY.PEEK[HEADER.FIELDS (FROM TO CC BCC)])")
1305                .await
1306                .context("IMAP Could not fetch")?;
1307
1308            while let Some(msg) = list.try_next().await? {
1309                match get_fetch_headers(&msg) {
1310                    Ok(headers) => {
1311                        if let Some(from) = mimeparser::get_from(&headers) {
1312                            if context.is_self_addr(&from.addr).await? {
1313                                result.extend(mimeparser::get_recipients(&headers));
1314                            }
1315                        }
1316                    }
1317                    Err(err) => {
1318                        warn!(context, "{}", err);
1319                        continue;
1320                    }
1321                };
1322            }
1323        }
1324        Ok(result)
1325    }
1326
1327    /// Fetches a list of messages by server UID.
1328    ///
1329    /// Sends pairs of UID and info about each downloaded message to the provided channel.
1330    /// Received message info is optional because UID may be ignored
1331    /// if the message has a `\Deleted` flag.
1332    ///
1333    /// The channel is used to return the results because the function may fail
1334    /// due to network errors before it finishes fetching all the messages.
1335    /// In this case caller still may want to process all the results
1336    /// received over the channel and persist last seen UID in the database
1337    /// before bubbling up the failure.
1338    ///
1339    /// If the message is incorrect or there is a failure to write a message to the database,
1340    /// it is skipped and the error is logged.
1341    #[expect(clippy::too_many_arguments)]
1342    pub(crate) async fn fetch_many_msgs(
1343        &mut self,
1344        context: &Context,
1345        folder: &str,
1346        uidvalidity: u32,
1347        request_uids: Vec<u32>,
1348        uid_message_ids: &BTreeMap<u32, String>,
1349        fetch_partially: bool,
1350        received_msgs_channel: Sender<(u32, Option<ReceivedMsg>)>,
1351    ) -> Result<()> {
1352        if request_uids.is_empty() {
1353            return Ok(());
1354        }
1355
1356        for (request_uids, set) in build_sequence_sets(&request_uids)? {
1357            info!(
1358                context,
1359                "Starting a {} FETCH of message set \"{}\".",
1360                if fetch_partially { "partial" } else { "full" },
1361                set
1362            );
1363            let mut fetch_responses = self
1364                .uid_fetch(
1365                    &set,
1366                    if fetch_partially {
1367                        BODY_PARTIAL
1368                    } else {
1369                        BODY_FULL
1370                    },
1371                )
1372                .await
1373                .with_context(|| {
1374                    format!("fetching messages {} from folder \"{}\"", &set, folder)
1375                })?;
1376
1377            // Map from UIDs to unprocessed FETCH results. We put unprocessed FETCH results here
1378            // when we want to process other messages first.
1379            let mut uid_msgs = HashMap::with_capacity(request_uids.len());
1380
1381            let mut count = 0;
1382            for &request_uid in &request_uids {
1383                // Check if FETCH response is already in `uid_msgs`.
1384                let mut fetch_response = uid_msgs.remove(&request_uid);
1385
1386                // Try to find a requested UID in returned FETCH responses.
1387                while fetch_response.is_none() {
1388                    let Some(next_fetch_response) = fetch_responses
1389                        .try_next()
1390                        .await
1391                        .context("Failed to process IMAP FETCH result")?
1392                    else {
1393                        // No more FETCH responses received from the server.
1394                        break;
1395                    };
1396
1397                    if let Some(next_uid) = next_fetch_response.uid {
1398                        if next_uid == request_uid {
1399                            fetch_response = Some(next_fetch_response);
1400                        } else if !request_uids.contains(&next_uid) {
1401                            // (size of `request_uids` is bounded by IMAP command length limit,
1402                            // search in this vector is always fast)
1403
1404                            // Unwanted UIDs are possible because of unsolicited responses, e.g. if
1405                            // another client changes \Seen flag on a message after we do a prefetch but
1406                            // before fetch. It's not an error if we receive such unsolicited response.
1407                            info!(
1408                                context,
1409                                "Skipping not requested FETCH response for UID {}.", next_uid
1410                            );
1411                        } else if uid_msgs.insert(next_uid, next_fetch_response).is_some() {
1412                            warn!(context, "Got duplicated UID {}.", next_uid);
1413                        }
1414                    } else {
1415                        info!(context, "Skipping FETCH response without UID.");
1416                    }
1417                }
1418
1419                let fetch_response = match fetch_response {
1420                    Some(fetch) => fetch,
1421                    None => {
1422                        warn!(
1423                            context,
1424                            "Missed UID {} in the server response.", request_uid
1425                        );
1426                        continue;
1427                    }
1428                };
1429                count += 1;
1430
1431                let is_deleted = fetch_response.flags().any(|flag| flag == Flag::Deleted);
1432                let (body, partial) = if fetch_partially {
1433                    (fetch_response.header(), fetch_response.size) // `BODY.PEEK[HEADER]` goes to header() ...
1434                } else {
1435                    (fetch_response.body(), None) // ... while `BODY.PEEK[]` goes to body() - and includes header()
1436                };
1437
1438                if is_deleted {
1439                    info!(context, "Not processing deleted msg {}.", request_uid);
1440                    received_msgs_channel.send((request_uid, None)).await?;
1441                    continue;
1442                }
1443
1444                let body = if let Some(body) = body {
1445                    body
1446                } else {
1447                    info!(
1448                        context,
1449                        "Not processing message {} without a BODY.", request_uid
1450                    );
1451                    received_msgs_channel.send((request_uid, None)).await?;
1452                    continue;
1453                };
1454
1455                let is_seen = fetch_response.flags().any(|flag| flag == Flag::Seen);
1456
1457                let Some(rfc724_mid) = uid_message_ids.get(&request_uid) else {
1458                    error!(
1459                        context,
1460                        "No Message-ID corresponding to UID {} passed in uid_messsage_ids.",
1461                        request_uid
1462                    );
1463                    continue;
1464                };
1465
1466                info!(
1467                    context,
1468                    "Passing message UID {} to receive_imf().", request_uid
1469                );
1470                match receive_imf_inner(
1471                    context,
1472                    folder,
1473                    uidvalidity,
1474                    request_uid,
1475                    rfc724_mid,
1476                    body,
1477                    is_seen,
1478                    partial,
1479                )
1480                .await
1481                {
1482                    Ok(received_msg) => {
1483                        received_msgs_channel
1484                            .send((request_uid, received_msg))
1485                            .await?;
1486                    }
1487                    Err(err) => {
1488                        warn!(context, "receive_imf error: {:#}.", err);
1489                        received_msgs_channel.send((request_uid, None)).await?;
1490                    }
1491                };
1492            }
1493
1494            // If we don't process the whole response, IMAP client is left in a broken state where
1495            // it will try to process the rest of response as the next response.
1496            //
1497            // Make sure to not ignore the errors, because
1498            // if connection times out, it will return
1499            // infinite stream of `Some(Err(_))` results.
1500            while fetch_responses
1501                .try_next()
1502                .await
1503                .context("Failed to drain FETCH responses")?
1504                .is_some()
1505            {}
1506
1507            if count != request_uids.len() {
1508                warn!(
1509                    context,
1510                    "Failed to fetch all UIDs: got {}, requested {}, we requested the UIDs {:?}.",
1511                    count,
1512                    request_uids.len(),
1513                    request_uids,
1514                );
1515            } else {
1516                info!(
1517                    context,
1518                    "Successfully received {} UIDs.",
1519                    request_uids.len()
1520                );
1521            }
1522        }
1523
1524        Ok(())
1525    }
1526
1527    /// Retrieves server metadata if it is supported.
1528    ///
1529    /// We get [`/shared/comment`](https://www.rfc-editor.org/rfc/rfc5464#section-6.2.1)
1530    /// and [`/shared/admin`](https://www.rfc-editor.org/rfc/rfc5464#section-6.2.2)
1531    /// metadata.
1532    pub(crate) async fn fetch_metadata(&mut self, context: &Context) -> Result<()> {
1533        if !self.can_metadata() {
1534            return Ok(());
1535        }
1536
1537        let mut lock = context.metadata.write().await;
1538        if (*lock).is_some() {
1539            return Ok(());
1540        }
1541
1542        info!(
1543            context,
1544            "Server supports metadata, retrieving server comment and admin contact."
1545        );
1546
1547        let mut comment = None;
1548        let mut admin = None;
1549        let mut iroh_relay = None;
1550
1551        let mailbox = "";
1552        let options = "";
1553        let metadata = self
1554            .get_metadata(
1555                mailbox,
1556                options,
1557                "(/shared/comment /shared/admin /shared/vendor/deltachat/irohrelay)",
1558            )
1559            .await?;
1560        for m in metadata {
1561            match m.entry.as_ref() {
1562                "/shared/comment" => {
1563                    comment = m.value;
1564                }
1565                "/shared/admin" => {
1566                    admin = m.value;
1567                }
1568                "/shared/vendor/deltachat/irohrelay" => {
1569                    if let Some(value) = m.value {
1570                        if let Ok(url) = Url::parse(&value) {
1571                            iroh_relay = Some(url);
1572                        } else {
1573                            warn!(
1574                                context,
1575                                "Got invalid URL from iroh relay metadata: {:?}.", value
1576                            );
1577                        }
1578                    }
1579                }
1580                _ => {}
1581            }
1582        }
1583        *lock = Some(ServerMetadata {
1584            comment,
1585            admin,
1586            iroh_relay,
1587        });
1588        Ok(())
1589    }
1590
1591    /// Stores device token into /private/devicetoken IMAP METADATA of the Inbox.
1592    pub(crate) async fn register_token(&mut self, context: &Context) -> Result<()> {
1593        if context.push_subscribed.load(Ordering::Relaxed) {
1594            return Ok(());
1595        }
1596
1597        let Some(device_token) = context.push_subscriber.device_token().await else {
1598            return Ok(());
1599        };
1600
1601        if self.can_metadata() && self.can_push() {
1602            let old_encrypted_device_token =
1603                context.get_config(Config::EncryptedDeviceToken).await?;
1604
1605            // Whether we need to update encrypted device token.
1606            let device_token_changed = old_encrypted_device_token.is_none()
1607                || context.get_config(Config::DeviceToken).await?.as_ref() != Some(&device_token);
1608
1609            let new_encrypted_device_token;
1610            if device_token_changed {
1611                let encrypted_device_token = encrypt_device_token(&device_token)
1612                    .context("Failed to encrypt device token")?;
1613
1614                // We expect that the server supporting `XDELTAPUSH` capability
1615                // has non-synchronizing literals support as well:
1616                // <https://www.rfc-editor.org/rfc/rfc7888>.
1617                let encrypted_device_token_len = encrypted_device_token.len();
1618
1619                // Store device token saved on the server
1620                // to prevent storing duplicate tokens.
1621                // The server cannot deduplicate on its own
1622                // because encryption gives a different
1623                // result each time.
1624                context
1625                    .set_config_internal(Config::DeviceToken, Some(&device_token))
1626                    .await?;
1627                context
1628                    .set_config_internal(
1629                        Config::EncryptedDeviceToken,
1630                        Some(&encrypted_device_token),
1631                    )
1632                    .await?;
1633
1634                if encrypted_device_token_len <= 4096 {
1635                    new_encrypted_device_token = Some(encrypted_device_token);
1636                } else {
1637                    // If Apple or Google (FCM) gives us a very large token,
1638                    // do not even try to give it to IMAP servers.
1639                    //
1640                    // Limit of 4096 is arbitrarily selected
1641                    // to be the same as required by LITERAL- IMAP extension.
1642                    //
1643                    // Dovecot supports LITERAL+ and non-synchronizing literals
1644                    // of any length, but there is no reason for tokens
1645                    // to be that large even after OpenPGP encryption.
1646                    warn!(context, "Device token is too long for LITERAL-, ignoring.");
1647                    new_encrypted_device_token = None;
1648                }
1649            } else {
1650                new_encrypted_device_token = old_encrypted_device_token;
1651            }
1652
1653            // Store new encrypted device token on the server
1654            // even if it is the same as the old one.
1655            if let Some(encrypted_device_token) = new_encrypted_device_token {
1656                let folder = context
1657                    .get_config(Config::ConfiguredInboxFolder)
1658                    .await?
1659                    .context("INBOX is not configured")?;
1660
1661                self.run_command_and_check_ok(&format_setmetadata(
1662                    &folder,
1663                    &encrypted_device_token,
1664                ))
1665                .await
1666                .context("SETMETADATA command failed")?;
1667
1668                context.push_subscribed.store(true, Ordering::Relaxed);
1669            }
1670        } else if !context.push_subscriber.heartbeat_subscribed().await {
1671            let context = context.clone();
1672            // Subscribe for heartbeat notifications.
1673            tokio::spawn(async move { context.push_subscriber.subscribe(&context).await });
1674        }
1675
1676        Ok(())
1677    }
1678}
1679
1680fn format_setmetadata(folder: &str, device_token: &str) -> String {
1681    let device_token_len = device_token.len();
1682    format!(
1683        "SETMETADATA \"{folder}\" (/private/devicetoken {{{device_token_len}+}}\r\n{device_token})"
1684    )
1685}
1686
1687impl Session {
1688    /// Returns success if we successfully set the flag or we otherwise
1689    /// think add_flag should not be retried: Disconnection during setting
1690    /// the flag, or other imap-errors, returns Ok as well.
1691    ///
1692    /// Returning error means that the operation can be retried.
1693    async fn add_flag_finalized_with_set(&mut self, uid_set: &str, flag: &str) -> Result<()> {
1694        if flag == "\\Deleted" {
1695            self.selected_folder_needs_expunge = true;
1696        }
1697        let query = format!("+FLAGS ({flag})");
1698        let mut responses = self
1699            .uid_store(uid_set, &query)
1700            .await
1701            .with_context(|| format!("IMAP failed to store: ({uid_set}, {query})"))?;
1702        while let Some(_response) = responses.try_next().await? {
1703            // Read all the responses
1704        }
1705        Ok(())
1706    }
1707
1708    /// Attempts to configure mvbox.
1709    ///
1710    /// Tries to find any folder examining `folders` in the order they go. If none is found, tries
1711    /// to create any folder in the same order. This method does not use LIST command to ensure that
1712    /// configuration works even if mailbox lookup is forbidden via Access Control List (see
1713    /// <https://datatracker.ietf.org/doc/html/rfc4314>).
1714    ///
1715    /// Returns first found or created folder name.
1716    async fn configure_mvbox<'a>(
1717        &mut self,
1718        context: &Context,
1719        folders: &[&'a str],
1720        create_mvbox: bool,
1721    ) -> Result<Option<&'a str>> {
1722        // Close currently selected folder if needed.
1723        // We are going to select folders using low-level EXAMINE operations below.
1724        self.maybe_close_folder(context).await?;
1725
1726        for folder in folders {
1727            info!(context, "Looking for MVBOX-folder \"{}\"...", &folder);
1728            let res = self.examine(&folder).await;
1729            if res.is_ok() {
1730                info!(
1731                    context,
1732                    "MVBOX-folder {:?} successfully selected, using it.", &folder
1733                );
1734                self.close().await?;
1735                // Before moving emails to the mvbox we need to remember its UIDVALIDITY, otherwise
1736                // emails moved before that wouldn't be fetched but considered "old" instead.
1737                let create = false;
1738                let folder_exists = self
1739                    .select_with_uidvalidity(context, folder, create)
1740                    .await?;
1741                ensure!(folder_exists, "No MVBOX folder {:?}??", &folder);
1742                return Ok(Some(folder));
1743            }
1744        }
1745
1746        if !create_mvbox {
1747            return Ok(None);
1748        }
1749        // Some servers require namespace-style folder names like "INBOX.DeltaChat", so we try all
1750        // the variants here.
1751        for folder in folders {
1752            match self
1753                .select_with_uidvalidity(context, folder, create_mvbox)
1754                .await
1755            {
1756                Ok(_) => {
1757                    info!(context, "MVBOX-folder {} created.", folder);
1758                    return Ok(Some(folder));
1759                }
1760                Err(err) => {
1761                    warn!(context, "Cannot create MVBOX-folder {:?}: {}", folder, err);
1762                }
1763            }
1764        }
1765        Ok(None)
1766    }
1767}
1768
1769impl Imap {
1770    pub(crate) async fn configure_folders(
1771        &mut self,
1772        context: &Context,
1773        session: &mut Session,
1774        create_mvbox: bool,
1775    ) -> Result<()> {
1776        let mut folders = session
1777            .list(Some(""), Some("*"))
1778            .await
1779            .context("list_folders failed")?;
1780        let mut delimiter = ".".to_string();
1781        let mut delimiter_is_default = true;
1782        let mut folder_configs = BTreeMap::new();
1783
1784        while let Some(folder) = folders.try_next().await? {
1785            info!(context, "Scanning folder: {:?}", folder);
1786
1787            // Update the delimiter iff there is a different one, but only once.
1788            if let Some(d) = folder.delimiter() {
1789                if delimiter_is_default && !d.is_empty() && delimiter != d {
1790                    delimiter = d.to_string();
1791                    delimiter_is_default = false;
1792                }
1793            }
1794
1795            let folder_meaning = get_folder_meaning_by_attrs(folder.attributes());
1796            let folder_name_meaning = get_folder_meaning_by_name(folder.name());
1797            if let Some(config) = folder_meaning.to_config() {
1798                // Always takes precedence
1799                folder_configs.insert(config, folder.name().to_string());
1800            } else if let Some(config) = folder_name_meaning.to_config() {
1801                // only set if none has been already set
1802                folder_configs
1803                    .entry(config)
1804                    .or_insert_with(|| folder.name().to_string());
1805            }
1806        }
1807        drop(folders);
1808
1809        info!(context, "Using \"{}\" as folder-delimiter.", delimiter);
1810
1811        let fallback_folder = format!("INBOX{delimiter}DeltaChat");
1812        let mvbox_folder = session
1813            .configure_mvbox(context, &["DeltaChat", &fallback_folder], create_mvbox)
1814            .await
1815            .context("failed to configure mvbox")?;
1816
1817        context
1818            .set_config_internal(Config::ConfiguredInboxFolder, Some("INBOX"))
1819            .await?;
1820        if let Some(mvbox_folder) = mvbox_folder {
1821            info!(context, "Setting MVBOX FOLDER TO {}", &mvbox_folder);
1822            context
1823                .set_config_internal(Config::ConfiguredMvboxFolder, Some(mvbox_folder))
1824                .await?;
1825        }
1826        for (config, name) in folder_configs {
1827            context.set_config_internal(config, Some(&name)).await?;
1828        }
1829        context
1830            .sql
1831            .set_raw_config_int(
1832                constants::DC_FOLDERS_CONFIGURED_KEY,
1833                constants::DC_FOLDERS_CONFIGURED_VERSION,
1834            )
1835            .await?;
1836
1837        info!(context, "FINISHED configuring IMAP-folders.");
1838        Ok(())
1839    }
1840}
1841
1842impl Session {
1843    /// Return whether the server sent an unsolicited EXISTS or FETCH response.
1844    ///
1845    /// Drains all responses from `session.unsolicited_responses` in the process.
1846    ///
1847    /// If this returns `true`, this means that new emails arrived
1848    /// or flags have been changed.
1849    /// In this case we may want to skip next IDLE and do a round
1850    /// of fetching new messages and synchronizing seen flags.
1851    fn drain_unsolicited_responses(&self, context: &Context) -> Result<bool> {
1852        use UnsolicitedResponse::*;
1853        use async_imap::imap_proto::Response;
1854        use async_imap::imap_proto::ResponseCode;
1855
1856        let folder = self.selected_folder.as_deref().unwrap_or_default();
1857        let mut should_refetch = false;
1858        while let Ok(response) = self.unsolicited_responses.try_recv() {
1859            match response {
1860                Exists(_) => {
1861                    info!(
1862                        context,
1863                        "Need to refetch {folder:?}, got unsolicited EXISTS {response:?}"
1864                    );
1865                    should_refetch = true;
1866                }
1867
1868                Expunge(_) | Recent(_) => {}
1869                Other(ref response_data) => {
1870                    match response_data.parsed() {
1871                        Response::Fetch { .. } => {
1872                            info!(
1873                                context,
1874                                "Need to refetch {folder:?}, got unsolicited FETCH {response:?}"
1875                            );
1876                            should_refetch = true;
1877                        }
1878
1879                        // We are not interested in the following responses and they are are
1880                        // sent quite frequently, so, we ignore them without logging them.
1881                        Response::Done {
1882                            code: Some(ResponseCode::CopyUid(_, _, _)),
1883                            ..
1884                        } => {}
1885
1886                        _ => {
1887                            info!(context, "{folder:?}: got unsolicited response {response:?}")
1888                        }
1889                    }
1890                }
1891                _ => {
1892                    info!(context, "{folder:?}: got unsolicited response {response:?}")
1893                }
1894            }
1895        }
1896        Ok(should_refetch)
1897    }
1898}
1899
1900async fn should_move_out_of_spam(
1901    context: &Context,
1902    headers: &[mailparse::MailHeader<'_>],
1903) -> Result<bool> {
1904    if headers.get_header_value(HeaderDef::ChatVersion).is_some() {
1905        // If this is a chat message (i.e. has a ChatVersion header), then this might be
1906        // a securejoin message. We can't find out at this point as we didn't prefetch
1907        // the SecureJoin header. So, we always move chat messages out of Spam.
1908        // Two possibilities to change this would be:
1909        // 1. Remove the `&& !context.is_spam_folder(folder).await?` check from
1910        // `fetch_new_messages()`, and then let `receive_imf()` check
1911        // if it's a spam message and should be hidden.
1912        // 2. Or add a flag to the ChatVersion header that this is a securejoin
1913        // request, and return `true` here only if the message has this flag.
1914        // `receive_imf()` can then check if the securejoin request is valid.
1915        return Ok(true);
1916    }
1917
1918    if let Some(msg) = get_prefetch_parent_message(context, headers).await? {
1919        if msg.chat_blocked != Blocked::Not {
1920            // Blocked or contact request message in the spam folder, leave it there.
1921            return Ok(false);
1922        }
1923    } else {
1924        let from = match mimeparser::get_from(headers) {
1925            Some(f) => f,
1926            None => return Ok(false),
1927        };
1928        // No chat found.
1929        let (from_id, blocked_contact, _origin) =
1930            match from_field_to_contact_id(context, &from, None, true, true)
1931                .await
1932                .context("from_field_to_contact_id")?
1933            {
1934                Some(res) => res,
1935                None => {
1936                    warn!(
1937                        context,
1938                        "Contact with From address {:?} cannot exist, not moving out of spam", from
1939                    );
1940                    return Ok(false);
1941                }
1942            };
1943        if blocked_contact {
1944            // Contact is blocked, leave the message in spam.
1945            return Ok(false);
1946        }
1947
1948        if let Some(chat_id_blocked) = ChatIdBlocked::lookup_by_contact(context, from_id).await? {
1949            if chat_id_blocked.blocked != Blocked::Not {
1950                return Ok(false);
1951            }
1952        } else if from_id != ContactId::SELF {
1953            // No chat with this contact found.
1954            return Ok(false);
1955        }
1956    }
1957
1958    Ok(true)
1959}
1960
1961/// Returns target folder for a message found in the Spam folder.
1962/// If this returns None, the message will not be moved out of the
1963/// Spam folder, and as `fetch_new_messages()` doesn't download
1964/// messages from the Spam folder, the message will be ignored.
1965async fn spam_target_folder_cfg(
1966    context: &Context,
1967    headers: &[mailparse::MailHeader<'_>],
1968) -> Result<Option<Config>> {
1969    if !should_move_out_of_spam(context, headers).await? {
1970        return Ok(None);
1971    }
1972
1973    if needs_move_to_mvbox(context, headers).await?
1974        // If OnlyFetchMvbox is set, we don't want to move the message to
1975        // the inbox or sentbox where we wouldn't fetch it again:
1976        || context.get_config_bool(Config::OnlyFetchMvbox).await?
1977    {
1978        Ok(Some(Config::ConfiguredMvboxFolder))
1979    } else {
1980        Ok(Some(Config::ConfiguredInboxFolder))
1981    }
1982}
1983
1984/// Returns `ConfiguredInboxFolder`, `ConfiguredMvboxFolder` or `ConfiguredSentboxFolder` if
1985/// the message needs to be moved from `folder`. Otherwise returns `None`.
1986pub async fn target_folder_cfg(
1987    context: &Context,
1988    folder: &str,
1989    folder_meaning: FolderMeaning,
1990    headers: &[mailparse::MailHeader<'_>],
1991) -> Result<Option<Config>> {
1992    if context.is_mvbox(folder).await? {
1993        return Ok(None);
1994    }
1995
1996    if folder_meaning == FolderMeaning::Spam {
1997        spam_target_folder_cfg(context, headers).await
1998    } else if needs_move_to_mvbox(context, headers).await? {
1999        Ok(Some(Config::ConfiguredMvboxFolder))
2000    } else {
2001        Ok(None)
2002    }
2003}
2004
2005pub async fn target_folder(
2006    context: &Context,
2007    folder: &str,
2008    folder_meaning: FolderMeaning,
2009    headers: &[mailparse::MailHeader<'_>],
2010) -> Result<String> {
2011    match target_folder_cfg(context, folder, folder_meaning, headers).await? {
2012        Some(config) => match context.get_config(config).await? {
2013            Some(target) => Ok(target),
2014            None => Ok(folder.to_string()),
2015        },
2016        None => Ok(folder.to_string()),
2017    }
2018}
2019
2020async fn needs_move_to_mvbox(
2021    context: &Context,
2022    headers: &[mailparse::MailHeader<'_>],
2023) -> Result<bool> {
2024    let has_chat_version = headers.get_header_value(HeaderDef::ChatVersion).is_some();
2025    if !context.get_config_bool(Config::IsChatmail).await?
2026        && has_chat_version
2027        && headers
2028            .get_header_value(HeaderDef::AutoSubmitted)
2029            .filter(|val| val.eq_ignore_ascii_case("auto-generated"))
2030            .is_some()
2031    {
2032        if let Some(from) = mimeparser::get_from(headers) {
2033            if context.is_self_addr(&from.addr).await? {
2034                return Ok(true);
2035            }
2036        }
2037    }
2038    if !context.get_config_bool(Config::MvboxMove).await? {
2039        return Ok(false);
2040    }
2041
2042    if headers
2043        .get_header_value(HeaderDef::AutocryptSetupMessage)
2044        .is_some()
2045    {
2046        // do not move setup messages;
2047        // there may be a non-delta device that wants to handle it
2048        return Ok(false);
2049    }
2050
2051    if has_chat_version {
2052        Ok(true)
2053    } else if let Some(parent) = get_prefetch_parent_message(context, headers).await? {
2054        match parent.is_dc_message {
2055            MessengerMessage::No => Ok(false),
2056            MessengerMessage::Yes | MessengerMessage::Reply => Ok(true),
2057        }
2058    } else {
2059        Ok(false)
2060    }
2061}
2062
2063/// Try to get the folder meaning by the name of the folder only used if the server does not support XLIST.
2064// TODO: lots languages missing - maybe there is a list somewhere on other MUAs?
2065// however, if we fail to find out the sent-folder,
2066// only watching this folder is not working. at least, this is no show stopper.
2067// CAVE: if possible, take care not to add a name here that is "sent" in one language
2068// but sth. different in others - a hard job.
2069fn get_folder_meaning_by_name(folder_name: &str) -> FolderMeaning {
2070    // source: <https://stackoverflow.com/questions/2185391/localized-gmail-imap-folders>
2071    const SENT_NAMES: &[&str] = &[
2072        "sent",
2073        "sentmail",
2074        "sent objects",
2075        "gesendet",
2076        "Sent Mail",
2077        "Sendte e-mails",
2078        "Enviados",
2079        "Messages envoyés",
2080        "Messages envoyes",
2081        "Posta inviata",
2082        "Verzonden berichten",
2083        "Wyslane",
2084        "E-mails enviados",
2085        "Correio enviado",
2086        "Enviada",
2087        "Enviado",
2088        "Gönderildi",
2089        "Inviati",
2090        "Odeslaná pošta",
2091        "Sendt",
2092        "Skickat",
2093        "Verzonden",
2094        "Wysłane",
2095        "Éléments envoyés",
2096        "Απεσταλμένα",
2097        "Отправленные",
2098        "寄件備份",
2099        "已发送邮件",
2100        "送信済み",
2101        "보낸편지함",
2102    ];
2103    const SPAM_NAMES: &[&str] = &[
2104        "spam",
2105        "junk",
2106        "Correio electrónico não solicitado",
2107        "Correo basura",
2108        "Lixo",
2109        "Nettsøppel",
2110        "Nevyžádaná pošta",
2111        "No solicitado",
2112        "Ongewenst",
2113        "Posta indesiderata",
2114        "Skräp",
2115        "Wiadomości-śmieci",
2116        "Önemsiz",
2117        "Ανεπιθύμητα",
2118        "Спам",
2119        "垃圾邮件",
2120        "垃圾郵件",
2121        "迷惑メール",
2122        "스팸",
2123    ];
2124    const TRASH_NAMES: &[&str] = &[
2125        "Trash",
2126        "Bin",
2127        "Caixote do lixo",
2128        "Cestino",
2129        "Corbeille",
2130        "Papelera",
2131        "Papierkorb",
2132        "Papirkurv",
2133        "Papperskorgen",
2134        "Prullenbak",
2135        "Rubujo",
2136        "Κάδος απορριμμάτων",
2137        "Корзина",
2138        "Кошик",
2139        "ゴミ箱",
2140        "垃圾桶",
2141        "已删除邮件",
2142        "휴지통",
2143    ];
2144    let lower = folder_name.to_lowercase();
2145
2146    if SENT_NAMES.iter().any(|s| s.to_lowercase() == lower) {
2147        FolderMeaning::Sent
2148    } else if SPAM_NAMES.iter().any(|s| s.to_lowercase() == lower) {
2149        FolderMeaning::Spam
2150    } else if TRASH_NAMES.iter().any(|s| s.to_lowercase() == lower) {
2151        FolderMeaning::Trash
2152    } else {
2153        FolderMeaning::Unknown
2154    }
2155}
2156
2157fn get_folder_meaning_by_attrs(folder_attrs: &[NameAttribute]) -> FolderMeaning {
2158    for attr in folder_attrs {
2159        match attr {
2160            NameAttribute::Trash => return FolderMeaning::Trash,
2161            NameAttribute::Sent => return FolderMeaning::Sent,
2162            NameAttribute::Junk => return FolderMeaning::Spam,
2163            NameAttribute::All | NameAttribute::Flagged => return FolderMeaning::Virtual,
2164            NameAttribute::Extension(label) => {
2165                match label.as_ref() {
2166                    "\\Spam" => return FolderMeaning::Spam,
2167                    "\\Important" => return FolderMeaning::Virtual,
2168                    _ => {}
2169                };
2170            }
2171            _ => {}
2172        }
2173    }
2174    FolderMeaning::Unknown
2175}
2176
2177pub(crate) fn get_folder_meaning(folder: &Name) -> FolderMeaning {
2178    match get_folder_meaning_by_attrs(folder.attributes()) {
2179        FolderMeaning::Unknown => get_folder_meaning_by_name(folder.name()),
2180        meaning => meaning,
2181    }
2182}
2183
2184/// Parses the headers from the FETCH result.
2185fn get_fetch_headers(prefetch_msg: &Fetch) -> Result<Vec<mailparse::MailHeader<'_>>> {
2186    match prefetch_msg.header() {
2187        Some(header_bytes) => {
2188            let (headers, _) = mailparse::parse_headers(header_bytes)?;
2189            Ok(headers)
2190        }
2191        None => Ok(Vec::new()),
2192    }
2193}
2194
2195pub(crate) fn prefetch_get_message_id(headers: &[mailparse::MailHeader]) -> Option<String> {
2196    headers
2197        .get_header_value(HeaderDef::XMicrosoftOriginalMessageId)
2198        .or_else(|| headers.get_header_value(HeaderDef::MessageId))
2199        .and_then(|msgid| mimeparser::parse_message_id(&msgid).ok())
2200}
2201
2202pub(crate) fn create_message_id() -> String {
2203    format!("{}{}", GENERATED_PREFIX, create_id())
2204}
2205
2206/// Returns chat by prefetched headers.
2207async fn prefetch_get_chat(
2208    context: &Context,
2209    headers: &[mailparse::MailHeader<'_>],
2210) -> Result<Option<chat::Chat>> {
2211    let parent = get_prefetch_parent_message(context, headers).await?;
2212    if let Some(parent) = &parent {
2213        return Ok(Some(
2214            chat::Chat::load_from_db(context, parent.get_chat_id()).await?,
2215        ));
2216    }
2217
2218    Ok(None)
2219}
2220
2221/// Determines whether the message should be downloaded based on prefetched headers.
2222pub(crate) async fn prefetch_should_download(
2223    context: &Context,
2224    headers: &[mailparse::MailHeader<'_>],
2225    message_id: &str,
2226    mut flags: impl Iterator<Item = Flag<'_>>,
2227) -> Result<bool> {
2228    if message::rfc724_mid_exists(context, message_id)
2229        .await?
2230        .is_some()
2231    {
2232        markseen_on_imap_table(context, message_id).await?;
2233        return Ok(false);
2234    }
2235
2236    // We do not know the Message-ID or the Message-ID is missing (in this case, we create one in
2237    // the further process).
2238
2239    if let Some(chat) = prefetch_get_chat(context, headers).await? {
2240        if chat.typ == Chattype::Group && !chat.id.is_special() {
2241            // This might be a group command, like removing a group member.
2242            // We really need to fetch this to avoid inconsistent group state.
2243            return Ok(true);
2244        }
2245    }
2246
2247    let maybe_ndn = if let Some(from) = headers.get_header_value(HeaderDef::From_) {
2248        let from = from.to_ascii_lowercase();
2249        from.contains("mailer-daemon") || from.contains("mail-daemon")
2250    } else {
2251        false
2252    };
2253
2254    // Autocrypt Setup Message should be shown even if it is from non-chat client.
2255    let is_autocrypt_setup_message = headers
2256        .get_header_value(HeaderDef::AutocryptSetupMessage)
2257        .is_some();
2258
2259    let from = match mimeparser::get_from(headers) {
2260        Some(f) => f,
2261        None => return Ok(false),
2262    };
2263    let (_from_id, blocked_contact, origin) =
2264        match from_field_to_contact_id(context, &from, None, true, true).await? {
2265            Some(res) => res,
2266            None => return Ok(false),
2267        };
2268    // prevent_rename=true as this might be a mailing list message and in this case it would be bad if we rename the contact.
2269    // (prevent_rename is the last argument of from_field_to_contact_id())
2270
2271    if flags.any(|f| f == Flag::Draft) {
2272        info!(context, "Ignoring draft message");
2273        return Ok(false);
2274    }
2275
2276    let is_chat_message = headers.get_header_value(HeaderDef::ChatVersion).is_some();
2277    let accepted_contact = origin.is_known();
2278    let is_reply_to_chat_message = get_prefetch_parent_message(context, headers)
2279        .await?
2280        .map(|parent| match parent.is_dc_message {
2281            MessengerMessage::No => false,
2282            MessengerMessage::Yes | MessengerMessage::Reply => true,
2283        })
2284        .unwrap_or_default();
2285
2286    let show_emails =
2287        ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await?).unwrap_or_default();
2288
2289    let show = is_autocrypt_setup_message
2290        || match show_emails {
2291            ShowEmails::Off => is_chat_message || is_reply_to_chat_message,
2292            ShowEmails::AcceptedContacts => {
2293                is_chat_message || is_reply_to_chat_message || accepted_contact
2294            }
2295            ShowEmails::All => true,
2296        };
2297
2298    let should_download = (show && !blocked_contact) || maybe_ndn;
2299    Ok(should_download)
2300}
2301
2302/// Returns whether a message is a duplicate (resent message).
2303pub(crate) fn is_dup_msg(is_chat_msg: bool, ts_sent: i64, ts_sent_old: i64) -> bool {
2304    // If the existing message has timestamp_sent == 0, that means we don't know its actual sent
2305    // timestamp, so don't delete the new message. E.g. outgoing messages have zero timestamp_sent
2306    // because they are stored to the db before sending. Also consider as duplicates only messages
2307    // with greater timestamp to avoid deleting both messages in a multi-device setting.
2308    is_chat_msg && ts_sent_old != 0 && ts_sent > ts_sent_old
2309}
2310
2311/// Marks messages in `msgs` table as seen, searching for them by UID.
2312///
2313/// Returns updated chat ID if any message was marked as seen.
2314async fn mark_seen_by_uid(
2315    context: &Context,
2316    folder: &str,
2317    uid_validity: u32,
2318    uid: u32,
2319) -> Result<Option<ChatId>> {
2320    if let Some((msg_id, chat_id)) = context
2321        .sql
2322        .query_row_optional(
2323            "SELECT id, chat_id FROM msgs
2324                 WHERE id > 9 AND rfc724_mid IN (
2325                   SELECT rfc724_mid FROM imap
2326                   WHERE folder=?1
2327                   AND uidvalidity=?2
2328                   AND uid=?3
2329                   LIMIT 1
2330                 )",
2331            (&folder, uid_validity, uid),
2332            |row| {
2333                let msg_id: MsgId = row.get(0)?;
2334                let chat_id: ChatId = row.get(1)?;
2335                Ok((msg_id, chat_id))
2336            },
2337        )
2338        .await
2339        .with_context(|| format!("failed to get msg and chat ID for IMAP message {folder}/{uid}"))?
2340    {
2341        let updated = context
2342            .sql
2343            .execute(
2344                "UPDATE msgs SET state=?1
2345                     WHERE (state=?2 OR state=?3)
2346                     AND id=?4",
2347                (
2348                    MessageState::InSeen,
2349                    MessageState::InFresh,
2350                    MessageState::InNoticed,
2351                    msg_id,
2352                ),
2353            )
2354            .await
2355            .with_context(|| format!("failed to update msg {msg_id} state"))?
2356            > 0;
2357
2358        if updated {
2359            msg_id
2360                .start_ephemeral_timer(context)
2361                .await
2362                .with_context(|| format!("failed to start ephemeral timer for message {msg_id}"))?;
2363            Ok(Some(chat_id))
2364        } else {
2365            // Message state has not changed.
2366            Ok(None)
2367        }
2368    } else {
2369        // There is no message is `msgs` table matching the given UID.
2370        Ok(None)
2371    }
2372}
2373
2374/// Schedule marking the message as Seen on IMAP by adding all known IMAP messages corresponding to
2375/// the given Message-ID to `imap_markseen` table.
2376pub(crate) async fn markseen_on_imap_table(context: &Context, message_id: &str) -> Result<()> {
2377    context
2378        .sql
2379        .execute(
2380            "INSERT OR IGNORE INTO imap_markseen (id)
2381             SELECT id FROM imap WHERE rfc724_mid=?",
2382            (message_id,),
2383        )
2384        .await?;
2385    context.scheduler.interrupt_inbox().await;
2386
2387    Ok(())
2388}
2389
2390/// uid_next is the next unique identifier value from the last time we fetched a folder
2391/// See <https://tools.ietf.org/html/rfc3501#section-2.3.1.1>
2392/// This function is used to update our uid_next after fetching messages.
2393pub(crate) async fn set_uid_next(context: &Context, folder: &str, uid_next: u32) -> Result<()> {
2394    context
2395        .sql
2396        .execute(
2397            "INSERT INTO imap_sync (folder, uid_next) VALUES (?,?)
2398                ON CONFLICT(folder) DO UPDATE SET uid_next=excluded.uid_next",
2399            (folder, uid_next),
2400        )
2401        .await?;
2402    Ok(())
2403}
2404
2405/// uid_next is the next unique identifier value from the last time we fetched a folder
2406/// See <https://tools.ietf.org/html/rfc3501#section-2.3.1.1>
2407/// This method returns the uid_next from the last time we fetched messages.
2408/// We can compare this to the current uid_next to find out whether there are new messages
2409/// and fetch from this value on to get all new messages.
2410async fn get_uid_next(context: &Context, folder: &str) -> Result<u32> {
2411    Ok(context
2412        .sql
2413        .query_get_value("SELECT uid_next FROM imap_sync WHERE folder=?;", (folder,))
2414        .await?
2415        .unwrap_or(0))
2416}
2417
2418pub(crate) async fn set_uidvalidity(
2419    context: &Context,
2420    folder: &str,
2421    uidvalidity: u32,
2422) -> Result<()> {
2423    context
2424        .sql
2425        .execute(
2426            "INSERT INTO imap_sync (folder, uidvalidity) VALUES (?,?)
2427                ON CONFLICT(folder) DO UPDATE SET uidvalidity=excluded.uidvalidity",
2428            (folder, uidvalidity),
2429        )
2430        .await?;
2431    Ok(())
2432}
2433
2434async fn get_uidvalidity(context: &Context, folder: &str) -> Result<u32> {
2435    Ok(context
2436        .sql
2437        .query_get_value(
2438            "SELECT uidvalidity FROM imap_sync WHERE folder=?;",
2439            (folder,),
2440        )
2441        .await?
2442        .unwrap_or(0))
2443}
2444
2445pub(crate) async fn set_modseq(context: &Context, folder: &str, modseq: u64) -> Result<()> {
2446    context
2447        .sql
2448        .execute(
2449            "INSERT INTO imap_sync (folder, modseq) VALUES (?,?)
2450                ON CONFLICT(folder) DO UPDATE SET modseq=excluded.modseq",
2451            (folder, modseq),
2452        )
2453        .await?;
2454    Ok(())
2455}
2456
2457async fn get_modseq(context: &Context, folder: &str) -> Result<u64> {
2458    Ok(context
2459        .sql
2460        .query_get_value("SELECT modseq FROM imap_sync WHERE folder=?;", (folder,))
2461        .await?
2462        .unwrap_or(0))
2463}
2464
2465/// Compute the imap search expression for all self-sent mails (for all self addresses)
2466pub(crate) async fn get_imap_self_sent_search_command(context: &Context) -> Result<String> {
2467    // See https://www.rfc-editor.org/rfc/rfc3501#section-6.4.4 for syntax of SEARCH and OR
2468    let mut search_command = format!("FROM \"{}\"", context.get_primary_self_addr().await?);
2469
2470    for item in context.get_secondary_self_addrs().await? {
2471        search_command = format!("OR ({search_command}) (FROM \"{item}\")");
2472    }
2473
2474    Ok(search_command)
2475}
2476
2477/// Deprecated, use get_uid_next() and get_uidvalidity()
2478pub async fn get_config_last_seen_uid(context: &Context, folder: &str) -> Result<(u32, u32)> {
2479    let key = format!("imap.mailbox.{folder}");
2480    if let Some(entry) = context.sql.get_raw_config(&key).await? {
2481        // the entry has the format `imap.mailbox.<folder>=<uidvalidity>:<lastseenuid>`
2482        let mut parts = entry.split(':');
2483        Ok((
2484            parts.next().unwrap_or_default().parse().unwrap_or(0),
2485            parts.next().unwrap_or_default().parse().unwrap_or(0),
2486        ))
2487    } else {
2488        Ok((0, 0))
2489    }
2490}
2491
2492/// Whether to ignore fetching messages from a folder.
2493///
2494/// This caters for the [`Config::OnlyFetchMvbox`] setting which means mails from folders
2495/// not explicitly watched should not be fetched.
2496async fn should_ignore_folder(
2497    context: &Context,
2498    folder: &str,
2499    folder_meaning: FolderMeaning,
2500) -> Result<bool> {
2501    if !context.get_config_bool(Config::OnlyFetchMvbox).await? {
2502        return Ok(false);
2503    }
2504    if context.is_sentbox(folder).await? {
2505        // Still respect the SentboxWatch setting.
2506        return Ok(!context.get_config_bool(Config::SentboxWatch).await?);
2507    }
2508    Ok(!(context.is_mvbox(folder).await? || folder_meaning == FolderMeaning::Spam))
2509}
2510
2511/// Builds a list of sequence/uid sets. The returned sets have each no more than around 1000
2512/// characters because according to <https://tools.ietf.org/html/rfc2683#section-3.2.1.5>
2513/// command lines should not be much more than 1000 chars (servers should allow at least 8000 chars)
2514fn build_sequence_sets(uids: &[u32]) -> Result<Vec<(Vec<u32>, String)>> {
2515    // first, try to find consecutive ranges:
2516    let mut ranges: Vec<UidRange> = vec![];
2517
2518    for &current in uids {
2519        if let Some(last) = ranges.last_mut() {
2520            if last.end + 1 == current {
2521                last.end = current;
2522                continue;
2523            }
2524        }
2525
2526        ranges.push(UidRange {
2527            start: current,
2528            end: current,
2529        });
2530    }
2531
2532    // Second, sort the uids into uid sets that are each below ~1000 characters
2533    let mut result = vec![];
2534    let (mut last_uids, mut last_str) = (Vec::new(), String::new());
2535    for range in ranges {
2536        last_uids.reserve((range.end - range.start + 1).try_into()?);
2537        (range.start..=range.end).for_each(|u| last_uids.push(u));
2538        if !last_str.is_empty() {
2539            last_str.push(',');
2540        }
2541        last_str.push_str(&range.to_string());
2542
2543        if last_str.len() > 990 {
2544            result.push((take(&mut last_uids), take(&mut last_str)));
2545        }
2546    }
2547    result.push((last_uids, last_str));
2548
2549    result.retain(|(_, s)| !s.is_empty());
2550    Ok(result)
2551}
2552
2553struct UidRange {
2554    start: u32,
2555    end: u32,
2556    // If start == end, then this range represents a single number
2557}
2558
2559impl std::fmt::Display for UidRange {
2560    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
2561        if self.start == self.end {
2562            write!(f, "{}", self.start)
2563        } else {
2564            write!(f, "{}:{}", self.start, self.end)
2565        }
2566    }
2567}
2568async fn add_all_recipients_as_contacts(
2569    context: &Context,
2570    session: &mut Session,
2571    folder: Config,
2572) -> Result<()> {
2573    let mailbox = if let Some(m) = context.get_config(folder).await? {
2574        m
2575    } else {
2576        info!(
2577            context,
2578            "Folder {} is not configured, skipping fetching contacts from it.", folder
2579        );
2580        return Ok(());
2581    };
2582    let create = false;
2583    let folder_exists = session
2584        .select_with_uidvalidity(context, &mailbox, create)
2585        .await
2586        .with_context(|| format!("could not select {mailbox}"))?;
2587    if !folder_exists {
2588        return Ok(());
2589    }
2590
2591    let recipients = session
2592        .get_all_recipients(context)
2593        .await
2594        .context("could not get recipients")?;
2595
2596    let mut any_modified = false;
2597    for recipient in recipients {
2598        let recipient_addr = match ContactAddress::new(&recipient.addr) {
2599            Err(err) => {
2600                warn!(
2601                    context,
2602                    "Could not add contact for recipient with address {:?}: {:#}",
2603                    recipient.addr,
2604                    err
2605                );
2606                continue;
2607            }
2608            Ok(recipient_addr) => recipient_addr,
2609        };
2610
2611        let (_, modified) = Contact::add_or_lookup(
2612            context,
2613            &recipient.display_name.unwrap_or_default(),
2614            &recipient_addr,
2615            Origin::OutgoingTo,
2616        )
2617        .await?;
2618        if modified != Modifier::None {
2619            any_modified = true;
2620        }
2621    }
2622    if any_modified {
2623        context.emit_event(EventType::ContactsChanged(None));
2624    }
2625
2626    Ok(())
2627}
2628
2629#[cfg(test)]
2630mod imap_tests;