deltachat/
imap.rs

1//! # IMAP handling module.
2//!
3//! uses [async-email/async-imap](https://github.com/async-email/async-imap)
4//! to implement connect, fetch, delete functionality with standard IMAP servers.
5
6use std::{
7    cmp::max,
8    cmp::min,
9    collections::{BTreeMap, BTreeSet, HashMap},
10    iter::Peekable,
11    mem::take,
12    sync::atomic::Ordering,
13    time::{Duration, UNIX_EPOCH},
14};
15
16use anyhow::{bail, ensure, format_err, Context as _, Result};
17use async_channel::Receiver;
18use async_imap::types::{Fetch, Flag, Name, NameAttribute, UnsolicitedResponse};
19use deltachat_contact_tools::ContactAddress;
20use futures::{FutureExt as _, StreamExt, TryStreamExt};
21use futures_lite::FutureExt;
22use num_traits::FromPrimitive;
23use rand::Rng;
24use ratelimit::Ratelimit;
25use url::Url;
26
27use crate::chat::{self, ChatId, ChatIdBlocked};
28use crate::chatlist_events;
29use crate::config::Config;
30use crate::constants::{self, Blocked, Chattype, ShowEmails};
31use crate::contact::{Contact, ContactId, Modifier, Origin};
32use crate::context::Context;
33use crate::events::EventType;
34use crate::headerdef::{HeaderDef, HeaderDefMap};
35use crate::log::LogExt;
36use crate::login_param::{
37    prioritize_server_login_params, ConfiguredLoginParam, ConfiguredServerLoginParam,
38};
39use crate::message::{self, Message, MessageState, MessengerMessage, MsgId};
40use crate::mimeparser;
41use crate::net::proxy::ProxyConfig;
42use crate::net::session::SessionStream;
43use crate::oauth2::get_oauth2_access_token;
44use crate::push::encrypt_device_token;
45use crate::receive_imf::{
46    from_field_to_contact_id, get_prefetch_parent_message, receive_imf_inner, ReceivedMsg,
47};
48use crate::scheduler::connectivity::ConnectivityStore;
49use crate::stock_str;
50use crate::tools::{self, create_id, duration_to_str};
51
52pub(crate) mod capabilities;
53mod client;
54mod idle;
55pub mod scan_folders;
56pub mod select_folder;
57pub(crate) mod session;
58
59use client::{determine_capabilities, Client};
60use mailparse::SingleInfo;
61use session::Session;
62
63pub(crate) const GENERATED_PREFIX: &str = "GEN_";
64
65const RFC724MID_UID: &str = "(UID BODY.PEEK[HEADER.FIELDS (\
66                             MESSAGE-ID \
67                             X-MICROSOFT-ORIGINAL-MESSAGE-ID\
68                             )])";
69const BODY_FULL: &str = "(FLAGS BODY.PEEK[])";
70const BODY_PARTIAL: &str = "(FLAGS RFC822.SIZE BODY.PEEK[HEADER])";
71
72#[derive(Debug)]
73pub(crate) struct Imap {
74    pub(crate) idle_interrupt_receiver: Receiver<()>,
75
76    /// Email address.
77    addr: String,
78
79    /// Login parameters.
80    lp: Vec<ConfiguredServerLoginParam>,
81
82    /// Password.
83    password: String,
84
85    /// Proxy configuration.
86    proxy_config: Option<ProxyConfig>,
87
88    strict_tls: bool,
89
90    oauth2: bool,
91
92    authentication_failed_once: bool,
93
94    pub(crate) connectivity: ConnectivityStore,
95
96    conn_last_try: tools::Time,
97    conn_backoff_ms: u64,
98
99    /// Rate limit for successful IMAP connections.
100    ///
101    /// This rate limit prevents busy loop in case the server refuses logins
102    /// or in case connection gets dropped over and over due to IMAP bug,
103    /// e.g. the server returning invalid response to SELECT command
104    /// immediately after logging in or returning an error in response to LOGIN command
105    /// due to internal server error.
106    ratelimit: Ratelimit,
107}
108
109#[derive(Debug)]
110struct OAuth2 {
111    user: String,
112    access_token: String,
113}
114
115#[derive(Debug)]
116pub(crate) struct ServerMetadata {
117    /// IMAP METADATA `/shared/comment` as defined in
118    /// <https://www.rfc-editor.org/rfc/rfc5464#section-6.2.1>.
119    pub comment: Option<String>,
120
121    /// IMAP METADATA `/shared/admin` as defined in
122    /// <https://www.rfc-editor.org/rfc/rfc5464#section-6.2.2>.
123    pub admin: Option<String>,
124
125    pub iroh_relay: Option<Url>,
126}
127
128impl async_imap::Authenticator for OAuth2 {
129    type Response = String;
130
131    fn process(&mut self, _data: &[u8]) -> Self::Response {
132        format!(
133            "user={}\x01auth=Bearer {}\x01\x01",
134            self.user, self.access_token
135        )
136    }
137}
138
139#[derive(Debug, Display, PartialEq, Eq, Clone, Copy)]
140pub enum FolderMeaning {
141    Unknown,
142
143    /// Spam folder.
144    Spam,
145    Inbox,
146    Mvbox,
147    Sent,
148    Trash,
149    Drafts,
150
151    /// Virtual folders.
152    ///
153    /// On Gmail there are virtual folders marked as \\All, \\Important and \\Flagged.
154    /// Delta Chat ignores these folders because the same messages can be fetched
155    /// from the real folder and the result of moving and deleting messages via
156    /// virtual folder is unclear.
157    Virtual,
158}
159
160impl FolderMeaning {
161    pub fn to_config(self) -> Option<Config> {
162        match self {
163            FolderMeaning::Unknown => None,
164            FolderMeaning::Spam => None,
165            FolderMeaning::Inbox => Some(Config::ConfiguredInboxFolder),
166            FolderMeaning::Mvbox => Some(Config::ConfiguredMvboxFolder),
167            FolderMeaning::Sent => Some(Config::ConfiguredSentboxFolder),
168            FolderMeaning::Trash => Some(Config::ConfiguredTrashFolder),
169            FolderMeaning::Drafts => None,
170            FolderMeaning::Virtual => None,
171        }
172    }
173}
174
175struct UidGrouper<T: Iterator<Item = (i64, u32, String)>> {
176    inner: Peekable<T>,
177}
178
179impl<T, I> From<I> for UidGrouper<T>
180where
181    T: Iterator<Item = (i64, u32, String)>,
182    I: IntoIterator<IntoIter = T>,
183{
184    fn from(inner: I) -> Self {
185        Self {
186            inner: inner.into_iter().peekable(),
187        }
188    }
189}
190
191impl<T: Iterator<Item = (i64, u32, String)>> Iterator for UidGrouper<T> {
192    // Tuple of folder, row IDs, and UID range as a string.
193    type Item = (String, Vec<i64>, String);
194
195    fn next(&mut self) -> Option<Self::Item> {
196        let (_, _, folder) = self.inner.peek().cloned()?;
197
198        let mut uid_set = String::new();
199        let mut rowid_set = Vec::new();
200
201        while uid_set.len() < 1000 {
202            // Construct a new range.
203            if let Some((start_rowid, start_uid, _)) = self
204                .inner
205                .next_if(|(_, _, start_folder)| start_folder == &folder)
206            {
207                rowid_set.push(start_rowid);
208                let mut end_uid = start_uid;
209
210                while let Some((next_rowid, next_uid, _)) =
211                    self.inner.next_if(|(_, next_uid, next_folder)| {
212                        next_folder == &folder && (*next_uid == end_uid + 1 || *next_uid == end_uid)
213                    })
214                {
215                    end_uid = next_uid;
216                    rowid_set.push(next_rowid);
217                }
218
219                let uid_range = UidRange {
220                    start: start_uid,
221                    end: end_uid,
222                };
223                if !uid_set.is_empty() {
224                    uid_set.push(',');
225                }
226                uid_set.push_str(&uid_range.to_string());
227            } else {
228                break;
229            }
230        }
231
232        Some((folder, rowid_set, uid_set))
233    }
234}
235
236impl Imap {
237    /// Creates new disconnected IMAP client using the specific login parameters.
238    ///
239    /// `addr` is used to renew token if OAuth2 authentication is used.
240    pub fn new(
241        lp: Vec<ConfiguredServerLoginParam>,
242        password: String,
243        proxy_config: Option<ProxyConfig>,
244        addr: &str,
245        strict_tls: bool,
246        oauth2: bool,
247        idle_interrupt_receiver: Receiver<()>,
248    ) -> Self {
249        Imap {
250            idle_interrupt_receiver,
251            addr: addr.to_string(),
252            lp,
253            password,
254            proxy_config,
255            strict_tls,
256            oauth2,
257            authentication_failed_once: false,
258            connectivity: Default::default(),
259            conn_last_try: UNIX_EPOCH,
260            conn_backoff_ms: 0,
261            // 1 connection per minute + a burst of 2.
262            ratelimit: Ratelimit::new(Duration::new(120, 0), 2.0),
263        }
264    }
265
266    /// Creates new disconnected IMAP client using configured parameters.
267    pub async fn new_configured(
268        context: &Context,
269        idle_interrupt_receiver: Receiver<()>,
270    ) -> Result<Self> {
271        let param = ConfiguredLoginParam::load(context)
272            .await?
273            .context("Not configured")?;
274        let proxy_config = ProxyConfig::load(context).await?;
275        let strict_tls = param.strict_tls(proxy_config.is_some());
276        let imap = Self::new(
277            param.imap.clone(),
278            param.imap_password.clone(),
279            proxy_config,
280            &param.addr,
281            strict_tls,
282            param.oauth2,
283            idle_interrupt_receiver,
284        );
285        Ok(imap)
286    }
287
288    /// Connects to IMAP server and returns a new IMAP session.
289    ///
290    /// Calling this function is not enough to perform IMAP operations. Use [`Imap::prepare`]
291    /// instead if you are going to actually use connection rather than trying connection
292    /// parameters.
293    pub(crate) async fn connect(
294        &mut self,
295        context: &Context,
296        configuring: bool,
297    ) -> Result<Session> {
298        let now = tools::Time::now();
299        let until_can_send = max(
300            min(self.conn_last_try, now)
301                .checked_add(Duration::from_millis(self.conn_backoff_ms))
302                .unwrap_or(now),
303            now,
304        )
305        .duration_since(now)?;
306        let ratelimit_duration = max(until_can_send, self.ratelimit.until_can_send());
307        if !ratelimit_duration.is_zero() {
308            warn!(
309                context,
310                "IMAP got rate limited, waiting for {} until can connect.",
311                duration_to_str(ratelimit_duration),
312            );
313            let interrupted = async {
314                tokio::time::sleep(ratelimit_duration).await;
315                false
316            }
317            .race(self.idle_interrupt_receiver.recv().map(|_| true))
318            .await;
319            if interrupted {
320                info!(
321                    context,
322                    "Connecting to IMAP without waiting for ratelimit due to interrupt."
323                );
324            }
325        }
326
327        info!(context, "Connecting to IMAP server.");
328        self.connectivity.set_connecting(context).await;
329
330        self.conn_last_try = tools::Time::now();
331        const BACKOFF_MIN_MS: u64 = 2000;
332        const BACKOFF_MAX_MS: u64 = 80_000;
333        self.conn_backoff_ms = min(self.conn_backoff_ms, BACKOFF_MAX_MS / 2);
334        self.conn_backoff_ms = self.conn_backoff_ms.saturating_add(
335            rand::thread_rng().gen_range((self.conn_backoff_ms / 2)..=self.conn_backoff_ms),
336        );
337        self.conn_backoff_ms = max(BACKOFF_MIN_MS, self.conn_backoff_ms);
338
339        let login_params = prioritize_server_login_params(&context.sql, &self.lp, "imap").await?;
340        let mut first_error = None;
341        for lp in login_params {
342            info!(context, "IMAP trying to connect to {}.", &lp.connection);
343            let connection_candidate = lp.connection.clone();
344            let client = match Client::connect(
345                context,
346                self.proxy_config.clone(),
347                self.strict_tls,
348                connection_candidate,
349            )
350            .await
351            .context("IMAP failed to connect")
352            {
353                Ok(client) => client,
354                Err(err) => {
355                    warn!(context, "{err:#}.");
356                    first_error.get_or_insert(err);
357                    continue;
358                }
359            };
360
361            self.conn_backoff_ms = BACKOFF_MIN_MS;
362            self.ratelimit.send();
363
364            let imap_user: &str = lp.user.as_ref();
365            let imap_pw: &str = &self.password;
366
367            let login_res = if self.oauth2 {
368                info!(context, "Logging into IMAP server with OAuth 2.");
369                let addr: &str = self.addr.as_ref();
370
371                let token = get_oauth2_access_token(context, addr, imap_pw, true)
372                    .await?
373                    .context("IMAP could not get OAUTH token")?;
374                let auth = OAuth2 {
375                    user: imap_user.into(),
376                    access_token: token,
377                };
378                client.authenticate("XOAUTH2", auth).await
379            } else {
380                info!(context, "Logging into IMAP server with LOGIN.");
381                client.login(imap_user, imap_pw).await
382            };
383
384            match login_res {
385                Ok(mut session) => {
386                    let capabilities = determine_capabilities(&mut session).await?;
387
388                    let session = if capabilities.can_compress {
389                        info!(context, "Enabling IMAP compression.");
390                        let compressed_session = session
391                            .compress(|s| {
392                                let session_stream: Box<dyn SessionStream> = Box::new(s);
393                                session_stream
394                            })
395                            .await
396                            .context("Failed to enable IMAP compression")?;
397                        Session::new(compressed_session, capabilities)
398                    } else {
399                        Session::new(session, capabilities)
400                    };
401
402                    // Store server ID in the context to display in account info.
403                    let mut lock = context.server_id.write().await;
404                    lock.clone_from(&session.capabilities.server_id);
405
406                    self.authentication_failed_once = false;
407                    context.emit_event(EventType::ImapConnected(format!(
408                        "IMAP-LOGIN as {}",
409                        lp.user
410                    )));
411                    self.connectivity.set_preparing(context).await;
412                    info!(context, "Successfully logged into IMAP server.");
413                    return Ok(session);
414                }
415
416                Err(err) => {
417                    let imap_user = lp.user.to_owned();
418                    let message = stock_str::cannot_login(context, &imap_user).await;
419
420                    warn!(context, "IMAP failed to login: {err:#}.");
421                    first_error.get_or_insert(format_err!("{message} ({err:#})"));
422
423                    // If it looks like the password is wrong, send a notification:
424                    let _lock = context.wrong_pw_warning_mutex.lock().await;
425                    if err.to_string().to_lowercase().contains("authentication") {
426                        if self.authentication_failed_once
427                            && !configuring
428                            && context.get_config_bool(Config::NotifyAboutWrongPw).await?
429                        {
430                            let mut msg = Message::new_text(message);
431                            if let Err(e) = chat::add_device_msg_with_importance(
432                                context,
433                                None,
434                                Some(&mut msg),
435                                true,
436                            )
437                            .await
438                            {
439                                warn!(context, "Failed to add device message: {e:#}.");
440                            } else {
441                                context
442                                    .set_config_internal(Config::NotifyAboutWrongPw, None)
443                                    .await
444                                    .log_err(context)
445                                    .ok();
446                            }
447                        } else {
448                            self.authentication_failed_once = true;
449                        }
450                    } else {
451                        self.authentication_failed_once = false;
452                    }
453                }
454            }
455        }
456
457        Err(first_error.unwrap_or_else(|| format_err!("No IMAP connection candidates provided")))
458    }
459
460    /// Prepare a new IMAP session.
461    ///
462    /// This creates a new IMAP connection and ensures
463    /// that folders are created and IMAP capabilities are determined.
464    pub(crate) async fn prepare(&mut self, context: &Context) -> Result<Session> {
465        let configuring = false;
466        let mut session = match self.connect(context, configuring).await {
467            Ok(session) => session,
468            Err(err) => {
469                self.connectivity.set_err(context, &err).await;
470                return Err(err);
471            }
472        };
473
474        let folders_configured = context
475            .sql
476            .get_raw_config_int(constants::DC_FOLDERS_CONFIGURED_KEY)
477            .await?;
478        if folders_configured.unwrap_or_default() < constants::DC_FOLDERS_CONFIGURED_VERSION {
479            let is_chatmail = match context.get_config_bool(Config::FixIsChatmail).await? {
480                false => session.is_chatmail(),
481                true => context.get_config_bool(Config::IsChatmail).await?,
482            };
483            let create_mvbox = !is_chatmail || context.get_config_bool(Config::MvboxMove).await?;
484            self.configure_folders(context, &mut session, create_mvbox)
485                .await?;
486        }
487
488        Ok(session)
489    }
490
491    /// FETCH-MOVE-DELETE iteration.
492    ///
493    /// Prefetches headers and downloads new message from the folder, moves messages away from the
494    /// folder and deletes messages in the folder.
495    pub async fn fetch_move_delete(
496        &mut self,
497        context: &Context,
498        session: &mut Session,
499        watch_folder: &str,
500        folder_meaning: FolderMeaning,
501    ) -> Result<()> {
502        if !context.sql.is_open().await {
503            // probably shutdown
504            bail!("IMAP operation attempted while it is torn down");
505        }
506
507        let msgs_fetched = self
508            .fetch_new_messages(context, session, watch_folder, folder_meaning)
509            .await
510            .context("fetch_new_messages")?;
511        if msgs_fetched && context.get_config_delete_device_after().await?.is_some() {
512            // New messages were fetched and shall be deleted later, restart ephemeral loop.
513            // Note that the `Config::DeleteDeviceAfter` timer starts as soon as the messages are
514            // fetched while the per-chat ephemeral timers start as soon as the messages are marked
515            // as noticed.
516            context.scheduler.interrupt_ephemeral_task().await;
517        }
518
519        session
520            .move_delete_messages(context, watch_folder)
521            .await
522            .context("move_delete_messages")?;
523
524        Ok(())
525    }
526
527    /// Fetches new messages.
528    ///
529    /// Returns true if at least one message was fetched.
530    pub(crate) async fn fetch_new_messages(
531        &mut self,
532        context: &Context,
533        session: &mut Session,
534        folder: &str,
535        folder_meaning: FolderMeaning,
536    ) -> Result<bool> {
537        if should_ignore_folder(context, folder, folder_meaning).await? {
538            info!(context, "Not fetching from {folder:?}.");
539            session.new_mail = false;
540            return Ok(false);
541        }
542
543        let create = false;
544        let folder_exists = session
545            .select_with_uidvalidity(context, folder, create)
546            .await
547            .with_context(|| format!("Failed to select folder {folder:?}"))?;
548        if !folder_exists {
549            return Ok(false);
550        }
551
552        if !session.new_mail {
553            info!(context, "No new emails in folder {folder:?}.");
554            return Ok(false);
555        }
556        session.new_mail = false;
557
558        let uid_validity = get_uidvalidity(context, folder).await?;
559        let old_uid_next = get_uid_next(context, folder).await?;
560
561        let msgs = session.prefetch(old_uid_next).await.context("prefetch")?;
562        let read_cnt = msgs.len();
563
564        let download_limit = context.download_limit().await?;
565        let mut uids_fetch = Vec::<(_, bool /* partially? */)>::with_capacity(msgs.len() + 1);
566        let mut uid_message_ids = BTreeMap::new();
567        let mut largest_uid_skipped = None;
568        let delete_target = context.get_delete_msgs_target().await?;
569
570        // Store the info about IMAP messages in the database.
571        for (uid, ref fetch_response) in msgs {
572            let headers = match get_fetch_headers(fetch_response) {
573                Ok(headers) => headers,
574                Err(err) => {
575                    warn!(context, "Failed to parse FETCH headers: {err:#}.");
576                    continue;
577                }
578            };
579
580            let message_id = prefetch_get_message_id(&headers);
581
582            // Determine the target folder where the message should be moved to.
583            //
584            // If we have seen the message on the IMAP server before, do not move it.
585            // This is required to avoid infinite MOVE loop on IMAP servers
586            // that alias `DeltaChat` folder to other names.
587            // For example, some Dovecot servers alias `DeltaChat` folder to `INBOX.DeltaChat`.
588            // In this case Delta Chat configured with `DeltaChat` as the destination folder
589            // would detect messages in the `INBOX.DeltaChat` folder
590            // and try to move them to the `DeltaChat` folder.
591            // Such move to the same folder results in the messages
592            // getting a new UID, so the messages will be detected as new
593            // in the `INBOX.DeltaChat` folder again.
594            let _target;
595            let target = if let Some(message_id) = &message_id {
596                let msg_info =
597                    message::rfc724_mid_exists_ex(context, message_id, "deleted=1").await?;
598                let delete = if let Some((_, _, true)) = msg_info {
599                    info!(context, "Deleting locally deleted message {message_id}.");
600                    true
601                } else if let Some((_, ts_sent_old, _)) = msg_info {
602                    let is_chat_msg = headers.get_header_value(HeaderDef::ChatVersion).is_some();
603                    let ts_sent = headers
604                        .get_header_value(HeaderDef::Date)
605                        .and_then(|v| mailparse::dateparse(&v).ok())
606                        .unwrap_or_default();
607                    let is_dup = is_dup_msg(is_chat_msg, ts_sent, ts_sent_old);
608                    if is_dup {
609                        info!(context, "Deleting duplicate message {message_id}.");
610                    }
611                    is_dup
612                } else {
613                    false
614                };
615                if delete {
616                    &delete_target
617                } else if context
618                    .sql
619                    .exists(
620                        "SELECT COUNT (*) FROM imap WHERE rfc724_mid=?",
621                        (message_id,),
622                    )
623                    .await?
624                {
625                    info!(
626                        context,
627                        "Not moving the message {} that we have seen before.", &message_id
628                    );
629                    folder
630                } else {
631                    _target = target_folder(context, folder, folder_meaning, &headers).await?;
632                    &_target
633                }
634            } else {
635                // Do not move the messages without Message-ID.
636                // We cannot reliably determine if we have seen them before,
637                // so it is safer not to move them.
638                warn!(
639                    context,
640                    "Not moving the message that does not have a Message-ID."
641                );
642                folder
643            };
644
645            // Generate a fake Message-ID to identify the message in the database
646            // if the message has no real Message-ID.
647            let message_id = message_id.unwrap_or_else(create_message_id);
648
649            context
650                .sql
651                .execute(
652                    "INSERT INTO imap (rfc724_mid, folder, uid, uidvalidity, target)
653                       VALUES         (?1,         ?2,     ?3,  ?4,          ?5)
654                       ON CONFLICT(folder, uid, uidvalidity)
655                       DO UPDATE SET rfc724_mid=excluded.rfc724_mid,
656                                     target=excluded.target",
657                    (&message_id, &folder, uid, uid_validity, target),
658                )
659                .await?;
660
661            // Download only the messages which have reached their target folder if there are
662            // multiple devices. This prevents race conditions in multidevice case, where one
663            // device tries to download the message while another device moves the message at the
664            // same time. Even in single device case it is possible to fail downloading the first
665            // message, move it to the movebox and then download the second message before
666            // downloading the first one, if downloading from inbox before moving is allowed.
667            if folder == target
668                // Never download messages directly from the spam folder.
669                // If the sender is known, the message will be moved to the Inbox or Mvbox
670                // and then we download the message from there.
671                // Also see `spam_target_folder_cfg()`.
672                && folder_meaning != FolderMeaning::Spam
673                && prefetch_should_download(
674                    context,
675                    &headers,
676                    &message_id,
677                    fetch_response.flags(),
678                )
679                .await.context("prefetch_should_download")?
680            {
681                match download_limit {
682                    Some(download_limit) => uids_fetch.push((
683                        uid,
684                        fetch_response.size.unwrap_or_default() > download_limit,
685                    )),
686                    None => uids_fetch.push((uid, false)),
687                }
688                uid_message_ids.insert(uid, message_id);
689            } else {
690                largest_uid_skipped = Some(uid);
691            }
692        }
693
694        if !uids_fetch.is_empty() {
695            self.connectivity.set_working(context).await;
696        }
697
698        // Actually download messages.
699        let mut largest_uid_fetched: u32 = 0;
700        let mut received_msgs = Vec::with_capacity(uids_fetch.len());
701        let mut uids_fetch_in_batch = Vec::with_capacity(max(uids_fetch.len(), 1));
702        let mut fetch_partially = false;
703        uids_fetch.push((0, !uids_fetch.last().unwrap_or(&(0, false)).1));
704        for (uid, fp) in uids_fetch {
705            if fp != fetch_partially {
706                let (largest_uid_fetched_in_batch, received_msgs_in_batch) = session
707                    .fetch_many_msgs(
708                        context,
709                        folder,
710                        uid_validity,
711                        uids_fetch_in_batch.split_off(0),
712                        &uid_message_ids,
713                        fetch_partially,
714                    )
715                    .await
716                    .context("fetch_many_msgs")?;
717                received_msgs.extend(received_msgs_in_batch);
718                largest_uid_fetched = max(
719                    largest_uid_fetched,
720                    largest_uid_fetched_in_batch.unwrap_or(0),
721                );
722                fetch_partially = fp;
723            }
724            uids_fetch_in_batch.push(uid);
725        }
726
727        // Advance uid_next to the maximum of the largest known UID plus 1
728        // and mailbox UIDNEXT.
729        // Largest known UID is normally less than UIDNEXT,
730        // but a message may have arrived between determining UIDNEXT
731        // and executing the FETCH command.
732        let mailbox_uid_next = session
733            .selected_mailbox
734            .as_ref()
735            .with_context(|| format!("Expected {folder:?} to be selected"))?
736            .uid_next
737            .unwrap_or_default();
738        let new_uid_next = max(
739            max(largest_uid_fetched, largest_uid_skipped.unwrap_or(0)) + 1,
740            mailbox_uid_next,
741        );
742
743        if new_uid_next > old_uid_next {
744            set_uid_next(context, folder, new_uid_next).await?;
745        }
746
747        info!(context, "{} mails read from \"{}\".", read_cnt, folder);
748
749        if !received_msgs.is_empty() {
750            context.emit_event(EventType::IncomingMsgBunch);
751        }
752
753        chat::mark_old_messages_as_noticed(context, received_msgs).await?;
754
755        Ok(read_cnt > 0)
756    }
757
758    /// Read the recipients from old emails sent by the user and add them as contacts.
759    /// This way, we can already offer them some email addresses they can write to.
760    ///
761    /// Then, Fetch the last messages DC_FETCH_EXISTING_MSGS_COUNT emails from the server
762    /// and show them in the chat list.
763    pub(crate) async fn fetch_existing_msgs(
764        &mut self,
765        context: &Context,
766        session: &mut Session,
767    ) -> Result<()> {
768        add_all_recipients_as_contacts(context, session, Config::ConfiguredSentboxFolder)
769            .await
770            .context("failed to get recipients from the sentbox")?;
771        add_all_recipients_as_contacts(context, session, Config::ConfiguredMvboxFolder)
772            .await
773            .context("failed to get recipients from the movebox")?;
774        add_all_recipients_as_contacts(context, session, Config::ConfiguredInboxFolder)
775            .await
776            .context("failed to get recipients from the inbox")?;
777
778        info!(context, "Done fetching existing messages.");
779        Ok(())
780    }
781}
782
783impl Session {
784    /// Synchronizes UIDs for all folders.
785    pub(crate) async fn resync_folders(&mut self, context: &Context) -> Result<()> {
786        let all_folders = self
787            .list_folders()
788            .await
789            .context("listing folders for resync")?;
790        for folder in all_folders {
791            let folder_meaning = get_folder_meaning(&folder);
792            if folder_meaning != FolderMeaning::Virtual {
793                self.resync_folder_uids(context, folder.name(), folder_meaning)
794                    .await?;
795            }
796        }
797        Ok(())
798    }
799
800    /// Synchronizes UIDs in the database with UIDs on the server.
801    ///
802    /// It is assumed that no operations are taking place on the same
803    /// folder at the moment. Make sure to run it in the same
804    /// thread/task as other network operations on this folder to
805    /// avoid race conditions.
806    pub(crate) async fn resync_folder_uids(
807        &mut self,
808        context: &Context,
809        folder: &str,
810        folder_meaning: FolderMeaning,
811    ) -> Result<()> {
812        let uid_validity;
813        // Collect pairs of UID and Message-ID.
814        let mut msgs = BTreeMap::new();
815
816        let create = false;
817        let folder_exists = self
818            .select_with_uidvalidity(context, folder, create)
819            .await?;
820        if folder_exists {
821            let mut list = self
822                .uid_fetch("1:*", RFC724MID_UID)
823                .await
824                .with_context(|| format!("Can't resync folder {folder}"))?;
825            while let Some(fetch) = list.try_next().await? {
826                let headers = match get_fetch_headers(&fetch) {
827                    Ok(headers) => headers,
828                    Err(err) => {
829                        warn!(context, "Failed to parse FETCH headers: {}", err);
830                        continue;
831                    }
832                };
833                let message_id = prefetch_get_message_id(&headers);
834
835                if let (Some(uid), Some(rfc724_mid)) = (fetch.uid, message_id) {
836                    msgs.insert(
837                        uid,
838                        (
839                            rfc724_mid,
840                            target_folder(context, folder, folder_meaning, &headers).await?,
841                        ),
842                    );
843                }
844            }
845
846            info!(
847                context,
848                "resync_folder_uids: Collected {} message IDs in {folder}.",
849                msgs.len(),
850            );
851
852            uid_validity = get_uidvalidity(context, folder).await?;
853        } else {
854            warn!(context, "resync_folder_uids: No folder {folder}.");
855            uid_validity = 0;
856        }
857
858        // Write collected UIDs to SQLite database.
859        context
860            .sql
861            .transaction(move |transaction| {
862                transaction.execute("DELETE FROM imap WHERE folder=?", (folder,))?;
863                for (uid, (rfc724_mid, target)) in &msgs {
864                    // This may detect previously undetected moved
865                    // messages, so we update server_folder too.
866                    transaction.execute(
867                        "INSERT INTO imap (rfc724_mid, folder, uid, uidvalidity, target)
868                         VALUES           (?1,         ?2,     ?3,  ?4,          ?5)
869                         ON CONFLICT(folder, uid, uidvalidity)
870                         DO UPDATE SET rfc724_mid=excluded.rfc724_mid,
871                                       target=excluded.target",
872                        (rfc724_mid, folder, uid, uid_validity, target),
873                    )?;
874                }
875                Ok(())
876            })
877            .await?;
878        Ok(())
879    }
880
881    /// Deletes batch of messages identified by their UID from the currently
882    /// selected folder.
883    async fn delete_message_batch(
884        &mut self,
885        context: &Context,
886        uid_set: &str,
887        row_ids: Vec<i64>,
888    ) -> Result<()> {
889        // mark the message for deletion
890        self.add_flag_finalized_with_set(uid_set, "\\Deleted")
891            .await?;
892        context
893            .sql
894            .transaction(|transaction| {
895                let mut stmt = transaction.prepare("DELETE FROM imap WHERE id = ?")?;
896                for row_id in row_ids {
897                    stmt.execute((row_id,))?;
898                }
899                Ok(())
900            })
901            .await
902            .context("Cannot remove deleted messages from imap table")?;
903
904        context.emit_event(EventType::ImapMessageDeleted(format!(
905            "IMAP messages {uid_set} marked as deleted"
906        )));
907        Ok(())
908    }
909
910    /// Moves batch of messages identified by their UID from the currently
911    /// selected folder to the target folder.
912    async fn move_message_batch(
913        &mut self,
914        context: &Context,
915        set: &str,
916        row_ids: Vec<i64>,
917        target: &str,
918    ) -> Result<()> {
919        if self.can_move() {
920            match self.uid_mv(set, &target).await {
921                Ok(()) => {
922                    // Messages are moved or don't exist, IMAP returns OK response in both cases.
923                    context
924                        .sql
925                        .transaction(|transaction| {
926                            let mut stmt = transaction.prepare("DELETE FROM imap WHERE id = ?")?;
927                            for row_id in row_ids {
928                                stmt.execute((row_id,))?;
929                            }
930                            Ok(())
931                        })
932                        .await
933                        .context("Cannot delete moved messages from imap table")?;
934                    context.emit_event(EventType::ImapMessageMoved(format!(
935                        "IMAP messages {set} moved to {target}"
936                    )));
937                    return Ok(());
938                }
939                Err(err) => {
940                    if context.should_delete_to_trash().await? {
941                        error!(
942                            context,
943                            "Cannot move messages {} to {}, no fallback to COPY/DELETE because \
944                            delete_to_trash is set. Error: {:#}",
945                            set,
946                            target,
947                            err,
948                        );
949                        return Err(err.into());
950                    }
951                    warn!(
952                        context,
953                        "Cannot move messages, fallback to COPY/DELETE {} to {}: {}",
954                        set,
955                        target,
956                        err
957                    );
958                }
959            }
960        }
961
962        // Server does not support MOVE or MOVE failed.
963        // Copy messages to the destination folder if needed and mark records for deletion.
964        let copy = !context.is_trash(target).await?;
965        if copy {
966            info!(
967                context,
968                "Server does not support MOVE, fallback to COPY/DELETE {} to {}", set, target
969            );
970            self.uid_copy(&set, &target).await?;
971        } else {
972            error!(
973                context,
974                "Server does not support MOVE, fallback to DELETE {} to {}", set, target,
975            );
976        }
977        context
978            .sql
979            .transaction(|transaction| {
980                let mut stmt = transaction.prepare("UPDATE imap SET target='' WHERE id = ?")?;
981                for row_id in row_ids {
982                    stmt.execute((row_id,))?;
983                }
984                Ok(())
985            })
986            .await
987            .context("Cannot plan deletion of messages")?;
988        if copy {
989            context.emit_event(EventType::ImapMessageMoved(format!(
990                "IMAP messages {set} copied to {target}"
991            )));
992        }
993        Ok(())
994    }
995
996    /// Moves and deletes messages as planned in the `imap` table.
997    ///
998    /// This is the only place where messages are moved or deleted on the IMAP server.
999    async fn move_delete_messages(&mut self, context: &Context, folder: &str) -> Result<()> {
1000        let rows = context
1001            .sql
1002            .query_map(
1003                "SELECT id, uid, target FROM imap
1004        WHERE folder = ?
1005        AND target != folder
1006        ORDER BY target, uid",
1007                (folder,),
1008                |row| {
1009                    let rowid: i64 = row.get(0)?;
1010                    let uid: u32 = row.get(1)?;
1011                    let target: String = row.get(2)?;
1012                    Ok((rowid, uid, target))
1013                },
1014                |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
1015            )
1016            .await?;
1017
1018        for (target, rowid_set, uid_set) in UidGrouper::from(rows) {
1019            // Select folder inside the loop to avoid selecting it if there are no pending
1020            // MOVE/DELETE operations. This does not result in multiple SELECT commands
1021            // being sent because `select_folder()` does nothing if the folder is already
1022            // selected.
1023            let create = false;
1024            let folder_exists = self
1025                .select_with_uidvalidity(context, folder, create)
1026                .await?;
1027            ensure!(folder_exists, "No folder {folder}");
1028
1029            // Empty target folder name means messages should be deleted.
1030            if target.is_empty() {
1031                self.delete_message_batch(context, &uid_set, rowid_set)
1032                    .await
1033                    .with_context(|| format!("cannot delete batch of messages {:?}", &uid_set))?;
1034            } else {
1035                self.move_message_batch(context, &uid_set, rowid_set, &target)
1036                    .await
1037                    .with_context(|| {
1038                        format!(
1039                            "cannot move batch of messages {:?} to folder {:?}",
1040                            &uid_set, target
1041                        )
1042                    })?;
1043            }
1044        }
1045
1046        // Expunge folder if needed, e.g. if some jobs have
1047        // deleted messages on the server.
1048        if let Err(err) = self.maybe_close_folder(context).await {
1049            warn!(context, "Failed to close folder: {err:#}.");
1050        }
1051
1052        Ok(())
1053    }
1054
1055    /// Uploads sync messages from the `imap_send` table with `\Seen` flag set.
1056    pub(crate) async fn send_sync_msgs(&mut self, context: &Context, folder: &str) -> Result<()> {
1057        context.send_sync_msg().await?;
1058        while let Some((id, mime, msg_id, attempts)) = context
1059            .sql
1060            .query_row_optional(
1061                "SELECT id, mime, msg_id, attempts FROM imap_send ORDER BY id LIMIT 1",
1062                (),
1063                |row| {
1064                    let id: i64 = row.get(0)?;
1065                    let mime: String = row.get(1)?;
1066                    let msg_id: MsgId = row.get(2)?;
1067                    let attempts: i64 = row.get(3)?;
1068                    Ok((id, mime, msg_id, attempts))
1069                },
1070            )
1071            .await
1072            .context("Failed to SELECT from imap_send")?
1073        {
1074            let res = self
1075                .append(folder, Some("(\\Seen)"), None, mime)
1076                .await
1077                .with_context(|| format!("IMAP APPEND to {folder} failed for {msg_id}"))
1078                .log_err(context);
1079            if res.is_ok() {
1080                msg_id.set_delivered(context).await?;
1081            }
1082            const MAX_ATTEMPTS: i64 = 2;
1083            if res.is_ok() || attempts >= MAX_ATTEMPTS - 1 {
1084                context
1085                    .sql
1086                    .execute("DELETE FROM imap_send WHERE id=?", (id,))
1087                    .await
1088                    .context("Failed to delete from imap_send")?;
1089            } else {
1090                context
1091                    .sql
1092                    .execute("UPDATE imap_send SET attempts=attempts+1 WHERE id=?", (id,))
1093                    .await
1094                    .context("Failed to update imap_send.attempts")?;
1095                res?;
1096            }
1097        }
1098        Ok(())
1099    }
1100
1101    /// Stores pending `\Seen` flags for messages in `imap_markseen` table.
1102    pub(crate) async fn store_seen_flags_on_imap(&mut self, context: &Context) -> Result<()> {
1103        let rows = context
1104            .sql
1105            .query_map(
1106                "SELECT imap.id, uid, folder FROM imap, imap_markseen
1107                 WHERE imap.id = imap_markseen.id AND target = folder
1108                 ORDER BY folder, uid",
1109                [],
1110                |row| {
1111                    let rowid: i64 = row.get(0)?;
1112                    let uid: u32 = row.get(1)?;
1113                    let folder: String = row.get(2)?;
1114                    Ok((rowid, uid, folder))
1115                },
1116                |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
1117            )
1118            .await?;
1119
1120        for (folder, rowid_set, uid_set) in UidGrouper::from(rows) {
1121            let create = false;
1122            let folder_exists = match self.select_with_uidvalidity(context, &folder, create).await {
1123                Err(err) => {
1124                    warn!(
1125                        context,
1126                        "store_seen_flags_on_imap: Failed to select {folder}, will retry later: {err:#}.");
1127                    continue;
1128                }
1129                Ok(folder_exists) => folder_exists,
1130            };
1131            if !folder_exists {
1132                warn!(context, "store_seen_flags_on_imap: No folder {folder}.");
1133            } else if let Err(err) = self.add_flag_finalized_with_set(&uid_set, "\\Seen").await {
1134                warn!(
1135                    context,
1136                    "Cannot mark messages {uid_set} in {folder} as seen, will retry later: {err:#}.");
1137                continue;
1138            } else {
1139                info!(
1140                    context,
1141                    "Marked messages {} in folder {} as seen.", uid_set, folder
1142                );
1143            }
1144            context
1145                .sql
1146                .transaction(|transaction| {
1147                    let mut stmt = transaction.prepare("DELETE FROM imap_markseen WHERE id = ?")?;
1148                    for rowid in rowid_set {
1149                        stmt.execute((rowid,))?;
1150                    }
1151                    Ok(())
1152                })
1153                .await
1154                .context("Cannot remove messages marked as seen from imap_markseen table")?;
1155        }
1156
1157        Ok(())
1158    }
1159
1160    /// Synchronizes `\Seen` flags using `CONDSTORE` extension.
1161    pub(crate) async fn sync_seen_flags(&mut self, context: &Context, folder: &str) -> Result<()> {
1162        if !self.can_condstore() {
1163            info!(
1164                context,
1165                "Server does not support CONDSTORE, skipping flag synchronization."
1166            );
1167            return Ok(());
1168        }
1169
1170        let create = false;
1171        let folder_exists = self
1172            .select_with_uidvalidity(context, folder, create)
1173            .await
1174            .context("Failed to select folder")?;
1175        if !folder_exists {
1176            return Ok(());
1177        }
1178
1179        let mailbox = self
1180            .selected_mailbox
1181            .as_ref()
1182            .with_context(|| format!("No mailbox selected, folder: {folder}"))?;
1183
1184        // Check if the mailbox supports MODSEQ.
1185        // We are not interested in actual value of HIGHESTMODSEQ.
1186        if mailbox.highest_modseq.is_none() {
1187            info!(
1188                context,
1189                "Mailbox {} does not support mod-sequences, skipping flag synchronization.", folder
1190            );
1191            return Ok(());
1192        }
1193
1194        let mut updated_chat_ids = BTreeSet::new();
1195        let uid_validity = get_uidvalidity(context, folder)
1196            .await
1197            .with_context(|| format!("failed to get UID validity for folder {folder}"))?;
1198        let mut highest_modseq = get_modseq(context, folder)
1199            .await
1200            .with_context(|| format!("failed to get MODSEQ for folder {folder}"))?;
1201        let mut list = self
1202            .uid_fetch("1:*", format!("(FLAGS) (CHANGEDSINCE {highest_modseq})"))
1203            .await
1204            .context("failed to fetch flags")?;
1205
1206        let mut got_unsolicited_fetch = false;
1207
1208        while let Some(fetch) = list
1209            .try_next()
1210            .await
1211            .context("failed to get FETCH result")?
1212        {
1213            let uid = if let Some(uid) = fetch.uid {
1214                uid
1215            } else {
1216                info!(context, "FETCH result contains no UID, skipping");
1217                got_unsolicited_fetch = true;
1218                continue;
1219            };
1220            let is_seen = fetch.flags().any(|flag| flag == Flag::Seen);
1221            if is_seen {
1222                if let Some(chat_id) = mark_seen_by_uid(context, folder, uid_validity, uid)
1223                    .await
1224                    .with_context(|| {
1225                        format!("failed to update seen status for msg {folder}/{uid}")
1226                    })?
1227                {
1228                    updated_chat_ids.insert(chat_id);
1229                }
1230            }
1231
1232            if let Some(modseq) = fetch.modseq {
1233                if modseq > highest_modseq {
1234                    highest_modseq = modseq;
1235                }
1236            } else {
1237                warn!(context, "FETCH result contains no MODSEQ");
1238            }
1239        }
1240        drop(list);
1241
1242        if got_unsolicited_fetch {
1243            // We got unsolicited FETCH, which means some flags
1244            // have been modified while our request was in progress.
1245            // We may or may not have these new flags as a part of the response,
1246            // so better skip next IDLE and do another round of flag synchronization.
1247            self.new_mail = true;
1248        }
1249
1250        set_modseq(context, folder, highest_modseq)
1251            .await
1252            .with_context(|| format!("failed to set MODSEQ for folder {folder}"))?;
1253        if !updated_chat_ids.is_empty() {
1254            context.on_archived_chats_maybe_noticed();
1255        }
1256        for updated_chat_id in updated_chat_ids {
1257            context.emit_event(EventType::MsgsNoticed(updated_chat_id));
1258            chatlist_events::emit_chatlist_item_changed(context, updated_chat_id);
1259        }
1260
1261        Ok(())
1262    }
1263
1264    /// Gets the from, to and bcc addresses from all existing outgoing emails.
1265    pub async fn get_all_recipients(&mut self, context: &Context) -> Result<Vec<SingleInfo>> {
1266        let mut uids: Vec<_> = self
1267            .uid_search(get_imap_self_sent_search_command(context).await?)
1268            .await?
1269            .into_iter()
1270            .collect();
1271        uids.sort_unstable();
1272
1273        let mut result = Vec::new();
1274        for (_, uid_set) in build_sequence_sets(&uids)? {
1275            let mut list = self
1276                .uid_fetch(uid_set, "(UID BODY.PEEK[HEADER.FIELDS (FROM TO CC BCC)])")
1277                .await
1278                .context("IMAP Could not fetch")?;
1279
1280            while let Some(msg) = list.try_next().await? {
1281                match get_fetch_headers(&msg) {
1282                    Ok(headers) => {
1283                        if let Some(from) = mimeparser::get_from(&headers) {
1284                            if context.is_self_addr(&from.addr).await? {
1285                                result.extend(mimeparser::get_recipients(&headers));
1286                            }
1287                        }
1288                    }
1289                    Err(err) => {
1290                        warn!(context, "{}", err);
1291                        continue;
1292                    }
1293                };
1294            }
1295        }
1296        Ok(result)
1297    }
1298
1299    /// Fetches a list of messages by server UID.
1300    ///
1301    /// Returns the last UID fetched successfully and the info about each downloaded message.
1302    /// If the message is incorrect or there is a failure to write a message to the database,
1303    /// it is skipped and the error is logged.
1304    pub(crate) async fn fetch_many_msgs(
1305        &mut self,
1306        context: &Context,
1307        folder: &str,
1308        uidvalidity: u32,
1309        request_uids: Vec<u32>,
1310        uid_message_ids: &BTreeMap<u32, String>,
1311        fetch_partially: bool,
1312    ) -> Result<(Option<u32>, Vec<ReceivedMsg>)> {
1313        let mut last_uid = None;
1314        let mut received_msgs = Vec::new();
1315
1316        if request_uids.is_empty() {
1317            return Ok((last_uid, received_msgs));
1318        }
1319
1320        for (request_uids, set) in build_sequence_sets(&request_uids)? {
1321            info!(
1322                context,
1323                "Starting a {} FETCH of message set \"{}\".",
1324                if fetch_partially { "partial" } else { "full" },
1325                set
1326            );
1327            let mut fetch_responses = self
1328                .uid_fetch(
1329                    &set,
1330                    if fetch_partially {
1331                        BODY_PARTIAL
1332                    } else {
1333                        BODY_FULL
1334                    },
1335                )
1336                .await
1337                .with_context(|| {
1338                    format!("fetching messages {} from folder \"{}\"", &set, folder)
1339                })?;
1340
1341            // Map from UIDs to unprocessed FETCH results. We put unprocessed FETCH results here
1342            // when we want to process other messages first.
1343            let mut uid_msgs = HashMap::with_capacity(request_uids.len());
1344
1345            let mut count = 0;
1346            for &request_uid in &request_uids {
1347                // Check if FETCH response is already in `uid_msgs`.
1348                let mut fetch_response = uid_msgs.remove(&request_uid);
1349
1350                // Try to find a requested UID in returned FETCH responses.
1351                while fetch_response.is_none() {
1352                    let next_fetch_response =
1353                        if let Some(next_fetch_response) = fetch_responses.next().await {
1354                            next_fetch_response
1355                        } else {
1356                            // No more FETCH responses received from the server.
1357                            break;
1358                        };
1359
1360                    let next_fetch_response =
1361                        next_fetch_response.context("Failed to process IMAP FETCH result")?;
1362
1363                    if let Some(next_uid) = next_fetch_response.uid {
1364                        if next_uid == request_uid {
1365                            fetch_response = Some(next_fetch_response);
1366                        } else if !request_uids.contains(&next_uid) {
1367                            // (size of `request_uids` is bounded by IMAP command length limit,
1368                            // search in this vector is always fast)
1369
1370                            // Unwanted UIDs are possible because of unsolicited responses, e.g. if
1371                            // another client changes \Seen flag on a message after we do a prefetch but
1372                            // before fetch. It's not an error if we receive such unsolicited response.
1373                            info!(
1374                                context,
1375                                "Skipping not requested FETCH response for UID {}.", next_uid
1376                            );
1377                        } else if uid_msgs.insert(next_uid, next_fetch_response).is_some() {
1378                            warn!(context, "Got duplicated UID {}.", next_uid);
1379                        }
1380                    } else {
1381                        info!(context, "Skipping FETCH response without UID.");
1382                    }
1383                }
1384
1385                let fetch_response = match fetch_response {
1386                    Some(fetch) => fetch,
1387                    None => {
1388                        warn!(
1389                            context,
1390                            "Missed UID {} in the server response.", request_uid
1391                        );
1392                        continue;
1393                    }
1394                };
1395                count += 1;
1396
1397                let is_deleted = fetch_response.flags().any(|flag| flag == Flag::Deleted);
1398                let (body, partial) = if fetch_partially {
1399                    (fetch_response.header(), fetch_response.size) // `BODY.PEEK[HEADER]` goes to header() ...
1400                } else {
1401                    (fetch_response.body(), None) // ... while `BODY.PEEK[]` goes to body() - and includes header()
1402                };
1403
1404                if is_deleted {
1405                    info!(context, "Not processing deleted msg {}.", request_uid);
1406                    last_uid = Some(request_uid);
1407                    continue;
1408                }
1409
1410                let body = if let Some(body) = body {
1411                    body
1412                } else {
1413                    info!(
1414                        context,
1415                        "Not processing message {} without a BODY.", request_uid
1416                    );
1417                    last_uid = Some(request_uid);
1418                    continue;
1419                };
1420
1421                let is_seen = fetch_response.flags().any(|flag| flag == Flag::Seen);
1422
1423                let Some(rfc724_mid) = uid_message_ids.get(&request_uid) else {
1424                    error!(
1425                        context,
1426                        "No Message-ID corresponding to UID {} passed in uid_messsage_ids.",
1427                        request_uid
1428                    );
1429                    continue;
1430                };
1431
1432                info!(
1433                    context,
1434                    "Passing message UID {} to receive_imf().", request_uid
1435                );
1436                match receive_imf_inner(
1437                    context,
1438                    folder,
1439                    uidvalidity,
1440                    request_uid,
1441                    rfc724_mid,
1442                    body,
1443                    is_seen,
1444                    partial,
1445                )
1446                .await
1447                {
1448                    Ok(received_msg) => {
1449                        if let Some(m) = received_msg {
1450                            received_msgs.push(m);
1451                        }
1452                    }
1453                    Err(err) => {
1454                        warn!(context, "receive_imf error: {:#}.", err);
1455                    }
1456                };
1457                last_uid = Some(request_uid)
1458            }
1459
1460            // If we don't process the whole response, IMAP client is left in a broken state where
1461            // it will try to process the rest of response as the next response.
1462            while fetch_responses.next().await.is_some() {}
1463
1464            if count != request_uids.len() {
1465                warn!(
1466                    context,
1467                    "Failed to fetch all UIDs: got {}, requested {}, we requested the UIDs {:?}.",
1468                    count,
1469                    request_uids.len(),
1470                    request_uids,
1471                );
1472            } else {
1473                info!(
1474                    context,
1475                    "Successfully received {} UIDs.",
1476                    request_uids.len()
1477                );
1478            }
1479        }
1480
1481        Ok((last_uid, received_msgs))
1482    }
1483
1484    /// Retrieves server metadata if it is supported.
1485    ///
1486    /// We get [`/shared/comment`](https://www.rfc-editor.org/rfc/rfc5464#section-6.2.1)
1487    /// and [`/shared/admin`](https://www.rfc-editor.org/rfc/rfc5464#section-6.2.2)
1488    /// metadata.
1489    pub(crate) async fn fetch_metadata(&mut self, context: &Context) -> Result<()> {
1490        if !self.can_metadata() {
1491            return Ok(());
1492        }
1493
1494        let mut lock = context.metadata.write().await;
1495        if (*lock).is_some() {
1496            return Ok(());
1497        }
1498
1499        info!(
1500            context,
1501            "Server supports metadata, retrieving server comment and admin contact."
1502        );
1503
1504        let mut comment = None;
1505        let mut admin = None;
1506        let mut iroh_relay = None;
1507
1508        let mailbox = "";
1509        let options = "";
1510        let metadata = self
1511            .get_metadata(
1512                mailbox,
1513                options,
1514                "(/shared/comment /shared/admin /shared/vendor/deltachat/irohrelay)",
1515            )
1516            .await?;
1517        for m in metadata {
1518            match m.entry.as_ref() {
1519                "/shared/comment" => {
1520                    comment = m.value;
1521                }
1522                "/shared/admin" => {
1523                    admin = m.value;
1524                }
1525                "/shared/vendor/deltachat/irohrelay" => {
1526                    if let Some(value) = m.value {
1527                        if let Ok(url) = Url::parse(&value) {
1528                            iroh_relay = Some(url);
1529                        } else {
1530                            warn!(
1531                                context,
1532                                "Got invalid URL from iroh relay metadata: {:?}.", value
1533                            );
1534                        }
1535                    }
1536                }
1537                _ => {}
1538            }
1539        }
1540        *lock = Some(ServerMetadata {
1541            comment,
1542            admin,
1543            iroh_relay,
1544        });
1545        Ok(())
1546    }
1547
1548    /// Stores device token into /private/devicetoken IMAP METADATA of the Inbox.
1549    pub(crate) async fn register_token(&mut self, context: &Context) -> Result<()> {
1550        if context.push_subscribed.load(Ordering::Relaxed) {
1551            return Ok(());
1552        }
1553
1554        let Some(device_token) = context.push_subscriber.device_token().await else {
1555            return Ok(());
1556        };
1557
1558        if self.can_metadata() && self.can_push() {
1559            let old_encrypted_device_token =
1560                context.get_config(Config::EncryptedDeviceToken).await?;
1561
1562            // Whether we need to update encrypted device token.
1563            let device_token_changed = old_encrypted_device_token.is_none()
1564                || context.get_config(Config::DeviceToken).await?.as_ref() != Some(&device_token);
1565
1566            let new_encrypted_device_token;
1567            if device_token_changed {
1568                let encrypted_device_token = encrypt_device_token(&device_token)
1569                    .context("Failed to encrypt device token")?;
1570
1571                // We expect that the server supporting `XDELTAPUSH` capability
1572                // has non-synchronizing literals support as well:
1573                // <https://www.rfc-editor.org/rfc/rfc7888>.
1574                let encrypted_device_token_len = encrypted_device_token.len();
1575
1576                // Store device token saved on the server
1577                // to prevent storing duplicate tokens.
1578                // The server cannot deduplicate on its own
1579                // because encryption gives a different
1580                // result each time.
1581                context
1582                    .set_config_internal(Config::DeviceToken, Some(&device_token))
1583                    .await?;
1584                context
1585                    .set_config_internal(
1586                        Config::EncryptedDeviceToken,
1587                        Some(&encrypted_device_token),
1588                    )
1589                    .await?;
1590
1591                if encrypted_device_token_len <= 4096 {
1592                    new_encrypted_device_token = Some(encrypted_device_token);
1593                } else {
1594                    // If Apple or Google (FCM) gives us a very large token,
1595                    // do not even try to give it to IMAP servers.
1596                    //
1597                    // Limit of 4096 is arbitrarily selected
1598                    // to be the same as required by LITERAL- IMAP extension.
1599                    //
1600                    // Dovecot supports LITERAL+ and non-synchronizing literals
1601                    // of any length, but there is no reason for tokens
1602                    // to be that large even after OpenPGP encryption.
1603                    warn!(context, "Device token is too long for LITERAL-, ignoring.");
1604                    new_encrypted_device_token = None;
1605                }
1606            } else {
1607                new_encrypted_device_token = old_encrypted_device_token;
1608            }
1609
1610            // Store new encrypted device token on the server
1611            // even if it is the same as the old one.
1612            if let Some(encrypted_device_token) = new_encrypted_device_token {
1613                let folder = context
1614                    .get_config(Config::ConfiguredInboxFolder)
1615                    .await?
1616                    .context("INBOX is not configured")?;
1617
1618                self.run_command_and_check_ok(&format_setmetadata(
1619                    &folder,
1620                    &encrypted_device_token,
1621                ))
1622                .await
1623                .context("SETMETADATA command failed")?;
1624
1625                context.push_subscribed.store(true, Ordering::Relaxed);
1626            }
1627        } else if !context.push_subscriber.heartbeat_subscribed().await {
1628            let context = context.clone();
1629            // Subscribe for heartbeat notifications.
1630            tokio::spawn(async move { context.push_subscriber.subscribe(&context).await });
1631        }
1632
1633        Ok(())
1634    }
1635}
1636
1637fn format_setmetadata(folder: &str, device_token: &str) -> String {
1638    let device_token_len = device_token.len();
1639    format!(
1640        "SETMETADATA \"{folder}\" (/private/devicetoken {{{device_token_len}+}}\r\n{device_token})"
1641    )
1642}
1643
1644impl Session {
1645    /// Returns success if we successfully set the flag or we otherwise
1646    /// think add_flag should not be retried: Disconnection during setting
1647    /// the flag, or other imap-errors, returns Ok as well.
1648    ///
1649    /// Returning error means that the operation can be retried.
1650    async fn add_flag_finalized_with_set(&mut self, uid_set: &str, flag: &str) -> Result<()> {
1651        if flag == "\\Deleted" {
1652            self.selected_folder_needs_expunge = true;
1653        }
1654        let query = format!("+FLAGS ({flag})");
1655        let mut responses = self
1656            .uid_store(uid_set, &query)
1657            .await
1658            .with_context(|| format!("IMAP failed to store: ({uid_set}, {query})"))?;
1659        while let Some(_response) = responses.next().await {
1660            // Read all the responses
1661        }
1662        Ok(())
1663    }
1664
1665    /// Attempts to configure mvbox.
1666    ///
1667    /// Tries to find any folder examining `folders` in the order they go. If none is found, tries
1668    /// to create any folder in the same order. This method does not use LIST command to ensure that
1669    /// configuration works even if mailbox lookup is forbidden via Access Control List (see
1670    /// <https://datatracker.ietf.org/doc/html/rfc4314>).
1671    ///
1672    /// Returns first found or created folder name.
1673    async fn configure_mvbox<'a>(
1674        &mut self,
1675        context: &Context,
1676        folders: &[&'a str],
1677        create_mvbox: bool,
1678    ) -> Result<Option<&'a str>> {
1679        // Close currently selected folder if needed.
1680        // We are going to select folders using low-level EXAMINE operations below.
1681        self.maybe_close_folder(context).await?;
1682
1683        for folder in folders {
1684            info!(context, "Looking for MVBOX-folder \"{}\"...", &folder);
1685            let res = self.examine(&folder).await;
1686            if res.is_ok() {
1687                info!(
1688                    context,
1689                    "MVBOX-folder {:?} successfully selected, using it.", &folder
1690                );
1691                self.close().await?;
1692                // Before moving emails to the mvbox we need to remember its UIDVALIDITY, otherwise
1693                // emails moved before that wouldn't be fetched but considered "old" instead.
1694                let create = false;
1695                let folder_exists = self
1696                    .select_with_uidvalidity(context, folder, create)
1697                    .await?;
1698                ensure!(folder_exists, "No MVBOX folder {:?}??", &folder);
1699                return Ok(Some(folder));
1700            }
1701        }
1702
1703        if !create_mvbox {
1704            return Ok(None);
1705        }
1706        // Some servers require namespace-style folder names like "INBOX.DeltaChat", so we try all
1707        // the variants here.
1708        for folder in folders {
1709            match self
1710                .select_with_uidvalidity(context, folder, create_mvbox)
1711                .await
1712            {
1713                Ok(_) => {
1714                    info!(context, "MVBOX-folder {} created.", folder);
1715                    return Ok(Some(folder));
1716                }
1717                Err(err) => {
1718                    warn!(context, "Cannot create MVBOX-folder {:?}: {}", folder, err);
1719                }
1720            }
1721        }
1722        Ok(None)
1723    }
1724}
1725
1726impl Imap {
1727    pub(crate) async fn configure_folders(
1728        &mut self,
1729        context: &Context,
1730        session: &mut Session,
1731        create_mvbox: bool,
1732    ) -> Result<()> {
1733        let mut folders = session
1734            .list(Some(""), Some("*"))
1735            .await
1736            .context("list_folders failed")?;
1737        let mut delimiter = ".".to_string();
1738        let mut delimiter_is_default = true;
1739        let mut folder_configs = BTreeMap::new();
1740
1741        while let Some(folder) = folders.try_next().await? {
1742            info!(context, "Scanning folder: {:?}", folder);
1743
1744            // Update the delimiter iff there is a different one, but only once.
1745            if let Some(d) = folder.delimiter() {
1746                if delimiter_is_default && !d.is_empty() && delimiter != d {
1747                    delimiter = d.to_string();
1748                    delimiter_is_default = false;
1749                }
1750            }
1751
1752            let folder_meaning = get_folder_meaning_by_attrs(folder.attributes());
1753            let folder_name_meaning = get_folder_meaning_by_name(folder.name());
1754            if let Some(config) = folder_meaning.to_config() {
1755                // Always takes precedence
1756                folder_configs.insert(config, folder.name().to_string());
1757            } else if let Some(config) = folder_name_meaning.to_config() {
1758                // only set if none has been already set
1759                folder_configs
1760                    .entry(config)
1761                    .or_insert_with(|| folder.name().to_string());
1762            }
1763        }
1764        drop(folders);
1765
1766        info!(context, "Using \"{}\" as folder-delimiter.", delimiter);
1767
1768        let fallback_folder = format!("INBOX{delimiter}DeltaChat");
1769        let mvbox_folder = session
1770            .configure_mvbox(context, &["DeltaChat", &fallback_folder], create_mvbox)
1771            .await
1772            .context("failed to configure mvbox")?;
1773
1774        context
1775            .set_config_internal(Config::ConfiguredInboxFolder, Some("INBOX"))
1776            .await?;
1777        if let Some(mvbox_folder) = mvbox_folder {
1778            info!(context, "Setting MVBOX FOLDER TO {}", &mvbox_folder);
1779            context
1780                .set_config_internal(Config::ConfiguredMvboxFolder, Some(mvbox_folder))
1781                .await?;
1782        }
1783        for (config, name) in folder_configs {
1784            context.set_config_internal(config, Some(&name)).await?;
1785        }
1786        context
1787            .sql
1788            .set_raw_config_int(
1789                constants::DC_FOLDERS_CONFIGURED_KEY,
1790                constants::DC_FOLDERS_CONFIGURED_VERSION,
1791            )
1792            .await?;
1793
1794        info!(context, "FINISHED configuring IMAP-folders.");
1795        Ok(())
1796    }
1797}
1798
1799impl Session {
1800    /// Return whether the server sent an unsolicited EXISTS or FETCH response.
1801    ///
1802    /// Drains all responses from `session.unsolicited_responses` in the process.
1803    ///
1804    /// If this returns `true`, this means that new emails arrived
1805    /// or flags have been changed.
1806    /// In this case we may want to skip next IDLE and do a round
1807    /// of fetching new messages and synchronizing seen flags.
1808    fn drain_unsolicited_responses(&self, context: &Context) -> Result<bool> {
1809        use async_imap::imap_proto::Response;
1810        use async_imap::imap_proto::ResponseCode;
1811        use UnsolicitedResponse::*;
1812
1813        let folder = self.selected_folder.as_deref().unwrap_or_default();
1814        let mut should_refetch = false;
1815        while let Ok(response) = self.unsolicited_responses.try_recv() {
1816            match response {
1817                Exists(_) => {
1818                    info!(
1819                        context,
1820                        "Need to refetch {folder:?}, got unsolicited EXISTS {response:?}"
1821                    );
1822                    should_refetch = true;
1823                }
1824
1825                Expunge(_) | Recent(_) => {}
1826                Other(ref response_data) => {
1827                    match response_data.parsed() {
1828                        Response::Fetch { .. } => {
1829                            info!(
1830                                context,
1831                                "Need to refetch {folder:?}, got unsolicited FETCH {response:?}"
1832                            );
1833                            should_refetch = true;
1834                        }
1835
1836                        // We are not interested in the following responses and they are are
1837                        // sent quite frequently, so, we ignore them without logging them.
1838                        Response::Done {
1839                            code: Some(ResponseCode::CopyUid(_, _, _)),
1840                            ..
1841                        } => {}
1842
1843                        _ => {
1844                            info!(context, "{folder:?}: got unsolicited response {response:?}")
1845                        }
1846                    }
1847                }
1848                _ => {
1849                    info!(context, "{folder:?}: got unsolicited response {response:?}")
1850                }
1851            }
1852        }
1853        Ok(should_refetch)
1854    }
1855}
1856
1857async fn should_move_out_of_spam(
1858    context: &Context,
1859    headers: &[mailparse::MailHeader<'_>],
1860) -> Result<bool> {
1861    if headers.get_header_value(HeaderDef::ChatVersion).is_some() {
1862        // If this is a chat message (i.e. has a ChatVersion header), then this might be
1863        // a securejoin message. We can't find out at this point as we didn't prefetch
1864        // the SecureJoin header. So, we always move chat messages out of Spam.
1865        // Two possibilities to change this would be:
1866        // 1. Remove the `&& !context.is_spam_folder(folder).await?` check from
1867        // `fetch_new_messages()`, and then let `receive_imf()` check
1868        // if it's a spam message and should be hidden.
1869        // 2. Or add a flag to the ChatVersion header that this is a securejoin
1870        // request, and return `true` here only if the message has this flag.
1871        // `receive_imf()` can then check if the securejoin request is valid.
1872        return Ok(true);
1873    }
1874
1875    if let Some(msg) = get_prefetch_parent_message(context, headers).await? {
1876        if msg.chat_blocked != Blocked::Not {
1877            // Blocked or contact request message in the spam folder, leave it there.
1878            return Ok(false);
1879        }
1880    } else {
1881        let from = match mimeparser::get_from(headers) {
1882            Some(f) => f,
1883            None => return Ok(false),
1884        };
1885        // No chat found.
1886        let (from_id, blocked_contact, _origin) =
1887            match from_field_to_contact_id(context, &from, true)
1888                .await
1889                .context("from_field_to_contact_id")?
1890            {
1891                Some(res) => res,
1892                None => {
1893                    warn!(
1894                        context,
1895                        "Contact with From address {:?} cannot exist, not moving out of spam", from
1896                    );
1897                    return Ok(false);
1898                }
1899            };
1900        if blocked_contact {
1901            // Contact is blocked, leave the message in spam.
1902            return Ok(false);
1903        }
1904
1905        if let Some(chat_id_blocked) = ChatIdBlocked::lookup_by_contact(context, from_id).await? {
1906            if chat_id_blocked.blocked != Blocked::Not {
1907                return Ok(false);
1908            }
1909        } else if from_id != ContactId::SELF {
1910            // No chat with this contact found.
1911            return Ok(false);
1912        }
1913    }
1914
1915    Ok(true)
1916}
1917
1918/// Returns target folder for a message found in the Spam folder.
1919/// If this returns None, the message will not be moved out of the
1920/// Spam folder, and as `fetch_new_messages()` doesn't download
1921/// messages from the Spam folder, the message will be ignored.
1922async fn spam_target_folder_cfg(
1923    context: &Context,
1924    headers: &[mailparse::MailHeader<'_>],
1925) -> Result<Option<Config>> {
1926    if !should_move_out_of_spam(context, headers).await? {
1927        return Ok(None);
1928    }
1929
1930    if needs_move_to_mvbox(context, headers).await?
1931        // If OnlyFetchMvbox is set, we don't want to move the message to
1932        // the inbox or sentbox where we wouldn't fetch it again:
1933        || context.get_config_bool(Config::OnlyFetchMvbox).await?
1934    {
1935        Ok(Some(Config::ConfiguredMvboxFolder))
1936    } else {
1937        Ok(Some(Config::ConfiguredInboxFolder))
1938    }
1939}
1940
1941/// Returns `ConfiguredInboxFolder`, `ConfiguredMvboxFolder` or `ConfiguredSentboxFolder` if
1942/// the message needs to be moved from `folder`. Otherwise returns `None`.
1943pub async fn target_folder_cfg(
1944    context: &Context,
1945    folder: &str,
1946    folder_meaning: FolderMeaning,
1947    headers: &[mailparse::MailHeader<'_>],
1948) -> Result<Option<Config>> {
1949    if context.is_mvbox(folder).await? {
1950        return Ok(None);
1951    }
1952
1953    if folder_meaning == FolderMeaning::Spam {
1954        spam_target_folder_cfg(context, headers).await
1955    } else if needs_move_to_mvbox(context, headers).await? {
1956        Ok(Some(Config::ConfiguredMvboxFolder))
1957    } else {
1958        Ok(None)
1959    }
1960}
1961
1962pub async fn target_folder(
1963    context: &Context,
1964    folder: &str,
1965    folder_meaning: FolderMeaning,
1966    headers: &[mailparse::MailHeader<'_>],
1967) -> Result<String> {
1968    match target_folder_cfg(context, folder, folder_meaning, headers).await? {
1969        Some(config) => match context.get_config(config).await? {
1970            Some(target) => Ok(target),
1971            None => Ok(folder.to_string()),
1972        },
1973        None => Ok(folder.to_string()),
1974    }
1975}
1976
1977async fn needs_move_to_mvbox(
1978    context: &Context,
1979    headers: &[mailparse::MailHeader<'_>],
1980) -> Result<bool> {
1981    let has_chat_version = headers.get_header_value(HeaderDef::ChatVersion).is_some();
1982    if !context.get_config_bool(Config::IsChatmail).await?
1983        && has_chat_version
1984        && headers
1985            .get_header_value(HeaderDef::AutoSubmitted)
1986            .filter(|val| val.eq_ignore_ascii_case("auto-generated"))
1987            .is_some()
1988    {
1989        if let Some(from) = mimeparser::get_from(headers) {
1990            if context.is_self_addr(&from.addr).await? {
1991                return Ok(true);
1992            }
1993        }
1994    }
1995    if !context.get_config_bool(Config::MvboxMove).await? {
1996        return Ok(false);
1997    }
1998
1999    if headers
2000        .get_header_value(HeaderDef::AutocryptSetupMessage)
2001        .is_some()
2002    {
2003        // do not move setup messages;
2004        // there may be a non-delta device that wants to handle it
2005        return Ok(false);
2006    }
2007
2008    if has_chat_version {
2009        Ok(true)
2010    } else if let Some(parent) = get_prefetch_parent_message(context, headers).await? {
2011        match parent.is_dc_message {
2012            MessengerMessage::No => Ok(false),
2013            MessengerMessage::Yes | MessengerMessage::Reply => Ok(true),
2014        }
2015    } else {
2016        Ok(false)
2017    }
2018}
2019
2020/// Try to get the folder meaning by the name of the folder only used if the server does not support XLIST.
2021// TODO: lots languages missing - maybe there is a list somewhere on other MUAs?
2022// however, if we fail to find out the sent-folder,
2023// only watching this folder is not working. at least, this is no show stopper.
2024// CAVE: if possible, take care not to add a name here that is "sent" in one language
2025// but sth. different in others - a hard job.
2026fn get_folder_meaning_by_name(folder_name: &str) -> FolderMeaning {
2027    // source: <https://stackoverflow.com/questions/2185391/localized-gmail-imap-folders>
2028    const SENT_NAMES: &[&str] = &[
2029        "sent",
2030        "sentmail",
2031        "sent objects",
2032        "gesendet",
2033        "Sent Mail",
2034        "Sendte e-mails",
2035        "Enviados",
2036        "Messages envoyés",
2037        "Messages envoyes",
2038        "Posta inviata",
2039        "Verzonden berichten",
2040        "Wyslane",
2041        "E-mails enviados",
2042        "Correio enviado",
2043        "Enviada",
2044        "Enviado",
2045        "Gönderildi",
2046        "Inviati",
2047        "Odeslaná pošta",
2048        "Sendt",
2049        "Skickat",
2050        "Verzonden",
2051        "Wysłane",
2052        "Éléments envoyés",
2053        "Απεσταλμένα",
2054        "Отправленные",
2055        "寄件備份",
2056        "已发送邮件",
2057        "送信済み",
2058        "보낸편지함",
2059    ];
2060    const SPAM_NAMES: &[&str] = &[
2061        "spam",
2062        "junk",
2063        "Correio electrónico não solicitado",
2064        "Correo basura",
2065        "Lixo",
2066        "Nettsøppel",
2067        "Nevyžádaná pošta",
2068        "No solicitado",
2069        "Ongewenst",
2070        "Posta indesiderata",
2071        "Skräp",
2072        "Wiadomości-śmieci",
2073        "Önemsiz",
2074        "Ανεπιθύμητα",
2075        "Спам",
2076        "垃圾邮件",
2077        "垃圾郵件",
2078        "迷惑メール",
2079        "스팸",
2080    ];
2081    const DRAFT_NAMES: &[&str] = &[
2082        "Drafts",
2083        "Kladder",
2084        "Entw?rfe",
2085        "Borradores",
2086        "Brouillons",
2087        "Bozze",
2088        "Concepten",
2089        "Wersje robocze",
2090        "Rascunhos",
2091        "Entwürfe",
2092        "Koncepty",
2093        "Kopie robocze",
2094        "Taslaklar",
2095        "Utkast",
2096        "Πρόχειρα",
2097        "Черновики",
2098        "下書き",
2099        "草稿",
2100        "임시보관함",
2101    ];
2102    const TRASH_NAMES: &[&str] = &[
2103        "Trash",
2104        "Bin",
2105        "Caixote do lixo",
2106        "Cestino",
2107        "Corbeille",
2108        "Papelera",
2109        "Papierkorb",
2110        "Papirkurv",
2111        "Papperskorgen",
2112        "Prullenbak",
2113        "Rubujo",
2114        "Κάδος απορριμμάτων",
2115        "Корзина",
2116        "Кошик",
2117        "ゴミ箱",
2118        "垃圾桶",
2119        "已删除邮件",
2120        "휴지통",
2121    ];
2122    let lower = folder_name.to_lowercase();
2123
2124    if SENT_NAMES.iter().any(|s| s.to_lowercase() == lower) {
2125        FolderMeaning::Sent
2126    } else if SPAM_NAMES.iter().any(|s| s.to_lowercase() == lower) {
2127        FolderMeaning::Spam
2128    } else if DRAFT_NAMES.iter().any(|s| s.to_lowercase() == lower) {
2129        FolderMeaning::Drafts
2130    } else if TRASH_NAMES.iter().any(|s| s.to_lowercase() == lower) {
2131        FolderMeaning::Trash
2132    } else {
2133        FolderMeaning::Unknown
2134    }
2135}
2136
2137fn get_folder_meaning_by_attrs(folder_attrs: &[NameAttribute]) -> FolderMeaning {
2138    for attr in folder_attrs {
2139        match attr {
2140            NameAttribute::Trash => return FolderMeaning::Trash,
2141            NameAttribute::Sent => return FolderMeaning::Sent,
2142            NameAttribute::Junk => return FolderMeaning::Spam,
2143            NameAttribute::Drafts => return FolderMeaning::Drafts,
2144            NameAttribute::All | NameAttribute::Flagged => return FolderMeaning::Virtual,
2145            NameAttribute::Extension(ref label) => {
2146                match label.as_ref() {
2147                    "\\Spam" => return FolderMeaning::Spam,
2148                    "\\Important" => return FolderMeaning::Virtual,
2149                    _ => {}
2150                };
2151            }
2152            _ => {}
2153        }
2154    }
2155    FolderMeaning::Unknown
2156}
2157
2158pub(crate) fn get_folder_meaning(folder: &Name) -> FolderMeaning {
2159    match get_folder_meaning_by_attrs(folder.attributes()) {
2160        FolderMeaning::Unknown => get_folder_meaning_by_name(folder.name()),
2161        meaning => meaning,
2162    }
2163}
2164
2165/// Parses the headers from the FETCH result.
2166fn get_fetch_headers(prefetch_msg: &Fetch) -> Result<Vec<mailparse::MailHeader>> {
2167    match prefetch_msg.header() {
2168        Some(header_bytes) => {
2169            let (headers, _) = mailparse::parse_headers(header_bytes)?;
2170            Ok(headers)
2171        }
2172        None => Ok(Vec::new()),
2173    }
2174}
2175
2176pub(crate) fn prefetch_get_message_id(headers: &[mailparse::MailHeader]) -> Option<String> {
2177    headers
2178        .get_header_value(HeaderDef::XMicrosoftOriginalMessageId)
2179        .or_else(|| headers.get_header_value(HeaderDef::MessageId))
2180        .and_then(|msgid| mimeparser::parse_message_id(&msgid).ok())
2181}
2182
2183pub(crate) fn create_message_id() -> String {
2184    format!("{}{}", GENERATED_PREFIX, create_id())
2185}
2186
2187/// Returns chat by prefetched headers.
2188async fn prefetch_get_chat(
2189    context: &Context,
2190    headers: &[mailparse::MailHeader<'_>],
2191) -> Result<Option<chat::Chat>> {
2192    let parent = get_prefetch_parent_message(context, headers).await?;
2193    if let Some(parent) = &parent {
2194        return Ok(Some(
2195            chat::Chat::load_from_db(context, parent.get_chat_id()).await?,
2196        ));
2197    }
2198
2199    Ok(None)
2200}
2201
2202/// Determines whether the message should be downloaded based on prefetched headers.
2203pub(crate) async fn prefetch_should_download(
2204    context: &Context,
2205    headers: &[mailparse::MailHeader<'_>],
2206    message_id: &str,
2207    mut flags: impl Iterator<Item = Flag<'_>>,
2208) -> Result<bool> {
2209    if message::rfc724_mid_exists(context, message_id)
2210        .await?
2211        .is_some()
2212    {
2213        markseen_on_imap_table(context, message_id).await?;
2214        return Ok(false);
2215    }
2216
2217    // We do not know the Message-ID or the Message-ID is missing (in this case, we create one in
2218    // the further process).
2219
2220    if let Some(chat) = prefetch_get_chat(context, headers).await? {
2221        if chat.typ == Chattype::Group && !chat.id.is_special() {
2222            // This might be a group command, like removing a group member.
2223            // We really need to fetch this to avoid inconsistent group state.
2224            return Ok(true);
2225        }
2226    }
2227
2228    let maybe_ndn = if let Some(from) = headers.get_header_value(HeaderDef::From_) {
2229        let from = from.to_ascii_lowercase();
2230        from.contains("mailer-daemon") || from.contains("mail-daemon")
2231    } else {
2232        false
2233    };
2234
2235    // Autocrypt Setup Message should be shown even if it is from non-chat client.
2236    let is_autocrypt_setup_message = headers
2237        .get_header_value(HeaderDef::AutocryptSetupMessage)
2238        .is_some();
2239
2240    let from = match mimeparser::get_from(headers) {
2241        Some(f) => f,
2242        None => return Ok(false),
2243    };
2244    let (_from_id, blocked_contact, origin) =
2245        match from_field_to_contact_id(context, &from, true).await? {
2246            Some(res) => res,
2247            None => return Ok(false),
2248        };
2249    // prevent_rename=true as this might be a mailing list message and in this case it would be bad if we rename the contact.
2250    // (prevent_rename is the last argument of from_field_to_contact_id())
2251
2252    if flags.any(|f| f == Flag::Draft) {
2253        info!(context, "Ignoring draft message");
2254        return Ok(false);
2255    }
2256
2257    let is_chat_message = headers.get_header_value(HeaderDef::ChatVersion).is_some();
2258    let accepted_contact = origin.is_known();
2259    let is_reply_to_chat_message = get_prefetch_parent_message(context, headers)
2260        .await?
2261        .map(|parent| match parent.is_dc_message {
2262            MessengerMessage::No => false,
2263            MessengerMessage::Yes | MessengerMessage::Reply => true,
2264        })
2265        .unwrap_or_default();
2266
2267    let show_emails =
2268        ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await?).unwrap_or_default();
2269
2270    let show = is_autocrypt_setup_message
2271        || match show_emails {
2272            ShowEmails::Off => is_chat_message || is_reply_to_chat_message,
2273            ShowEmails::AcceptedContacts => {
2274                is_chat_message || is_reply_to_chat_message || accepted_contact
2275            }
2276            ShowEmails::All => true,
2277        };
2278
2279    let should_download = (show && !blocked_contact) || maybe_ndn;
2280    Ok(should_download)
2281}
2282
2283/// Returns whether a message is a duplicate (resent message).
2284pub(crate) fn is_dup_msg(is_chat_msg: bool, ts_sent: i64, ts_sent_old: i64) -> bool {
2285    // If the existing message has timestamp_sent == 0, that means we don't know its actual sent
2286    // timestamp, so don't delete the new message. E.g. outgoing messages have zero timestamp_sent
2287    // because they are stored to the db before sending. Also consider as duplicates only messages
2288    // with greater timestamp to avoid deleting both messages in a multi-device setting.
2289    is_chat_msg && ts_sent_old != 0 && ts_sent > ts_sent_old
2290}
2291
2292/// Marks messages in `msgs` table as seen, searching for them by UID.
2293///
2294/// Returns updated chat ID if any message was marked as seen.
2295async fn mark_seen_by_uid(
2296    context: &Context,
2297    folder: &str,
2298    uid_validity: u32,
2299    uid: u32,
2300) -> Result<Option<ChatId>> {
2301    if let Some((msg_id, chat_id)) = context
2302        .sql
2303        .query_row_optional(
2304            "SELECT id, chat_id FROM msgs
2305                 WHERE id > 9 AND rfc724_mid IN (
2306                   SELECT rfc724_mid FROM imap
2307                   WHERE folder=?1
2308                   AND uidvalidity=?2
2309                   AND uid=?3
2310                   LIMIT 1
2311                 )",
2312            (&folder, uid_validity, uid),
2313            |row| {
2314                let msg_id: MsgId = row.get(0)?;
2315                let chat_id: ChatId = row.get(1)?;
2316                Ok((msg_id, chat_id))
2317            },
2318        )
2319        .await
2320        .with_context(|| format!("failed to get msg and chat ID for IMAP message {folder}/{uid}"))?
2321    {
2322        let updated = context
2323            .sql
2324            .execute(
2325                "UPDATE msgs SET state=?1
2326                     WHERE (state=?2 OR state=?3)
2327                     AND id=?4",
2328                (
2329                    MessageState::InSeen,
2330                    MessageState::InFresh,
2331                    MessageState::InNoticed,
2332                    msg_id,
2333                ),
2334            )
2335            .await
2336            .with_context(|| format!("failed to update msg {msg_id} state"))?
2337            > 0;
2338
2339        if updated {
2340            msg_id
2341                .start_ephemeral_timer(context)
2342                .await
2343                .with_context(|| format!("failed to start ephemeral timer for message {msg_id}"))?;
2344            Ok(Some(chat_id))
2345        } else {
2346            // Message state has not changed.
2347            Ok(None)
2348        }
2349    } else {
2350        // There is no message is `msgs` table matching the given UID.
2351        Ok(None)
2352    }
2353}
2354
2355/// Schedule marking the message as Seen on IMAP by adding all known IMAP messages corresponding to
2356/// the given Message-ID to `imap_markseen` table.
2357pub(crate) async fn markseen_on_imap_table(context: &Context, message_id: &str) -> Result<()> {
2358    context
2359        .sql
2360        .execute(
2361            "INSERT OR IGNORE INTO imap_markseen (id)
2362             SELECT id FROM imap WHERE rfc724_mid=?",
2363            (message_id,),
2364        )
2365        .await?;
2366    context.scheduler.interrupt_inbox().await;
2367
2368    Ok(())
2369}
2370
2371/// uid_next is the next unique identifier value from the last time we fetched a folder
2372/// See <https://tools.ietf.org/html/rfc3501#section-2.3.1.1>
2373/// This function is used to update our uid_next after fetching messages.
2374pub(crate) async fn set_uid_next(context: &Context, folder: &str, uid_next: u32) -> Result<()> {
2375    context
2376        .sql
2377        .execute(
2378            "INSERT INTO imap_sync (folder, uid_next) VALUES (?,?)
2379                ON CONFLICT(folder) DO UPDATE SET uid_next=excluded.uid_next",
2380            (folder, uid_next),
2381        )
2382        .await?;
2383    Ok(())
2384}
2385
2386/// uid_next is the next unique identifier value from the last time we fetched a folder
2387/// See <https://tools.ietf.org/html/rfc3501#section-2.3.1.1>
2388/// This method returns the uid_next from the last time we fetched messages.
2389/// We can compare this to the current uid_next to find out whether there are new messages
2390/// and fetch from this value on to get all new messages.
2391async fn get_uid_next(context: &Context, folder: &str) -> Result<u32> {
2392    Ok(context
2393        .sql
2394        .query_get_value("SELECT uid_next FROM imap_sync WHERE folder=?;", (folder,))
2395        .await?
2396        .unwrap_or(0))
2397}
2398
2399pub(crate) async fn set_uidvalidity(
2400    context: &Context,
2401    folder: &str,
2402    uidvalidity: u32,
2403) -> Result<()> {
2404    context
2405        .sql
2406        .execute(
2407            "INSERT INTO imap_sync (folder, uidvalidity) VALUES (?,?)
2408                ON CONFLICT(folder) DO UPDATE SET uidvalidity=excluded.uidvalidity",
2409            (folder, uidvalidity),
2410        )
2411        .await?;
2412    Ok(())
2413}
2414
2415async fn get_uidvalidity(context: &Context, folder: &str) -> Result<u32> {
2416    Ok(context
2417        .sql
2418        .query_get_value(
2419            "SELECT uidvalidity FROM imap_sync WHERE folder=?;",
2420            (folder,),
2421        )
2422        .await?
2423        .unwrap_or(0))
2424}
2425
2426pub(crate) async fn set_modseq(context: &Context, folder: &str, modseq: u64) -> Result<()> {
2427    context
2428        .sql
2429        .execute(
2430            "INSERT INTO imap_sync (folder, modseq) VALUES (?,?)
2431                ON CONFLICT(folder) DO UPDATE SET modseq=excluded.modseq",
2432            (folder, modseq),
2433        )
2434        .await?;
2435    Ok(())
2436}
2437
2438async fn get_modseq(context: &Context, folder: &str) -> Result<u64> {
2439    Ok(context
2440        .sql
2441        .query_get_value("SELECT modseq FROM imap_sync WHERE folder=?;", (folder,))
2442        .await?
2443        .unwrap_or(0))
2444}
2445
2446/// Compute the imap search expression for all self-sent mails (for all self addresses)
2447pub(crate) async fn get_imap_self_sent_search_command(context: &Context) -> Result<String> {
2448    // See https://www.rfc-editor.org/rfc/rfc3501#section-6.4.4 for syntax of SEARCH and OR
2449    let mut search_command = format!("FROM \"{}\"", context.get_primary_self_addr().await?);
2450
2451    for item in context.get_secondary_self_addrs().await? {
2452        search_command = format!("OR ({search_command}) (FROM \"{item}\")");
2453    }
2454
2455    Ok(search_command)
2456}
2457
2458/// Deprecated, use get_uid_next() and get_uidvalidity()
2459pub async fn get_config_last_seen_uid(context: &Context, folder: &str) -> Result<(u32, u32)> {
2460    let key = format!("imap.mailbox.{folder}");
2461    if let Some(entry) = context.sql.get_raw_config(&key).await? {
2462        // the entry has the format `imap.mailbox.<folder>=<uidvalidity>:<lastseenuid>`
2463        let mut parts = entry.split(':');
2464        Ok((
2465            parts.next().unwrap_or_default().parse().unwrap_or(0),
2466            parts.next().unwrap_or_default().parse().unwrap_or(0),
2467        ))
2468    } else {
2469        Ok((0, 0))
2470    }
2471}
2472
2473/// Whether to ignore fetching messages from a folder.
2474///
2475/// This caters for the [`Config::OnlyFetchMvbox`] setting which means mails from folders
2476/// not explicitly watched should not be fetched.
2477async fn should_ignore_folder(
2478    context: &Context,
2479    folder: &str,
2480    folder_meaning: FolderMeaning,
2481) -> Result<bool> {
2482    if !context.get_config_bool(Config::OnlyFetchMvbox).await? {
2483        return Ok(false);
2484    }
2485    if context.is_sentbox(folder).await? {
2486        // Still respect the SentboxWatch setting.
2487        return Ok(!context.get_config_bool(Config::SentboxWatch).await?);
2488    }
2489    Ok(!(context.is_mvbox(folder).await? || folder_meaning == FolderMeaning::Spam))
2490}
2491
2492/// Builds a list of sequence/uid sets. The returned sets have each no more than around 1000
2493/// characters because according to <https://tools.ietf.org/html/rfc2683#section-3.2.1.5>
2494/// command lines should not be much more than 1000 chars (servers should allow at least 8000 chars)
2495fn build_sequence_sets(uids: &[u32]) -> Result<Vec<(Vec<u32>, String)>> {
2496    // first, try to find consecutive ranges:
2497    let mut ranges: Vec<UidRange> = vec![];
2498
2499    for &current in uids {
2500        if let Some(last) = ranges.last_mut() {
2501            if last.end + 1 == current {
2502                last.end = current;
2503                continue;
2504            }
2505        }
2506
2507        ranges.push(UidRange {
2508            start: current,
2509            end: current,
2510        });
2511    }
2512
2513    // Second, sort the uids into uid sets that are each below ~1000 characters
2514    let mut result = vec![];
2515    let (mut last_uids, mut last_str) = (Vec::new(), String::new());
2516    for range in ranges {
2517        last_uids.reserve((range.end - range.start + 1).try_into()?);
2518        (range.start..=range.end).for_each(|u| last_uids.push(u));
2519        if !last_str.is_empty() {
2520            last_str.push(',');
2521        }
2522        last_str.push_str(&range.to_string());
2523
2524        if last_str.len() > 990 {
2525            result.push((take(&mut last_uids), take(&mut last_str)));
2526        }
2527    }
2528    result.push((last_uids, last_str));
2529
2530    result.retain(|(_, s)| !s.is_empty());
2531    Ok(result)
2532}
2533
2534struct UidRange {
2535    start: u32,
2536    end: u32,
2537    // If start == end, then this range represents a single number
2538}
2539
2540impl std::fmt::Display for UidRange {
2541    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
2542        if self.start == self.end {
2543            write!(f, "{}", self.start)
2544        } else {
2545            write!(f, "{}:{}", self.start, self.end)
2546        }
2547    }
2548}
2549async fn add_all_recipients_as_contacts(
2550    context: &Context,
2551    session: &mut Session,
2552    folder: Config,
2553) -> Result<()> {
2554    let mailbox = if let Some(m) = context.get_config(folder).await? {
2555        m
2556    } else {
2557        info!(
2558            context,
2559            "Folder {} is not configured, skipping fetching contacts from it.", folder
2560        );
2561        return Ok(());
2562    };
2563    let create = false;
2564    let folder_exists = session
2565        .select_with_uidvalidity(context, &mailbox, create)
2566        .await
2567        .with_context(|| format!("could not select {mailbox}"))?;
2568    if !folder_exists {
2569        return Ok(());
2570    }
2571
2572    let recipients = session
2573        .get_all_recipients(context)
2574        .await
2575        .context("could not get recipients")?;
2576
2577    let mut any_modified = false;
2578    for recipient in recipients {
2579        let recipient_addr = match ContactAddress::new(&recipient.addr) {
2580            Err(err) => {
2581                warn!(
2582                    context,
2583                    "Could not add contact for recipient with address {:?}: {:#}",
2584                    recipient.addr,
2585                    err
2586                );
2587                continue;
2588            }
2589            Ok(recipient_addr) => recipient_addr,
2590        };
2591
2592        let (_, modified) = Contact::add_or_lookup(
2593            context,
2594            &recipient.display_name.unwrap_or_default(),
2595            &recipient_addr,
2596            Origin::OutgoingTo,
2597        )
2598        .await?;
2599        if modified != Modifier::None {
2600            any_modified = true;
2601        }
2602    }
2603    if any_modified {
2604        context.emit_event(EventType::ContactsChanged(None));
2605    }
2606
2607    Ok(())
2608}
2609
2610#[cfg(test)]
2611mod imap_tests;