deltachat/sql/
migrations.rs

1//! Migrations module.
2
3use std::collections::BTreeMap;
4use std::collections::BTreeSet;
5use std::time::Duration;
6
7use anyhow::{Context as _, Result, ensure};
8use deltachat_contact_tools::EmailAddress;
9use deltachat_contact_tools::addr_cmp;
10use pgp::composed::SignedPublicKey;
11use rusqlite::OptionalExtension;
12
13use crate::config::Config;
14use crate::configure::EnteredLoginParam;
15use crate::constants::ShowEmails;
16use crate::context::Context;
17use crate::key::DcKey;
18use crate::log::warn;
19use crate::message::MsgId;
20use crate::provider::get_provider_info;
21use crate::sql::Sql;
22use crate::tools::{Time, inc_and_check, time_elapsed};
23use crate::transport::ConfiguredLoginParam;
24
25const DBVERSION: i32 = 68;
26const VERSION_CFG: &str = "dbversion";
27const TABLES: &str = include_str!("./tables.sql");
28
29#[cfg(test)]
30tokio::task_local! {
31    static STOP_MIGRATIONS_AT: i32;
32}
33
34pub async fn run(context: &Context, sql: &Sql) -> Result<(bool, bool, bool)> {
35    let mut exists_before_update = false;
36    let mut dbversion_before_update = DBVERSION;
37
38    if !sql
39        .table_exists("config")
40        .await
41        .context("failed to check if config table exists")?
42    {
43        sql.transaction(move |transaction| {
44            transaction.execute_batch(TABLES)?;
45
46            // set raw config inside the transaction
47            transaction.execute(
48                "INSERT INTO config (keyname, value) VALUES (?, ?);",
49                (VERSION_CFG, format!("{dbversion_before_update}")),
50            )?;
51            Ok(())
52        })
53        .await
54        .context("Creating tables failed")?;
55
56        let mut lock = context.sql.config_cache.write().await;
57        lock.insert(
58            VERSION_CFG.to_string(),
59            Some(format!("{dbversion_before_update}")),
60        );
61        drop(lock);
62    } else {
63        exists_before_update = true;
64        dbversion_before_update = sql
65            .get_raw_config_int(VERSION_CFG)
66            .await?
67            .unwrap_or_default();
68    }
69
70    let dbversion = dbversion_before_update;
71    let mut update_icons = !exists_before_update;
72    let mut disable_server_delete = false;
73    let mut recode_avatar = false;
74
75    if dbversion < 1 {
76        sql.execute_migration(
77            r#"
78CREATE TABLE leftgrps ( id INTEGER PRIMARY KEY, grpid TEXT DEFAULT '');
79CREATE INDEX leftgrps_index1 ON leftgrps (grpid);"#,
80            1,
81        )
82        .await?;
83    }
84    if dbversion < 2 {
85        sql.execute_migration(
86            "ALTER TABLE contacts ADD COLUMN authname TEXT DEFAULT '';",
87            2,
88        )
89        .await?;
90    }
91    if dbversion < 7 {
92        sql.execute_migration(
93            "CREATE TABLE keypairs (\
94                 id INTEGER PRIMARY KEY, \
95                 addr TEXT DEFAULT '' COLLATE NOCASE, \
96                 is_default INTEGER DEFAULT 0, \
97                 private_key, \
98                 public_key, \
99                 created INTEGER DEFAULT 0);",
100            7,
101        )
102        .await?;
103    }
104    if dbversion < 10 {
105        sql.execute_migration(
106            "CREATE TABLE acpeerstates (\
107                 id INTEGER PRIMARY KEY, \
108                 addr TEXT DEFAULT '' COLLATE NOCASE, \
109                 last_seen INTEGER DEFAULT 0, \
110                 last_seen_autocrypt INTEGER DEFAULT 0, \
111                 public_key, \
112                 prefer_encrypted INTEGER DEFAULT 0); \
113              CREATE INDEX acpeerstates_index1 ON acpeerstates (addr);",
114            10,
115        )
116        .await?;
117    }
118    if dbversion < 12 {
119        sql.execute_migration(
120            r#"
121CREATE TABLE msgs_mdns ( msg_id INTEGER,  contact_id INTEGER);
122CREATE INDEX msgs_mdns_index1 ON msgs_mdns (msg_id);"#,
123            12,
124        )
125        .await?;
126    }
127    if dbversion < 17 {
128        sql.execute_migration(
129            r#"
130ALTER TABLE chats ADD COLUMN archived INTEGER DEFAULT 0;
131CREATE INDEX chats_index2 ON chats (archived);
132ALTER TABLE msgs ADD COLUMN starred INTEGER DEFAULT 0;
133CREATE INDEX msgs_index5 ON msgs (starred);"#,
134            17,
135        )
136        .await?;
137    }
138    if dbversion < 18 {
139        sql.execute_migration(
140            r#"
141ALTER TABLE acpeerstates ADD COLUMN gossip_timestamp INTEGER DEFAULT 0;
142ALTER TABLE acpeerstates ADD COLUMN gossip_key;"#,
143            18,
144        )
145        .await?;
146    }
147    if dbversion < 27 {
148        // chat.id=1 and chat.id=2 are the old deaddrops,
149        // the current ones are defined by chats.blocked=2
150        sql.execute_migration(
151            r#"
152DELETE FROM msgs WHERE chat_id=1 OR chat_id=2;
153CREATE INDEX chats_contacts_index2 ON chats_contacts (contact_id);
154ALTER TABLE msgs ADD COLUMN timestamp_sent INTEGER DEFAULT 0;
155ALTER TABLE msgs ADD COLUMN timestamp_rcvd INTEGER DEFAULT 0;"#,
156            27,
157        )
158        .await?;
159    }
160    if dbversion < 34 {
161        sql.execute_migration(
162            r#"
163ALTER TABLE msgs ADD COLUMN hidden INTEGER DEFAULT 0;
164ALTER TABLE msgs_mdns ADD COLUMN timestamp_sent INTEGER DEFAULT 0;
165ALTER TABLE acpeerstates ADD COLUMN public_key_fingerprint TEXT DEFAULT '';
166ALTER TABLE acpeerstates ADD COLUMN gossip_key_fingerprint TEXT DEFAULT '';
167CREATE INDEX acpeerstates_index3 ON acpeerstates (public_key_fingerprint);
168CREATE INDEX acpeerstates_index4 ON acpeerstates (gossip_key_fingerprint);"#,
169            34,
170        )
171        .await?;
172    }
173    if dbversion < 39 {
174        sql.execute_migration(
175            r#"
176CREATE TABLE tokens ( 
177  id INTEGER PRIMARY KEY, 
178  namespc INTEGER DEFAULT 0, 
179  foreign_id INTEGER DEFAULT 0, 
180  token TEXT DEFAULT '', 
181  timestamp INTEGER DEFAULT 0
182);
183ALTER TABLE acpeerstates ADD COLUMN verified_key;
184ALTER TABLE acpeerstates ADD COLUMN verified_key_fingerprint TEXT DEFAULT '';
185CREATE INDEX acpeerstates_index5 ON acpeerstates (verified_key_fingerprint);"#,
186            39,
187        )
188        .await?;
189    }
190    if dbversion < 40 {
191        sql.execute_migration("ALTER TABLE jobs ADD COLUMN thread INTEGER DEFAULT 0;", 40)
192            .await?;
193    }
194    if dbversion < 44 {
195        sql.execute_migration("ALTER TABLE msgs ADD COLUMN mime_headers TEXT;", 44)
196            .await?;
197    }
198    if dbversion < 46 {
199        sql.execute_migration(
200            r#"
201ALTER TABLE msgs ADD COLUMN mime_in_reply_to TEXT;
202ALTER TABLE msgs ADD COLUMN mime_references TEXT;"#,
203            46,
204        )
205        .await?;
206    }
207    if dbversion < 47 {
208        sql.execute_migration("ALTER TABLE jobs ADD COLUMN tries INTEGER DEFAULT 0;", 47)
209            .await?;
210    }
211    if dbversion < 48 {
212        // NOTE: move_state is not used anymore
213        sql.execute_migration(
214            "ALTER TABLE msgs ADD COLUMN move_state INTEGER DEFAULT 1;",
215            48,
216        )
217        .await?;
218    }
219    if dbversion < 49 {
220        sql.execute_migration(
221            "ALTER TABLE chats ADD COLUMN gossiped_timestamp INTEGER DEFAULT 0;",
222            49,
223        )
224        .await?;
225    }
226    if dbversion < 50 {
227        // installations <= 0.100.1 used DC_SHOW_EMAILS_ALL implicitly;
228        // keep this default and use DC_SHOW_EMAILS_NO
229        // only for new installations
230        if exists_before_update {
231            sql.set_raw_config_int("show_emails", ShowEmails::All as i32)
232                .await?;
233        }
234        sql.set_db_version(50).await?;
235    }
236    if dbversion < 53 {
237        // the messages containing _only_ locations
238        // are also added to the database as _hidden_.
239        sql.execute_migration(
240            r#"
241CREATE TABLE locations ( 
242  id INTEGER PRIMARY KEY AUTOINCREMENT, 
243  latitude REAL DEFAULT 0.0, 
244  longitude REAL DEFAULT 0.0, 
245  accuracy REAL DEFAULT 0.0, 
246  timestamp INTEGER DEFAULT 0, 
247  chat_id INTEGER DEFAULT 0, 
248  from_id INTEGER DEFAULT 0
249);"
250CREATE INDEX locations_index1 ON locations (from_id);
251CREATE INDEX locations_index2 ON locations (timestamp);
252ALTER TABLE chats ADD COLUMN locations_send_begin INTEGER DEFAULT 0;
253ALTER TABLE chats ADD COLUMN locations_send_until INTEGER DEFAULT 0;
254ALTER TABLE chats ADD COLUMN locations_last_sent INTEGER DEFAULT 0;
255CREATE INDEX chats_index3 ON chats (locations_send_until);"#,
256            53,
257        )
258        .await?;
259    }
260    if dbversion < 54 {
261        sql.execute_migration(
262            r#"
263ALTER TABLE msgs ADD COLUMN location_id INTEGER DEFAULT 0;
264CREATE INDEX msgs_index6 ON msgs (location_id);"#,
265            54,
266        )
267        .await?;
268    }
269    if dbversion < 55 {
270        sql.execute_migration(
271            "ALTER TABLE locations ADD COLUMN independent INTEGER DEFAULT 0;",
272            55,
273        )
274        .await?;
275    }
276    if dbversion < 59 {
277        // records in the devmsglabels are kept when the message is deleted.
278        // so, msg_id may or may not exist.
279        sql.execute_migration(
280            r#"
281CREATE TABLE devmsglabels (id INTEGER PRIMARY KEY AUTOINCREMENT, label TEXT, msg_id INTEGER DEFAULT 0);
282CREATE INDEX devmsglabels_index1 ON devmsglabels (label);"#, 59)
283            .await?;
284        if exists_before_update && sql.get_raw_config_int("bcc_self").await?.is_none() {
285            sql.set_raw_config_int("bcc_self", 1).await?;
286        }
287    }
288
289    if dbversion < 60 {
290        sql.execute_migration(
291            "ALTER TABLE chats ADD COLUMN created_timestamp INTEGER DEFAULT 0;",
292            60,
293        )
294        .await?;
295    }
296    if dbversion < 61 {
297        sql.execute_migration(
298            "ALTER TABLE contacts ADD COLUMN selfavatar_sent INTEGER DEFAULT 0;",
299            61,
300        )
301        .await?;
302        update_icons = true;
303    }
304    if dbversion < 62 {
305        sql.execute_migration(
306            "ALTER TABLE chats ADD COLUMN muted_until INTEGER DEFAULT 0;",
307            62,
308        )
309        .await?;
310    }
311    if dbversion < 63 {
312        sql.execute_migration("UPDATE chats SET grpid='' WHERE type=100", 63)
313            .await?;
314    }
315    if dbversion < 64 {
316        sql.execute_migration("ALTER TABLE msgs ADD COLUMN error TEXT DEFAULT '';", 64)
317            .await?;
318    }
319    if dbversion < 65 {
320        sql.execute_migration(
321            r#"
322ALTER TABLE chats ADD COLUMN ephemeral_timer INTEGER;
323ALTER TABLE msgs ADD COLUMN ephemeral_timer INTEGER DEFAULT 0;
324ALTER TABLE msgs ADD COLUMN ephemeral_timestamp INTEGER DEFAULT 0;"#,
325            65,
326        )
327        .await?;
328    }
329    if dbversion < 66 {
330        update_icons = true;
331        sql.set_db_version(66).await?;
332    }
333    if dbversion < 67 {
334        for prefix in &["", "configured_"] {
335            if let Some(server_flags) = sql
336                .get_raw_config_int(&format!("{prefix}server_flags"))
337                .await?
338            {
339                let imap_socket_flags = server_flags & 0x700;
340                let key = &format!("{prefix}mail_security");
341                match imap_socket_flags {
342                    0x100 => sql.set_raw_config_int(key, 2).await?, // STARTTLS
343                    0x200 => sql.set_raw_config_int(key, 1).await?, // SSL/TLS
344                    0x400 => sql.set_raw_config_int(key, 3).await?, // Plain
345                    _ => sql.set_raw_config_int(key, 0).await?,
346                }
347                let smtp_socket_flags = server_flags & 0x70000;
348                let key = &format!("{prefix}send_security");
349                match smtp_socket_flags {
350                    0x10000 => sql.set_raw_config_int(key, 2).await?, // STARTTLS
351                    0x20000 => sql.set_raw_config_int(key, 1).await?, // SSL/TLS
352                    0x40000 => sql.set_raw_config_int(key, 3).await?, // Plain
353                    _ => sql.set_raw_config_int(key, 0).await?,
354                }
355            }
356        }
357        sql.set_db_version(67).await?;
358    }
359    if dbversion < 68 {
360        // the index is used to speed up get_fresh_msg_cnt() (see comment there for more details) and marknoticed_chat()
361        sql.execute_migration(
362            "CREATE INDEX IF NOT EXISTS msgs_index7 ON msgs (state, hidden, chat_id);",
363            68,
364        )
365        .await?;
366    }
367    if dbversion < 69 {
368        sql.execute_migration(
369            r#"
370ALTER TABLE chats ADD COLUMN protected INTEGER DEFAULT 0;
371-- 120=group, 130=old verified group
372UPDATE chats SET protected=1, type=120 WHERE type=130;"#,
373            69,
374        )
375        .await?;
376    }
377
378    if dbversion < 71 {
379        if let Ok(addr) = context.get_primary_self_addr().await {
380            if let Ok(domain) = EmailAddress::new(&addr).map(|email| email.domain) {
381                context
382                    .set_config_internal(
383                        Config::ConfiguredProvider,
384                        get_provider_info(&domain).map(|provider| provider.id),
385                    )
386                    .await?;
387            } else {
388                warn!(context, "Can't parse configured address: {:?}", addr);
389            }
390        }
391
392        sql.set_db_version(71).await?;
393    }
394    if dbversion < 72 && !sql.col_exists("msgs", "mime_modified").await? {
395        sql.execute_migration(
396            r#"
397    ALTER TABLE msgs ADD COLUMN mime_modified INTEGER DEFAULT 0;"#,
398            72,
399        )
400        .await?;
401    }
402    if dbversion < 73 {
403        sql.execute(
404            r#"
405CREATE TABLE imap_sync (folder TEXT PRIMARY KEY, uidvalidity INTEGER DEFAULT 0, uid_next INTEGER DEFAULT 0);"#,
406()
407        )
408            .await?;
409        for c in [
410            "configured_inbox_folder",
411            "configured_sentbox_folder",
412            "configured_mvbox_folder",
413        ] {
414            if let Some(folder) = context.sql.get_raw_config(c).await? {
415                let key = format!("imap.mailbox.{folder}");
416
417                let (uid_validity, last_seen_uid) =
418                    if let Some(entry) = context.sql.get_raw_config(&key).await? {
419                        // the entry has the format `imap.mailbox.<folder>=<uidvalidity>:<lastseenuid>`
420                        let mut parts = entry.split(':');
421                        (
422                            parts.next().unwrap_or_default().parse().unwrap_or(0),
423                            parts.next().unwrap_or_default().parse().unwrap_or(0),
424                        )
425                    } else {
426                        (0, 0)
427                    };
428                if last_seen_uid > 0 {
429                    context
430                        .sql
431                        .execute(
432                            "INSERT INTO imap_sync (folder, uid_next) VALUES (?,?)
433                             ON CONFLICT(folder) DO UPDATE SET uid_next=excluded.uid_next",
434                            (&folder, last_seen_uid + 1),
435                        )
436                        .await?;
437                    context
438                        .sql
439                        .execute(
440                            "INSERT INTO imap_sync (folder, uidvalidity) VALUES (?,?)
441                             ON CONFLICT(folder) DO UPDATE SET uidvalidity=excluded.uidvalidity",
442                            (&folder, uid_validity),
443                        )
444                        .await?;
445                }
446            }
447        }
448        if exists_before_update {
449            disable_server_delete = true;
450
451            // Don't disable server delete if it was on by default (Nauta):
452            if let Some(provider) = context.get_configured_provider().await?
453                && let Some(defaults) = &provider.config_defaults
454                && defaults.iter().any(|d| d.key == Config::DeleteServerAfter)
455            {
456                disable_server_delete = false;
457            }
458        }
459        sql.set_db_version(73).await?;
460    }
461    if dbversion < 74 {
462        sql.execute_migration("UPDATE contacts SET name='' WHERE name=authname", 74)
463            .await?;
464    }
465    if dbversion < 75 {
466        sql.execute_migration(
467            "ALTER TABLE contacts ADD COLUMN status TEXT DEFAULT '';",
468            75,
469        )
470        .await?;
471    }
472    if dbversion < 76 {
473        sql.execute_migration("ALTER TABLE msgs ADD COLUMN subject TEXT DEFAULT '';", 76)
474            .await?;
475    }
476    if dbversion < 77 {
477        recode_avatar = true;
478        sql.set_db_version(77).await?;
479    }
480    if dbversion < 78 {
481        // move requests to "Archived Chats",
482        // this way, the app looks familiar after the contact request upgrade.
483        sql.execute_migration("UPDATE chats SET archived=1 WHERE blocked=2;", 78)
484            .await?;
485    }
486    if dbversion < 79 {
487        sql.execute_migration(
488            r#"
489        ALTER TABLE msgs ADD COLUMN download_state INTEGER DEFAULT 0;
490        "#,
491            79,
492        )
493        .await?;
494    }
495    if dbversion < 80 {
496        sql.execute_migration(
497            r#"CREATE TABLE multi_device_sync (
498id INTEGER PRIMARY KEY AUTOINCREMENT,
499item TEXT DEFAULT '');"#,
500            80,
501        )
502        .await?;
503    }
504    if dbversion < 81 {
505        sql.execute_migration("ALTER TABLE msgs ADD COLUMN hop_info TEXT;", 81)
506            .await?;
507    }
508    if dbversion < 82 {
509        sql.execute_migration(
510            r#"CREATE TABLE imap (
511id INTEGER PRIMARY KEY AUTOINCREMENT,
512rfc724_mid TEXT DEFAULT '', -- Message-ID header
513folder TEXT DEFAULT '', -- IMAP folder
514target TEXT DEFAULT '', -- Destination folder, empty to delete.
515uid INTEGER DEFAULT 0, -- UID
516uidvalidity INTEGER DEFAULT 0,
517UNIQUE (folder, uid, uidvalidity)
518);
519CREATE INDEX imap_folder ON imap(folder);
520CREATE INDEX imap_messageid ON imap(rfc724_mid);
521
522INSERT INTO imap
523(rfc724_mid, folder, target, uid, uidvalidity)
524SELECT
525rfc724_mid,
526server_folder AS folder,
527server_folder AS target,
528server_uid AS uid,
529(SELECT uidvalidity FROM imap_sync WHERE folder=server_folder) AS uidvalidity
530FROM msgs
531WHERE server_uid>0
532ON CONFLICT (folder, uid, uidvalidity)
533DO UPDATE SET rfc724_mid=excluded.rfc724_mid,
534              target=excluded.target;
535"#,
536            82,
537        )
538        .await?;
539    }
540    if dbversion < 83 {
541        sql.execute_migration(
542            "ALTER TABLE imap_sync
543             ADD COLUMN modseq -- Highest modification sequence
544             INTEGER DEFAULT 0",
545            83,
546        )
547        .await?;
548    }
549    if dbversion < 84 {
550        sql.execute_migration(
551            r#"CREATE TABLE msgs_status_updates (
552id INTEGER PRIMARY KEY AUTOINCREMENT,
553msg_id INTEGER,
554update_item TEXT DEFAULT '',
555update_item_read INTEGER DEFAULT 0 -- XXX unused
556);
557CREATE INDEX msgs_status_updates_index1 ON msgs_status_updates (msg_id);"#,
558            84,
559        )
560        .await?;
561    }
562    if dbversion < 85 {
563        sql.execute_migration(
564            r#"CREATE TABLE smtp (
565id INTEGER PRIMARY KEY,
566rfc724_mid TEXT NOT NULL,          -- Message-ID
567mime TEXT NOT NULL,                -- SMTP payload
568msg_id INTEGER NOT NULL,           -- ID of the message in `msgs` table
569recipients TEXT NOT NULL,          -- List of recipients separated by space
570retries INTEGER NOT NULL DEFAULT 0 -- Number of failed attempts to send the message
571);
572CREATE INDEX smtp_messageid ON imap(rfc724_mid);
573"#,
574            85,
575        )
576        .await?;
577    }
578    if dbversion < 86 {
579        sql.execute_migration(
580            r#"CREATE TABLE bobstate (
581                   id INTEGER PRIMARY KEY AUTOINCREMENT,
582                   invite TEXT NOT NULL,
583                   next_step INTEGER NOT NULL,
584                   chat_id INTEGER NOT NULL
585            );"#,
586            86,
587        )
588        .await?;
589    }
590    if dbversion < 87 {
591        // the index is used to speed up delete_expired_messages()
592        sql.execute_migration(
593            "CREATE INDEX IF NOT EXISTS msgs_index8 ON msgs (ephemeral_timestamp);",
594            87,
595        )
596        .await?;
597    }
598    if dbversion < 88 {
599        sql.execute_migration("DROP TABLE IF EXISTS backup_blobs;", 88)
600            .await?;
601    }
602    if dbversion < 89 {
603        sql.execute_migration(
604            r#"CREATE TABLE imap_markseen (
605              id INTEGER,
606              FOREIGN KEY(id) REFERENCES imap(id) ON DELETE CASCADE
607            );"#,
608            89,
609        )
610        .await?;
611    }
612    if dbversion < 90 {
613        sql.execute_migration(
614            r#"CREATE TABLE smtp_mdns (
615              msg_id INTEGER NOT NULL, -- id of the message in msgs table which requested MDN (DEPRECATED 2024-06-21)
616              from_id INTEGER NOT NULL, -- id of the contact that sent the message, MDN destination
617              rfc724_mid TEXT NOT NULL, -- Message-ID header
618              retries INTEGER NOT NULL DEFAULT 0 -- Number of failed attempts to send MDN
619            );"#,
620            90,
621        )
622        .await?;
623    }
624    if dbversion < 91 {
625        sql.execute_migration(
626            r#"CREATE TABLE smtp_status_updates (
627              msg_id INTEGER NOT NULL UNIQUE, -- msg_id of the webxdc instance with pending updates
628              first_serial INTEGER NOT NULL, -- id in msgs_status_updates
629              last_serial INTEGER NOT NULL, -- id in msgs_status_updates
630              descr TEXT NOT NULL -- text to send along with the updates
631            );"#,
632            91,
633        )
634        .await?;
635    }
636    if dbversion < 92 {
637        sql.execute_migration(
638            r#"CREATE TABLE reactions (
639              msg_id INTEGER NOT NULL, -- id of the message reacted to
640              contact_id INTEGER NOT NULL, -- id of the contact reacting to the message
641              reaction TEXT DEFAULT '' NOT NULL, -- a sequence of emojis separated by spaces
642              PRIMARY KEY(msg_id, contact_id),
643              FOREIGN KEY(msg_id) REFERENCES msgs(id) ON DELETE CASCADE -- delete reactions when message is deleted
644              FOREIGN KEY(contact_id) REFERENCES contacts(id) ON DELETE CASCADE -- delete reactions when contact is deleted
645            )"#,
646            92
647        ).await?;
648    }
649    if dbversion < 93 {
650        // `sending_domains` is now unused, but was not removed for backwards compatibility.
651        sql.execute_migration(
652            "CREATE TABLE sending_domains(domain TEXT PRIMARY KEY, dkim_works INTEGER DEFAULT 0);",
653            93,
654        )
655        .await?;
656    }
657    if dbversion < 94 {
658        sql.execute_migration(
659            // Create new `acpeerstates` table, same as before but with unique constraint.
660            //
661            // This allows to use `UPSERT` to update existing or insert a new peerstate
662            // depending on whether one exists already.
663            "CREATE TABLE new_acpeerstates (
664             id INTEGER PRIMARY KEY,
665             addr TEXT DEFAULT '' COLLATE NOCASE,
666             last_seen INTEGER DEFAULT 0,
667             last_seen_autocrypt INTEGER DEFAULT 0,
668             public_key,
669             prefer_encrypted INTEGER DEFAULT 0,
670             gossip_timestamp INTEGER DEFAULT 0,
671             gossip_key,
672             public_key_fingerprint TEXT DEFAULT '',
673             gossip_key_fingerprint TEXT DEFAULT '',
674             verified_key,
675             verified_key_fingerprint TEXT DEFAULT '',
676             UNIQUE (addr) -- Only one peerstate per address
677             );
678            INSERT OR IGNORE INTO new_acpeerstates SELECT
679                id, addr, last_seen, last_seen_autocrypt, public_key, prefer_encrypted,
680                gossip_timestamp, gossip_key, public_key_fingerprint,
681                gossip_key_fingerprint, verified_key, verified_key_fingerprint
682            FROM acpeerstates;
683            DROP TABLE acpeerstates;
684            ALTER TABLE new_acpeerstates RENAME TO acpeerstates;
685            CREATE INDEX acpeerstates_index1 ON acpeerstates (addr);
686            CREATE INDEX acpeerstates_index3 ON acpeerstates (public_key_fingerprint);
687            CREATE INDEX acpeerstates_index4 ON acpeerstates (gossip_key_fingerprint);
688            CREATE INDEX acpeerstates_index5 ON acpeerstates (verified_key_fingerprint);
689            ",
690            94,
691        )
692        .await?;
693    }
694    if dbversion < 95 {
695        sql.execute_migration(
696            "CREATE TABLE new_chats_contacts (chat_id INTEGER, contact_id INTEGER, UNIQUE(chat_id, contact_id));\
697            INSERT OR IGNORE INTO new_chats_contacts SELECT chat_id, contact_id FROM chats_contacts;\
698            DROP TABLE chats_contacts;\
699            ALTER TABLE new_chats_contacts RENAME TO chats_contacts;\
700            CREATE INDEX chats_contacts_index1 ON chats_contacts (chat_id);\
701            CREATE INDEX chats_contacts_index2 ON chats_contacts (contact_id);",
702            95
703        ).await?;
704    }
705    if dbversion < 96 {
706        sql.execute_migration(
707            "ALTER TABLE acpeerstates ADD COLUMN verifier TEXT DEFAULT '';",
708            96,
709        )
710        .await?;
711    }
712    if dbversion < 97 {
713        sql.execute_migration(
714            "CREATE TABLE dns_cache (
715               hostname TEXT NOT NULL,
716               address TEXT NOT NULL, -- IPv4 or IPv6 address
717               timestamp INTEGER NOT NULL,
718               UNIQUE (hostname, address)
719             )",
720            97,
721        )
722        .await?;
723    }
724    if dbversion < 98 {
725        if exists_before_update && sql.get_raw_config_int("show_emails").await?.is_none() {
726            sql.set_raw_config_int("show_emails", ShowEmails::Off as i32)
727                .await?;
728        }
729        sql.set_db_version(98).await?;
730    }
731    if dbversion < 99 {
732        // sql.execute_migration(
733        //     "ALTER TABLE msgs DROP COLUMN server_folder;
734        //      ALTER TABLE msgs DROP COLUMN server_uid;
735        //      ALTER TABLE msgs DROP COLUMN move_state;
736        //      ALTER TABLE chats DROP COLUMN draft_timestamp;
737        //      ALTER TABLE chats DROP COLUMN draft_txt",
738        //     99,
739        // )
740        // .await?;
741
742        // Reverted above, as it requires to load the whole DB in memory.
743        sql.set_db_version(99).await?;
744    }
745    if dbversion < 100 {
746        sql.execute_migration(
747            "ALTER TABLE msgs ADD COLUMN mime_compressed INTEGER NOT NULL DEFAULT 0",
748            100,
749        )
750        .await?;
751    }
752    if dbversion < 101 {
753        // Recreate `smtp` table with autoincrement.
754        // rfc724_mid index is not recreated, because it is not used.
755        sql.execute_migration(
756            "DROP TABLE smtp;
757             CREATE TABLE smtp (
758             id INTEGER PRIMARY KEY AUTOINCREMENT,
759             rfc724_mid TEXT NOT NULL,          -- Message-ID
760             mime TEXT NOT NULL,                -- SMTP payload
761             msg_id INTEGER NOT NULL,           -- ID of the message in `msgs` table
762             recipients TEXT NOT NULL,          -- List of recipients separated by space
763             retries INTEGER NOT NULL DEFAULT 0 -- Number of failed attempts to send the message
764            );
765            ",
766            101,
767        )
768        .await?;
769    }
770
771    if dbversion < 102 {
772        sql.execute_migration(
773            "CREATE TABLE download (
774            msg_id INTEGER NOT NULL -- id of the message stub in msgs table
775            )",
776            102,
777        )
778        .await?;
779    }
780
781    // Add is_bot column to contacts table with default false.
782    if dbversion < 103 {
783        sql.execute_migration(
784            "ALTER TABLE contacts ADD COLUMN is_bot INTEGER NOT NULL DEFAULT 0",
785            103,
786        )
787        .await?;
788    }
789
790    if dbversion < 104 {
791        sql.execute_migration(
792            "ALTER TABLE acpeerstates
793             ADD COLUMN secondary_verified_key;
794             ALTER TABLE acpeerstates
795             ADD COLUMN secondary_verified_key_fingerprint TEXT DEFAULT '';
796             ALTER TABLE acpeerstates
797             ADD COLUMN secondary_verifier TEXT DEFAULT ''",
798            104,
799        )
800        .await?;
801    }
802
803    if dbversion < 105 {
804        // Create UNIQUE uid column and drop unused update_item_read column.
805        sql.execute_migration(
806            r#"CREATE TABLE new_msgs_status_updates (
807id INTEGER PRIMARY KEY AUTOINCREMENT,
808msg_id INTEGER,
809update_item TEXT DEFAULT '',
810uid TEXT UNIQUE
811);
812INSERT OR IGNORE INTO new_msgs_status_updates SELECT
813  id, msg_id, update_item, NULL
814FROM msgs_status_updates;
815DROP TABLE msgs_status_updates;
816ALTER TABLE new_msgs_status_updates RENAME TO msgs_status_updates;
817CREATE INDEX msgs_status_updates_index1 ON msgs_status_updates (msg_id);
818CREATE INDEX msgs_status_updates_index2 ON msgs_status_updates (uid);
819"#,
820            105,
821        )
822        .await?;
823    }
824
825    if dbversion < 106 {
826        // Recreate `config` table with UNIQUE constraint on `keyname`.
827        sql.execute_migration(
828            "CREATE TABLE new_config (
829               id INTEGER PRIMARY KEY,
830               keyname TEXT UNIQUE,
831               value TEXT NOT NULL
832             );
833             INSERT OR IGNORE INTO new_config SELECT
834               id, keyname, value
835             FROM config;
836             DROP TABLE config;
837             ALTER TABLE new_config RENAME TO config;
838             CREATE INDEX config_index1 ON config (keyname);",
839            106,
840        )
841        .await?;
842    }
843
844    if dbversion < 107 {
845        sql.execute_migration(
846            "CREATE TABLE new_keypairs (
847               id INTEGER PRIMARY KEY AUTOINCREMENT,
848               private_key UNIQUE NOT NULL,
849               public_key UNIQUE NOT NULL
850             );
851             INSERT OR IGNORE INTO new_keypairs SELECT id, private_key, public_key FROM keypairs;
852
853             INSERT OR IGNORE
854             INTO config (keyname, value)
855             VALUES
856             ('key_id', (SELECT id FROM new_keypairs
857                         WHERE private_key=
858                           (SELECT private_key FROM keypairs
859                            WHERE addr=(SELECT value FROM config WHERE keyname='configured_addr')
860                            AND is_default=1)));
861
862             -- We do not drop the old `keypairs` table for now,
863             -- but move it to `old_keypairs`. We can remove it later
864             -- in next migrations. This may be needed for recovery
865             -- in case something is wrong with the migration.
866             ALTER TABLE keypairs RENAME TO old_keypairs;
867             ALTER TABLE new_keypairs RENAME TO keypairs;
868             ",
869            107,
870        )
871        .await?;
872    }
873
874    if dbversion < 108 {
875        let version = 108;
876        let chunk_size = context.get_max_smtp_rcpt_to().await?;
877        sql.transaction(move |trans| {
878            Sql::set_db_version_trans(trans, version)?;
879            let id_max =
880                trans.query_row("SELECT IFNULL((SELECT MAX(id) FROM smtp), 0)", (), |row| {
881                    let id_max: i64 = row.get(0)?;
882                    Ok(id_max)
883                })?;
884            while let Some((id, rfc724_mid, mime, msg_id, recipients, retries)) = trans
885                .query_row(
886                    "SELECT id, rfc724_mid, mime, msg_id, recipients, retries FROM smtp \
887                    WHERE id<=? LIMIT 1",
888                    (id_max,),
889                    |row| {
890                        let id: i64 = row.get(0)?;
891                        let rfc724_mid: String = row.get(1)?;
892                        let mime: String = row.get(2)?;
893                        let msg_id: MsgId = row.get(3)?;
894                        let recipients: String = row.get(4)?;
895                        let retries: i64 = row.get(5)?;
896                        Ok((id, rfc724_mid, mime, msg_id, recipients, retries))
897                    },
898                )
899                .optional()?
900            {
901                trans.execute("DELETE FROM smtp WHERE id=?", (id,))?;
902                let recipients = recipients.split(' ').collect::<Vec<_>>();
903                for recipients in recipients.chunks(chunk_size) {
904                    let recipients = recipients.join(" ");
905                    trans.execute(
906                        "INSERT INTO smtp (rfc724_mid, mime, msg_id, recipients, retries) \
907                        VALUES (?, ?, ?, ?, ?)",
908                        (&rfc724_mid, &mime, msg_id, recipients, retries),
909                    )?;
910                }
911            }
912            Ok(())
913        })
914        .await
915        .with_context(|| format!("migration failed for version {version}"))?;
916
917        sql.set_db_version_in_cache(version).await?;
918    }
919
920    if dbversion < 109 {
921        sql.execute_migration(
922            r#"ALTER TABLE acpeerstates
923               ADD COLUMN backward_verified_key_id -- What we think the contact has as our verified key
924               INTEGER;
925               UPDATE acpeerstates
926               SET backward_verified_key_id=(SELECT value FROM config WHERE keyname='key_id')
927               WHERE verified_key IS NOT NULL
928               "#,
929            109,
930        )
931        .await?;
932    }
933
934    if dbversion < 110 {
935        sql.execute_migration(
936            "ALTER TABLE keypairs ADD COLUMN addr TEXT DEFAULT '' COLLATE NOCASE;
937            ALTER TABLE keypairs ADD COLUMN is_default INTEGER DEFAULT 0;
938            ALTER TABLE keypairs ADD COLUMN created INTEGER DEFAULT 0;
939            UPDATE keypairs SET addr=(SELECT value FROM config WHERE keyname='configured_addr'), is_default=1;",
940            110,
941        )
942        .await?;
943    }
944
945    if dbversion < 111 {
946        sql.execute_migration(
947            "CREATE TABLE iroh_gossip_peers (msg_id TEXT not NULL, topic TEXT NOT NULL, public_key TEXT NOT NULL)",
948            111,
949        )
950        .await?;
951    }
952
953    if dbversion < 112 {
954        sql.execute_migration(
955            "DROP TABLE iroh_gossip_peers; CREATE TABLE iroh_gossip_peers (msg_id INTEGER not NULL, topic BLOB NOT NULL, public_key BLOB NOT NULL, relay_server TEXT, UNIQUE (public_key, topic)) STRICT",
956            112,
957        )
958        .await?;
959    }
960
961    if dbversion < 113 {
962        sql.execute_migration(
963            "DROP TABLE iroh_gossip_peers; CREATE TABLE iroh_gossip_peers (msg_id INTEGER not NULL, topic BLOB NOT NULL, public_key BLOB NOT NULL, relay_server TEXT, UNIQUE (topic, public_key), PRIMARY KEY(topic, public_key)) STRICT",
964            113,
965        )
966        .await?;
967    }
968
969    if dbversion < 114 {
970        sql.execute_migration("CREATE INDEX reactions_index1 ON reactions (msg_id)", 114)
971            .await?;
972    }
973
974    if dbversion < 115 {
975        sql.execute_migration("ALTER TABLE msgs ADD COLUMN txt_normalized TEXT", 115)
976            .await?;
977    }
978    let mut migration_version: i32 = 115;
979
980    inc_and_check(&mut migration_version, 116)?;
981    if dbversion < migration_version {
982        // Whether the message part doesn't need to be stored on the server. If all parts are marked
983        // deleted, a server-side deletion is issued.
984        sql.execute_migration(
985            "ALTER TABLE msgs ADD COLUMN deleted INTEGER NOT NULL DEFAULT 0",
986            migration_version,
987        )
988        .await?;
989    }
990
991    inc_and_check(&mut migration_version, 117)?;
992    if dbversion < migration_version {
993        sql.execute_migration(
994            "CREATE TABLE connection_history (
995                host TEXT NOT NULL, -- server hostname
996                port INTEGER NOT NULL, -- server port
997                alpn TEXT NOT NULL, -- ALPN such as smtp or imap
998                addr TEXT NOT NULL, -- IP address
999                timestamp INTEGER NOT NULL, -- timestamp of the most recent successful connection
1000                UNIQUE (host, port, alpn, addr)
1001            ) STRICT",
1002            migration_version,
1003        )
1004        .await?;
1005    }
1006
1007    inc_and_check(&mut migration_version, 118)?;
1008    if dbversion < migration_version {
1009        sql.execute_migration(
1010            "CREATE TABLE tokens_new (
1011                id INTEGER PRIMARY KEY,
1012                namespc INTEGER DEFAULT 0,
1013                foreign_key TEXT DEFAULT '',
1014                token TEXT DEFAULT '',
1015                timestamp INTEGER DEFAULT 0
1016            ) STRICT;
1017            INSERT INTO tokens_new
1018                SELECT t.id, t.namespc, IFNULL(c.grpid, ''), t.token, t.timestamp
1019                FROM tokens t LEFT JOIN chats c ON t.foreign_id=c.id;
1020            DROP TABLE tokens;
1021            ALTER TABLE tokens_new RENAME TO tokens;",
1022            migration_version,
1023        )
1024        .await?;
1025    }
1026
1027    inc_and_check(&mut migration_version, 119)?;
1028    if dbversion < migration_version {
1029        sql.execute_migration(
1030            "CREATE TABLE imap_send (
1031                id INTEGER PRIMARY KEY AUTOINCREMENT,
1032                mime TEXT NOT NULL, -- Message content
1033                msg_id INTEGER NOT NULL, -- ID of the message in the `msgs` table
1034                attempts INTEGER NOT NULL DEFAULT 0 -- Number of failed attempts to send the message
1035            )",
1036            migration_version,
1037        )
1038        .await?;
1039    }
1040
1041    inc_and_check(&mut migration_version, 120)?;
1042    if dbversion < migration_version {
1043        // Core 1.143.0 changed the default for `delete_server_after`
1044        // to delete immediately (`1`) for chatmail accounts that don't have multidevice
1045        // and updating to `0` when backup is exported.
1046        //
1047        // Since we don't know if existing configurations
1048        // are multidevice, we set `delete_server_after` for them
1049        // to the old default of `0`, so only new configurations are
1050        // affected by the default change.
1051        //
1052        // `INSERT OR IGNORE` works
1053        // because `keyname` was made UNIQUE in migration 106.
1054        sql.execute_migration(
1055            "INSERT OR IGNORE INTO config (keyname, value)
1056             SELECT 'delete_server_after', '0'
1057             FROM config WHERE keyname='configured'
1058            ",
1059            migration_version,
1060        )
1061        .await?;
1062    }
1063
1064    inc_and_check(&mut migration_version, 121)?;
1065    if dbversion < migration_version {
1066        sql.execute_migration(
1067            "CREATE INDEX chats_index4 ON chats (name)",
1068            migration_version,
1069        )
1070        .await?;
1071    }
1072
1073    inc_and_check(&mut migration_version, 122)?;
1074    if dbversion < migration_version {
1075        sql.execute_migration(
1076            "ALTER TABLE tokens ADD COLUMN foreign_id INTEGER NOT NULL DEFAULT 0",
1077            migration_version,
1078        )
1079        .await?;
1080    }
1081
1082    inc_and_check(&mut migration_version, 123)?;
1083    if dbversion < migration_version {
1084        // Add FOREIGN KEY(msg_id).
1085        sql.execute_migration(
1086            "CREATE TABLE new_msgs_status_updates (
1087                id INTEGER PRIMARY KEY AUTOINCREMENT,
1088                msg_id INTEGER,
1089                update_item TEXT DEFAULT '',
1090                uid TEXT UNIQUE,
1091                FOREIGN KEY(msg_id) REFERENCES msgs(id) ON DELETE CASCADE
1092            );
1093            INSERT OR IGNORE INTO new_msgs_status_updates SELECT
1094                id, msg_id, update_item, uid
1095            FROM msgs_status_updates;
1096            DROP TABLE msgs_status_updates;
1097            ALTER TABLE new_msgs_status_updates RENAME TO msgs_status_updates;
1098            CREATE INDEX msgs_status_updates_index1 ON msgs_status_updates (msg_id);
1099            CREATE INDEX msgs_status_updates_index2 ON msgs_status_updates (uid);
1100            ",
1101            migration_version,
1102        )
1103        .await?;
1104    }
1105
1106    inc_and_check(&mut migration_version, 124)?;
1107    if dbversion < migration_version {
1108        // Mark Saved Messages chat as protected if it already exists.
1109        sql.execute_migration(
1110            "UPDATE chats
1111             SET protected=1 -- ProtectionStatus::Protected
1112             WHERE type==100 -- Chattype::Single
1113             AND EXISTS (
1114                 SELECT 1 FROM chats_contacts cc
1115                 WHERE cc.chat_id==chats.id
1116                 AND cc.contact_id=1
1117             )
1118             ",
1119            migration_version,
1120        )
1121        .await?;
1122    }
1123
1124    inc_and_check(&mut migration_version, 125)?;
1125    if dbversion < migration_version {
1126        sql.execute_migration(
1127            "CREATE TABLE http_cache (
1128                url TEXT PRIMARY KEY,
1129                expires INTEGER NOT NULL, -- When the cache entry is considered expired, timestamp in seconds.
1130                blobname TEXT NOT NULL,
1131                mimetype TEXT NOT NULL DEFAULT '', -- MIME type extracted from Content-Type header.
1132                encoding TEXT NOT NULL DEFAULT '' -- Encoding from Content-Type header.
1133            ) STRICT",
1134            migration_version,
1135        )
1136        .await?;
1137    }
1138
1139    inc_and_check(&mut migration_version, 126)?;
1140    if dbversion < migration_version {
1141        // Recreate http_cache table with new `stale` column.
1142        sql.execute_migration(
1143            "DROP TABLE http_cache;
1144             CREATE TABLE http_cache (
1145                url TEXT PRIMARY KEY,
1146                expires INTEGER NOT NULL, -- When the cache entry is considered expired, timestamp in seconds.
1147                stale INTEGER NOT NULL, -- When the cache entry is considered stale, timestamp in seconds.
1148                blobname TEXT NOT NULL,
1149                mimetype TEXT NOT NULL DEFAULT '', -- MIME type extracted from Content-Type header.
1150                encoding TEXT NOT NULL DEFAULT '' -- Encoding from Content-Type header.
1151            ) STRICT",
1152            migration_version,
1153        )
1154        .await?;
1155    }
1156
1157    inc_and_check(&mut migration_version, 127)?;
1158    if dbversion < migration_version {
1159        // This is buggy: `delete_server_after` > 1 isn't handled. Migration #129 fixes this.
1160        sql.execute_migration(
1161            "INSERT OR IGNORE INTO config (keyname, value)
1162             SELECT 'bcc_self', '1'
1163             FROM config WHERE keyname='delete_server_after' AND value='0'
1164            ",
1165            migration_version,
1166        )
1167        .await?;
1168    }
1169
1170    inc_and_check(&mut migration_version, 128)?;
1171    if dbversion < migration_version {
1172        // Add the timestamps of addition and removal.
1173        //
1174        // If `add_timestamp >= remove_timestamp`,
1175        // then the member is currently a member of the chat.
1176        // Otherwise the member is a past member.
1177        sql.execute_migration(
1178            "ALTER TABLE chats_contacts
1179             ADD COLUMN add_timestamp NOT NULL DEFAULT 0;
1180             ALTER TABLE chats_contacts
1181             ADD COLUMN remove_timestamp NOT NULL DEFAULT 0;
1182            ",
1183            migration_version,
1184        )
1185        .await?;
1186    }
1187
1188    inc_and_check(&mut migration_version, 129)?;
1189    if dbversion < migration_version {
1190        // Existing chatmail configurations having `delete_server_after` != "delete at once" should
1191        // get `bcc_self` enabled, they may be multidevice configurations:
1192        // - Before migration #127, `delete_server_after` was set to 0 upon a backup export, but
1193        //   then `bcc_self` is enabled instead (whose default is changed to 0 for chatmail).
1194        // - The user might set `delete_server_after` to a value other than 0 or 1 when that was
1195        //   possible in UIs.
1196        // We don't check `is_chatmail` for simplicity.
1197        sql.execute_migration(
1198            "INSERT OR IGNORE INTO config (keyname, value)
1199             SELECT 'bcc_self', '1'
1200             FROM config WHERE keyname='delete_server_after' AND value!='1'
1201            ",
1202            migration_version,
1203        )
1204        .await?;
1205    }
1206
1207    inc_and_check(&mut migration_version, 130)?;
1208    if dbversion < migration_version {
1209        sql.execute_migration(
1210            "
1211CREATE TABLE gossip_timestamp (
1212  chat_id INTEGER NOT NULL, 
1213  fingerprint TEXT NOT NULL, -- Upper-case fingerprint of the key.
1214  timestamp INTEGER NOT NULL,
1215  UNIQUE (chat_id, fingerprint)
1216) STRICT;
1217CREATE INDEX gossip_timestamp_index ON gossip_timestamp (chat_id, fingerprint);
1218",
1219            migration_version,
1220        )
1221        .await?;
1222    }
1223
1224    inc_and_check(&mut migration_version, 131)?;
1225    if dbversion < migration_version {
1226        let entered_param = EnteredLoginParam::load(context).await?;
1227        let configured_param = ConfiguredLoginParam::load_legacy(context).await?;
1228
1229        sql.execute_migration_transaction(
1230            |transaction| {
1231                transaction.execute(
1232                    "CREATE TABLE transports (
1233                        id INTEGER PRIMARY KEY AUTOINCREMENT,
1234                        addr TEXT NOT NULL,
1235                        entered_param TEXT NOT NULL,
1236                        configured_param TEXT NOT NULL,
1237                        UNIQUE(addr)
1238                        )",
1239                    (),
1240                )?;
1241                if let Some(configured_param) = configured_param {
1242                    transaction.execute(
1243                        "INSERT INTO transports (addr, entered_param, configured_param)
1244                         VALUES (?, ?, ?)",
1245                        (
1246                            configured_param.addr.clone(),
1247                            serde_json::to_string(&entered_param)?,
1248                            configured_param.into_json()?,
1249                        ),
1250                    )?;
1251                }
1252
1253                Ok(())
1254            },
1255            migration_version,
1256        )
1257        .await?;
1258    }
1259
1260    inc_and_check(&mut migration_version, 132)?;
1261    if dbversion < migration_version {
1262        let start = Time::now();
1263        sql.execute_migration_transaction(|t| migrate_key_contacts(context, t), migration_version)
1264            .await?;
1265        info!(
1266            context,
1267            "key-contacts migration took {:?} in total.",
1268            time_elapsed(&start),
1269        );
1270        // Schedule `msgs_to_key_contacts()`.
1271        context
1272            .set_config_internal(Config::LastHousekeeping, None)
1273            .await?;
1274    }
1275
1276    inc_and_check(&mut migration_version, 133)?;
1277    if dbversion < migration_version {
1278        // Make `ProtectionBroken` chats `Unprotected`. Chats can't break anymore.
1279        sql.execute_migration(
1280            "UPDATE chats SET protected=0 WHERE protected!=1",
1281            migration_version,
1282        )
1283        .await?;
1284    }
1285
1286    inc_and_check(&mut migration_version, 134)?; // Migration 134 was removed
1287
1288    inc_and_check(&mut migration_version, 135)?;
1289    if dbversion < migration_version {
1290        sql.execute_migration(
1291            "CREATE TABLE stats_securejoin_sources(
1292                source INTEGER PRIMARY KEY,
1293                count INTEGER NOT NULL DEFAULT 0
1294            ) STRICT;
1295            CREATE TABLE stats_securejoin_uipaths(
1296                uipath INTEGER PRIMARY KEY,
1297                count INTEGER NOT NULL DEFAULT 0
1298            ) STRICT;
1299            CREATE TABLE stats_securejoin_invites(
1300                already_existed INTEGER NOT NULL,
1301                already_verified INTEGER NOT NULL,
1302                type TEXT NOT NULL
1303            ) STRICT;
1304            CREATE TABLE stats_msgs(
1305                chattype INTEGER PRIMARY KEY,
1306                verified INTEGER NOT NULL DEFAULT 0,
1307                unverified_encrypted INTEGER NOT NULL DEFAULT 0,
1308                unencrypted INTEGER NOT NULL DEFAULT 0,
1309                only_to_self INTEGER NOT NULL DEFAULT 0,
1310                last_counted_msg_id INTEGER NOT NULL DEFAULT 0
1311            ) STRICT;",
1312            migration_version,
1313        )
1314        .await?;
1315    }
1316
1317    inc_and_check(&mut migration_version, 136)?;
1318    if dbversion < migration_version {
1319        sql.execute_migration(
1320            "CREATE TABLE stats_sending_enabled_events(timestamp INTEGER NOT NULL) STRICT;
1321            CREATE TABLE stats_sending_disabled_events(timestamp INTEGER NOT NULL) STRICT;",
1322            migration_version,
1323        )
1324        .await?;
1325    }
1326
1327    inc_and_check(&mut migration_version, 137)?;
1328    if dbversion < migration_version {
1329        sql.execute_migration(
1330            "DELETE FROM config WHERE keyname IN (
1331                'configured',
1332                'configured_imap_certificate_checks',
1333                'configured_imap_servers',
1334                'configured_mail_port',
1335                'configured_mail_pw',
1336                'configured_mail_security',
1337                'configured_mail_server',
1338                'configured_mail_user',
1339                'configured_send_port',
1340                'configured_send_pw',
1341                'configured_send_security',
1342                'configured_send_server',
1343                'configured_send_user',
1344                'configured_server_flags',
1345                'configured_smtp_certificate_checks',
1346                'configured_smtp_servers'
1347            )",
1348            migration_version,
1349        )
1350        .await?;
1351    }
1352
1353    inc_and_check(&mut migration_version, 138)?;
1354    if dbversion < migration_version {
1355        sql.execute_migration(
1356            "CREATE TABLE broadcast_secrets(
1357                chat_id INTEGER PRIMARY KEY NOT NULL,
1358                secret TEXT NOT NULL
1359            ) STRICT",
1360            migration_version,
1361        )
1362        .await?;
1363    }
1364
1365    inc_and_check(&mut migration_version, 139)?;
1366    if dbversion < migration_version {
1367        sql.execute_migration_transaction(
1368            |transaction| {
1369                if exists_before_update {
1370                    let is_chatmail = transaction
1371                        .query_row(
1372                            "SELECT value FROM config WHERE keyname='is_chatmail'",
1373                            (),
1374                            |row| {
1375                                let value: String = row.get(0)?;
1376                                Ok(value)
1377                            },
1378                        )
1379                        .optional()?
1380                        .as_deref()
1381                        == Some("1");
1382
1383                    // For non-chatmail accounts
1384                    // default "bcc_self" was "1".
1385                    // If it is not in the database,
1386                    // save the old default explicity
1387                    // as the new default is "0"
1388                    // for all accounts.
1389                    if !is_chatmail {
1390                        transaction.execute(
1391                            "INSERT OR IGNORE
1392                             INTO config (keyname, value)
1393                             VALUES (?, ?)",
1394                            ("bcc_self", "1"),
1395                        )?;
1396                    }
1397                }
1398                Ok(())
1399            },
1400            migration_version,
1401        )
1402        .await?;
1403    }
1404
1405    let new_version = sql
1406        .get_raw_config_int(VERSION_CFG)
1407        .await?
1408        .unwrap_or_default();
1409    if new_version != dbversion || !exists_before_update {
1410        let created_db = if exists_before_update {
1411            ""
1412        } else {
1413            "Created new database. "
1414        };
1415        info!(context, "{}Migration done from v{}.", created_db, dbversion);
1416    }
1417    info!(context, "Database version: v{new_version}.");
1418
1419    Ok((update_icons, disable_server_delete, recode_avatar))
1420}
1421
1422fn migrate_key_contacts(
1423    context: &Context,
1424    transaction: &mut rusqlite::Transaction<'_>,
1425) -> std::result::Result<(), anyhow::Error> {
1426    info!(context, "Starting key-contact transition.");
1427
1428    // =============================== Step 1: ===============================
1429    //                              Alter tables
1430    transaction.execute_batch(
1431        "ALTER TABLE contacts ADD COLUMN fingerprint TEXT NOT NULL DEFAULT '';
1432
1433        -- Verifier is an ID of the verifier contact.
1434        -- 0 if the contact is not verified.
1435        ALTER TABLE contacts ADD COLUMN verifier INTEGER NOT NULL DEFAULT 0;
1436
1437        CREATE INDEX contacts_fingerprint_index ON contacts (fingerprint);
1438
1439        CREATE TABLE public_keys (
1440        id INTEGER PRIMARY KEY AUTOINCREMENT,
1441        fingerprint TEXT NOT NULL UNIQUE, -- Upper-case fingerprint of the key.
1442        public_key BLOB NOT NULL -- Binary key, not ASCII-armored
1443        ) STRICT;
1444        CREATE INDEX public_key_index ON public_keys (fingerprint);
1445
1446        INSERT OR IGNORE INTO public_keys (fingerprint, public_key)
1447        SELECT public_key_fingerprint, public_key FROM acpeerstates
1448         WHERE public_key_fingerprint IS NOT NULL AND public_key IS NOT NULL;
1449
1450        INSERT OR IGNORE INTO public_keys (fingerprint, public_key)
1451        SELECT gossip_key_fingerprint, gossip_key FROM acpeerstates
1452         WHERE gossip_key_fingerprint IS NOT NULL AND gossip_key IS NOT NULL;
1453
1454        INSERT OR IGNORE INTO public_keys (fingerprint, public_key)
1455        SELECT verified_key_fingerprint, verified_key FROM acpeerstates
1456         WHERE verified_key_fingerprint IS NOT NULL AND verified_key IS NOT NULL;
1457
1458        INSERT OR IGNORE INTO public_keys (fingerprint, public_key)
1459        SELECT secondary_verified_key_fingerprint, secondary_verified_key FROM acpeerstates
1460         WHERE secondary_verified_key_fingerprint IS NOT NULL AND secondary_verified_key IS NOT NULL;",
1461    )
1462    .context("Creating key-contact tables")?;
1463
1464    let Some(self_addr): Option<String> = transaction
1465        .query_row(
1466            "SELECT value FROM config WHERE keyname='configured_addr'",
1467            (),
1468            |row| row.get(0),
1469        )
1470        .optional()
1471        .context("Step 0")?
1472    else {
1473        info!(
1474            context,
1475            "Not yet configured, no need to migrate key-contacts"
1476        );
1477        return Ok(());
1478    };
1479
1480    // =============================== Step 2: ===============================
1481    // Create up to 3 new contacts for every contact that has a peerstate:
1482    // one from the Autocrypt key fingerprint, one from the verified key fingerprint,
1483    // one from the secondary verified key fingerprint.
1484    // In the process, build maps from old contact id to new contact id:
1485    // one that maps to Autocrypt key-contact, one that maps to verified key-contact.
1486    let mut autocrypt_key_contacts: BTreeMap<u32, u32> = BTreeMap::new();
1487    let mut autocrypt_key_contacts_with_reset_peerstate: BTreeMap<u32, u32> = BTreeMap::new();
1488    let mut verified_key_contacts: BTreeMap<u32, u32> = BTreeMap::new();
1489    {
1490        // This maps from the verified contact to the original contact id of the verifier.
1491        // It can't map to the verified key contact id, because at the time of constructing
1492        // this map, not all key-contacts are in the database.
1493        let mut verifications: BTreeMap<u32, u32> = BTreeMap::new();
1494
1495        let mut load_contacts_stmt = transaction
1496            .prepare(
1497                "SELECT c.id, c.name, c.addr, c.origin, c.blocked, c.last_seen,
1498                c.authname, c.param, c.status, c.is_bot, c.selfavatar_sent,
1499                IFNULL(p.public_key, p.gossip_key),
1500                p.verified_key, IFNULL(p.verifier, ''),
1501                p.secondary_verified_key, p.secondary_verifier, p.prefer_encrypted
1502                FROM contacts c
1503                INNER JOIN acpeerstates p ON c.addr=p.addr
1504                WHERE c.id > 9
1505                ORDER BY p.last_seen DESC",
1506            )
1507            .context("Step 2")?;
1508
1509        let all_address_contacts: rusqlite::Result<Vec<_>> = load_contacts_stmt
1510            .query_map((), |row| {
1511                let id: i64 = row.get(0)?;
1512                let name: String = row.get(1)?;
1513                let addr: String = row.get(2)?;
1514                let origin: i64 = row.get(3)?;
1515                let blocked: Option<bool> = row.get(4)?;
1516                let last_seen: i64 = row.get(5)?;
1517                let authname: String = row.get(6)?;
1518                let param: String = row.get(7)?;
1519                let status: Option<String> = row.get(8)?;
1520                let is_bot: bool = row.get(9)?;
1521                let selfavatar_sent: i64 = row.get(10)?;
1522                let autocrypt_key = row
1523                    .get(11)
1524                    .ok()
1525                    .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok());
1526                let verified_key = row
1527                    .get(12)
1528                    .ok()
1529                    .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok());
1530                let verifier: String = row.get(13)?;
1531                let secondary_verified_key = row
1532                    .get(12)
1533                    .ok()
1534                    .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok());
1535                let secondary_verifier: String = row.get(15)?;
1536                let prefer_encrypt: u8 = row.get(16)?;
1537                Ok((
1538                    id,
1539                    name,
1540                    addr,
1541                    origin,
1542                    blocked,
1543                    last_seen,
1544                    authname,
1545                    param,
1546                    status,
1547                    is_bot,
1548                    selfavatar_sent,
1549                    autocrypt_key,
1550                    verified_key,
1551                    verifier,
1552                    secondary_verified_key,
1553                    secondary_verifier,
1554                    prefer_encrypt,
1555                ))
1556            })
1557            .context("Step 3")?
1558            .collect();
1559
1560        let mut insert_contact_stmt = transaction
1561            .prepare(
1562                "INSERT INTO contacts (name, addr, origin, blocked, last_seen,
1563                authname, param, status, is_bot, selfavatar_sent, fingerprint)
1564                VALUES(?,?,?,?,?,?,?,?,?,?,?)",
1565            )
1566            .context("Step 4")?;
1567        let mut fingerprint_to_id_stmt = transaction
1568            .prepare("SELECT id FROM contacts WHERE fingerprint=? AND id>9")
1569            .context("Step 5")?;
1570        let mut original_contact_id_from_addr_stmt = transaction
1571            .prepare("SELECT id FROM contacts WHERE addr=? AND fingerprint='' AND id>9")
1572            .context("Step 6")?;
1573
1574        for row in all_address_contacts? {
1575            let (
1576                original_id,
1577                name,
1578                addr,
1579                origin,
1580                blocked,
1581                last_seen,
1582                authname,
1583                param,
1584                status,
1585                is_bot,
1586                selfavatar_sent,
1587                autocrypt_key,
1588                verified_key,
1589                verifier,
1590                secondary_verified_key,
1591                secondary_verifier,
1592                prefer_encrypt,
1593            ) = row;
1594            let mut insert_contact = |key: SignedPublicKey| -> Result<u32> {
1595                let fingerprint = key.dc_fingerprint().hex();
1596                let existing_contact_id: Option<u32> = fingerprint_to_id_stmt
1597                    .query_row((&fingerprint,), |row| row.get(0))
1598                    .optional()
1599                    .context("Step 7")?;
1600                if let Some(existing_contact_id) = existing_contact_id {
1601                    return Ok(existing_contact_id);
1602                }
1603                insert_contact_stmt
1604                    .execute((
1605                        &name,
1606                        &addr,
1607                        origin,
1608                        blocked,
1609                        last_seen,
1610                        &authname,
1611                        &param,
1612                        &status,
1613                        is_bot,
1614                        selfavatar_sent,
1615                        fingerprint.clone(),
1616                    ))
1617                    .context("Step 8")?;
1618                let id = transaction
1619                    .last_insert_rowid()
1620                    .try_into()
1621                    .context("Step 9")?;
1622                info!(
1623                    context,
1624                    "Inserted new contact id={id} name='{name}' addr='{addr}' fingerprint={fingerprint}"
1625                );
1626                Ok(id)
1627            };
1628            let mut original_contact_id_from_addr = |addr: &str, default: u32| -> Result<u32> {
1629                if addr_cmp(addr, &self_addr) {
1630                    Ok(1) // ContactId::SELF
1631                } else if addr.is_empty() {
1632                    Ok(default)
1633                } else {
1634                    Ok(original_contact_id_from_addr_stmt
1635                        .query_row((addr,), |row| row.get(0))
1636                        .optional()
1637                        .with_context(|| format!("Original contact '{addr}' not found"))?
1638                        .unwrap_or(default))
1639                }
1640            };
1641
1642            let Some(autocrypt_key) = autocrypt_key else {
1643                continue;
1644            };
1645            let new_id = insert_contact(autocrypt_key).context("Step 10")?;
1646
1647            // prefer_encrypt == 20 would mean EncryptPreference::Reset,
1648            // i.e. we shouldn't encrypt if possible.
1649            if prefer_encrypt != 20 {
1650                autocrypt_key_contacts.insert(original_id.try_into().context("Step 11")?, new_id);
1651            } else {
1652                autocrypt_key_contacts_with_reset_peerstate
1653                    .insert(original_id.try_into().context("Step 12")?, new_id);
1654            }
1655
1656            let Some(verified_key) = verified_key else {
1657                continue;
1658            };
1659            let new_id = insert_contact(verified_key).context("Step 13")?;
1660            verified_key_contacts.insert(original_id.try_into().context("Step 14")?, new_id);
1661
1662            let verifier_id = if addr_cmp(&verifier, &addr) {
1663                // Earlier versions of Delta Chat signalled a direct verification
1664                // by putting the contact's own address into the verifier column
1665                1 // 1=ContactId::SELF
1666            } else {
1667                // If the original verifier is unknown, we represent this in the database
1668                // by putting `new_id` into the place of the verifier,
1669                // i.e. we say that this contact verified itself.
1670                original_contact_id_from_addr(&verifier, new_id).context("Step 15")?
1671            };
1672            verifications.insert(new_id, verifier_id);
1673
1674            let Some(secondary_verified_key) = secondary_verified_key else {
1675                continue;
1676            };
1677            let new_id = insert_contact(secondary_verified_key).context("Step 16")?;
1678            let verifier_id: u32 = if addr_cmp(&secondary_verifier, &addr) {
1679                1 // 1=ContactId::SELF
1680            } else {
1681                original_contact_id_from_addr(&secondary_verifier, new_id).context("Step 17")?
1682            };
1683            // Only use secondary verification if there is no primary verification:
1684            verifications.entry(new_id).or_insert(verifier_id);
1685        }
1686        info!(
1687            context,
1688            "Created key-contacts identified by autocrypt key: {autocrypt_key_contacts:?}"
1689        );
1690        info!(
1691            context,
1692            "Created key-contacts  with 'reset' peerstate identified by autocrypt key: {autocrypt_key_contacts_with_reset_peerstate:?}"
1693        );
1694        info!(
1695            context,
1696            "Created key-contacts identified by verified key: {verified_key_contacts:?}"
1697        );
1698
1699        for (&new_contact, &verifier_original_contact) in &verifications {
1700            let verifier = if verifier_original_contact == 1 {
1701                1 // Verified by ContactId::SELF
1702            } else if verifier_original_contact == new_contact {
1703                new_contact // unkwnown verifier
1704            } else {
1705                // `verifications` contains the original contact id.
1706                // We need to get the new, verified-pgp-identified contact id.
1707                match verified_key_contacts.get(&verifier_original_contact) {
1708                    Some(v) => *v,
1709                    None => {
1710                        warn!(
1711                            context,
1712                            "Couldn't find key-contact for {verifier_original_contact} who verified {new_contact}"
1713                        );
1714                        continue;
1715                    }
1716                }
1717            };
1718            transaction
1719                .execute(
1720                    "UPDATE contacts SET verifier=? WHERE id=?",
1721                    (verifier, new_contact),
1722                )
1723                .context("Step 18")?;
1724        }
1725        info!(context, "Migrated verifications: {verifications:?}");
1726    }
1727
1728    // ======================= Step 3: =======================
1729    // For each chat, modify the memberlist to retain the correct contacts
1730    // In the process, track the set of contacts which remained no any chat at all
1731    // in a `BTreeSet<u32>`, which initially contains all contact ids
1732    let mut orphaned_contacts: BTreeSet<u32> = transaction
1733        .prepare("SELECT id FROM contacts WHERE id>9")
1734        .context("Step 19")?
1735        .query_map((), |row| row.get::<usize, u32>(0))
1736        .context("Step 20")?
1737        .collect::<Result<BTreeSet<u32>, rusqlite::Error>>()
1738        .context("Step 21")?;
1739
1740    {
1741        let mut stmt = transaction
1742            .prepare(
1743                "SELECT c.id, c.type, c.grpid, c.protected
1744            FROM chats c
1745            WHERE id>9",
1746            )
1747            .context("Step 22")?;
1748        let all_chats = stmt
1749            .query_map((), |row| {
1750                let id: u32 = row.get(0)?;
1751                let typ: u32 = row.get(1)?;
1752                let grpid: String = row.get(2)?;
1753                let protected: u32 = row.get(3)?;
1754                Ok((id, typ, grpid, protected))
1755            })
1756            .context("Step 23")?;
1757        let mut load_chat_contacts_stmt = transaction.prepare(
1758            "SELECT contact_id, add_timestamp>=remove_timestamp FROM chats_contacts
1759             WHERE chat_id=? AND contact_id>9",
1760        )?;
1761        let is_chatmail: Option<String> = transaction
1762            .query_row(
1763                "SELECT value FROM config WHERE keyname='is_chatmail'",
1764                (),
1765                |row| row.get(0),
1766            )
1767            .optional()
1768            .context("Step 23.1")?;
1769        let is_chatmail = is_chatmail
1770            .and_then(|s| s.parse::<i32>().ok())
1771            .unwrap_or_default()
1772            != 0;
1773        let map_to_key_contact = |old_member: &u32| {
1774            autocrypt_key_contacts
1775                .get(old_member)
1776                .or_else(|| {
1777                    // For chatmail servers,
1778                    // we send encrypted even if the peerstate is reset,
1779                    // because an unencrypted message likely won't arrive.
1780                    // This is the same behavior as before key-contacts migration.
1781                    if is_chatmail {
1782                        autocrypt_key_contacts_with_reset_peerstate.get(old_member)
1783                    } else {
1784                        None
1785                    }
1786                })
1787                .copied()
1788        };
1789
1790        let mut update_member_stmt = transaction
1791            .prepare("UPDATE chats_contacts SET contact_id=? WHERE contact_id=? AND chat_id=?")?;
1792        let mut addr_cmp_stmt = transaction
1793            .prepare("SELECT c.addr=d.addr FROM contacts c, contacts d WHERE c.id=? AND d.id=?")?;
1794        for chat in all_chats {
1795            let (chat_id, typ, grpid, protected) = chat.context("Step 24")?;
1796            // In groups, this also contains past members, i.e. `(_, false)` entries.
1797            let old_members: Vec<(u32, bool)> = load_chat_contacts_stmt
1798                .query_map((chat_id,), |row| {
1799                    let id: u32 = row.get(0)?;
1800                    let present: bool = row.get(1)?;
1801                    Ok((id, present))
1802                })
1803                .context("Step 25")?
1804                .collect::<Result<Vec<_>, _>>()
1805                .context("Step 26")?;
1806
1807            let mut keep_address_contacts = |reason: &str| -> Result<()> {
1808                info!(
1809                    context,
1810                    "Chat {chat_id} will be an unencrypted chat with contacts identified by email address: {reason}."
1811                );
1812                for (m, _) in &old_members {
1813                    orphaned_contacts.remove(m);
1814                }
1815
1816                // Unprotect this chat if it was protected.
1817                //
1818                // Otherwise we get protected chat with address-contact(s).
1819                transaction
1820                    .execute("UPDATE chats SET protected=0 WHERE id=?", (chat_id,))
1821                    .context("Step 26.0")?;
1822
1823                Ok(())
1824            };
1825            let old_and_new_members: Vec<(u32, bool, Option<u32>)> = match typ {
1826                // 1:1 chats retain:
1827                // - address-contact if peerstate is in the "reset" state,
1828                //   or if there is no key-contact that has the right email address.
1829                // - key-contact identified by the Autocrypt key if Autocrypt key does not match the verified key.
1830                // - key-contact identified by the verified key if peerstate Autocrypt key matches the Verified key.
1831                //   Since the autocrypt and verified key-contact are identital in this case, we can add the Autocrypt key-contact,
1832                //   and the effect will be the same.
1833                100 => {
1834                    let Some((old_member, _)) = old_members.first() else {
1835                        info!(
1836                            context,
1837                            "1:1 chat {chat_id} doesn't contain contact, probably it's self or device chat."
1838                        );
1839                        continue;
1840                    };
1841
1842                    let Some(new_contact) = map_to_key_contact(old_member) else {
1843                        keep_address_contacts("No peerstate, or peerstate in 'reset' state")?;
1844                        continue;
1845                    };
1846                    if !addr_cmp_stmt
1847                        .query_row((old_member, new_contact), |row| row.get::<_, bool>(0))?
1848                    {
1849                        keep_address_contacts("key contact has different email")?;
1850                        continue;
1851                    }
1852                    vec![(*old_member, true, Some(new_contact))]
1853                }
1854
1855                // Group
1856                120 => {
1857                    if grpid.is_empty() {
1858                        // Ad-hoc group that has empty Chat-Group-ID
1859                        // because it was created in response to receiving a non-chat email.
1860                        keep_address_contacts("Empty chat-Group-ID")?;
1861                        continue;
1862                    } else if protected == 1 {
1863                        old_members
1864                            .iter()
1865                            .map(|&(id, present)| {
1866                                (id, present, verified_key_contacts.get(&id).copied())
1867                            })
1868                            .collect()
1869                    } else {
1870                        old_members
1871                            .iter()
1872                            .map(|&(id, present)| (id, present, map_to_key_contact(&id)))
1873                            .collect::<Vec<(u32, bool, Option<u32>)>>()
1874                    }
1875                }
1876
1877                // Mailinglist
1878                140 => {
1879                    keep_address_contacts("Mailinglist")?;
1880                    continue;
1881                }
1882
1883                // Broadcast channel
1884                160 => old_members
1885                    .iter()
1886                    .map(|(original, _)| {
1887                        (
1888                            *original,
1889                            true,
1890                            autocrypt_key_contacts
1891                                .get(original)
1892                                // There will be no unencrypted broadcast lists anymore,
1893                                // so, if a peerstate is reset,
1894                                // the best we can do is encrypting to this key regardless.
1895                                .or_else(|| {
1896                                    autocrypt_key_contacts_with_reset_peerstate.get(original)
1897                                })
1898                                .copied(),
1899                        )
1900                    })
1901                    .collect::<Vec<(u32, bool, Option<u32>)>>(),
1902                _ => {
1903                    warn!(context, "Invalid chat type {typ}");
1904                    continue;
1905                }
1906            };
1907
1908            // If a group contains a contact without a key or with 'reset' peerstate,
1909            // downgrade to unencrypted Ad-Hoc group.
1910            if typ == 120
1911                && old_and_new_members
1912                    .iter()
1913                    .any(|&(_old, present, new)| present && new.is_none())
1914            {
1915                transaction
1916                    .execute("UPDATE chats SET grpid='' WHERE id=?", (chat_id,))
1917                    .context("Step 26.1")?;
1918                keep_address_contacts("Group contains contact without peerstate")?;
1919                continue;
1920            }
1921
1922            let human_readable_transitions = old_and_new_members
1923                .iter()
1924                .map(|(old, _, new)| format!("{old}->{}", new.unwrap_or_default()))
1925                .collect::<Vec<String>>()
1926                .join(" ");
1927            info!(
1928                context,
1929                "Migrating chat {chat_id} to key-contacts: {human_readable_transitions}"
1930            );
1931
1932            for (old_member, _, new_member) in old_and_new_members {
1933                if let Some(new_member) = new_member {
1934                    orphaned_contacts.remove(&new_member);
1935                    let res = update_member_stmt.execute((new_member, old_member, chat_id));
1936                    if res.is_err() {
1937                        // The same chat partner exists multiple times in the chat,
1938                        // with mutliple profiles which have different email addresses
1939                        // but the same key.
1940                        // We can only keep one of them.
1941                        // So, if one of them is not in the chat anymore, delete it,
1942                        // otherwise delete the one that was added least recently.
1943                        let member_to_delete: u32 = transaction
1944                            .query_row(
1945                                "SELECT contact_id
1946                               FROM chats_contacts
1947                              WHERE chat_id=? AND contact_id IN (?,?)
1948                           ORDER BY add_timestamp>=remove_timestamp, add_timestamp LIMIT 1",
1949                                (chat_id, new_member, old_member),
1950                                |row| row.get(0),
1951                            )
1952                            .context("Step 27")?;
1953                        info!(
1954                            context,
1955                            "Chat partner is in the chat {chat_id} multiple times. \
1956                            Deleting {member_to_delete}, then trying to update \
1957                            {old_member}->{new_member} again"
1958                        );
1959                        transaction
1960                            .execute(
1961                                "DELETE FROM chats_contacts WHERE chat_id=? AND contact_id=?",
1962                                (chat_id, member_to_delete),
1963                            )
1964                            .context("Step 28")?;
1965                        // If we removed `old_member`, then this will be a no-op,
1966                        // which is exactly what we want in this case:
1967                        update_member_stmt.execute((new_member, old_member, chat_id))?;
1968                    }
1969                } else {
1970                    info!(
1971                        context,
1972                        "Old member {old_member} in chat {chat_id} can't be upgraded to key-contact, removing them"
1973                    );
1974                    transaction
1975                        .execute(
1976                            "DELETE FROM chats_contacts WHERE contact_id=? AND chat_id=?",
1977                            (old_member, chat_id),
1978                        )
1979                        .context("Step 29")?;
1980                }
1981            }
1982        }
1983    }
1984
1985    // ======================= Step 4: =======================
1986    {
1987        info!(
1988            context,
1989            "Marking contacts which remained in no chat at all as hidden: {orphaned_contacts:?}"
1990        );
1991        let mut mark_as_hidden_stmt = transaction
1992            .prepare("UPDATE contacts SET origin=? WHERE id=?")
1993            .context("Step 30")?;
1994        for contact in orphaned_contacts {
1995            mark_as_hidden_stmt
1996                .execute((0x8, contact))
1997                .context("Step 31")?;
1998        }
1999    }
2000
2001    // ======================= Step 5: =======================
2002    // Prepare for rewriting `from_id`, `to_id` in messages
2003    {
2004        let mut contacts_map = autocrypt_key_contacts_with_reset_peerstate;
2005        for (old, new) in autocrypt_key_contacts {
2006            contacts_map.insert(old, new);
2007        }
2008        transaction
2009            .execute(
2010                "CREATE TABLE key_contacts_map (
2011                    old_id INTEGER PRIMARY KEY NOT NULL,
2012                    new_id INTEGER NOT NULL
2013                ) STRICT",
2014                (),
2015            )
2016            .context("Step 32")?;
2017        {
2018            let mut stmt = transaction
2019                .prepare("INSERT INTO key_contacts_map (old_id, new_id) VALUES (?, ?)")
2020                .context("Step 33")?;
2021            for ids in contacts_map {
2022                stmt.execute(ids).context("Step 34")?;
2023            }
2024        }
2025        transaction
2026            .execute(
2027                "INSERT INTO config (keyname, value) VALUES (
2028                    'first_key_contacts_msg_id',
2029                    IFNULL((SELECT MAX(id)+1 FROM msgs), 0)
2030                )",
2031                (),
2032            )
2033            .context("Step 35")?;
2034    }
2035
2036    Ok(())
2037}
2038
2039/// Rewrite `from_id`, `to_id` in >= 1000 messages starting from the newest ones, to key-contacts.
2040pub(crate) async fn msgs_to_key_contacts(context: &Context) -> Result<()> {
2041    let sql = &context.sql;
2042    if sql
2043        .get_raw_config_int64("first_key_contacts_msg_id")
2044        .await?
2045        <= Some(0)
2046    {
2047        return Ok(());
2048    }
2049    let trans_fn = |t: &mut rusqlite::Transaction| {
2050        let mut first_key_contacts_msg_id: u64 = t
2051            .query_one(
2052                "SELECT CAST(value AS INTEGER) FROM config WHERE keyname='first_key_contacts_msg_id'",
2053                (),
2054                |row| row.get(0),
2055            )
2056            .context("Get first_key_contacts_msg_id")?;
2057        let mut stmt = t
2058            .prepare(
2059                "UPDATE msgs SET
2060                    from_id=IFNULL(
2061                        (SELECT new_id FROM key_contacts_map WHERE old_id=msgs.from_id),
2062                        from_id
2063                    ),
2064                    to_id=IFNULL(
2065                        (SELECT new_id FROM key_contacts_map WHERE old_id=msgs.to_id),
2066                        to_id
2067                    )
2068                WHERE id>=? AND id<?
2069                AND chat_id>9
2070                AND (param GLOB '*\nc=1*' OR param GLOB 'c=1*')",
2071            )
2072            .context("Prepare stmt")?;
2073        let msgs_to_migrate = 1000;
2074        let mut msgs_migrated: u64 = 0;
2075        while first_key_contacts_msg_id > 0 && msgs_migrated < msgs_to_migrate {
2076            let start_msg_id = first_key_contacts_msg_id.saturating_sub(msgs_to_migrate);
2077            let cnt: u64 = stmt
2078                .execute((start_msg_id, first_key_contacts_msg_id))
2079                .context("UPDATE msgs")?
2080                .try_into()?;
2081            msgs_migrated += cnt;
2082            first_key_contacts_msg_id = start_msg_id;
2083        }
2084        t.execute(
2085            "UPDATE config SET value=? WHERE keyname='first_key_contacts_msg_id'",
2086            (first_key_contacts_msg_id,),
2087        )
2088        .context("Update first_key_contacts_msg_id")?;
2089        Ok((msgs_migrated, first_key_contacts_msg_id))
2090    };
2091    let start = Time::now();
2092    let mut msgs_migrated = 0;
2093    loop {
2094        let (n, first_key_contacts_msg_id) = sql.transaction(trans_fn).await?;
2095        msgs_migrated += n;
2096        if first_key_contacts_msg_id == 0 || time_elapsed(&start) >= Duration::from_millis(500) {
2097            break;
2098        }
2099    }
2100    sql.uncache_raw_config("first_key_contacts_msg_id").await;
2101    info!(
2102        context,
2103        "Rewriting {msgs_migrated} msgs to key-contacts took {:?}.",
2104        time_elapsed(&start),
2105    );
2106    Ok(())
2107}
2108
2109impl Sql {
2110    async fn set_db_version(&self, version: i32) -> Result<()> {
2111        self.set_raw_config_int(VERSION_CFG, version).await?;
2112        Ok(())
2113    }
2114
2115    // Sets db `version` in the `transaction`.
2116    fn set_db_version_trans(transaction: &mut rusqlite::Transaction, version: i32) -> Result<()> {
2117        transaction.execute(
2118            "UPDATE config SET value=? WHERE keyname=?;",
2119            (format!("{version}"), VERSION_CFG),
2120        )?;
2121        Ok(())
2122    }
2123
2124    async fn set_db_version_in_cache(&self, version: i32) -> Result<()> {
2125        let mut lock = self.config_cache.write().await;
2126        lock.insert(VERSION_CFG.to_string(), Some(format!("{version}")));
2127        Ok(())
2128    }
2129
2130    async fn execute_migration(&self, query: &str, version: i32) -> Result<()> {
2131        self.execute_migration_transaction(
2132            |transaction| {
2133                transaction.execute_batch(query)?;
2134                Ok(())
2135            },
2136            version,
2137        )
2138        .await
2139    }
2140
2141    async fn execute_migration_transaction(
2142        &self,
2143        migration: impl Send + FnOnce(&mut rusqlite::Transaction) -> Result<()>,
2144        version: i32,
2145    ) -> Result<()> {
2146        #[cfg(test)]
2147        if STOP_MIGRATIONS_AT.try_with(|stop_migrations_at| version > *stop_migrations_at)
2148            == Ok(true)
2149        {
2150            println!("Not running migration {version}, because STOP_MIGRATIONS_AT is set");
2151            return Ok(());
2152        }
2153
2154        self.transaction(move |transaction| {
2155            let curr_version: String = transaction.query_row(
2156                "SELECT IFNULL(value, ?) FROM config WHERE keyname=?;",
2157                ("0", VERSION_CFG),
2158                |row| row.get(0),
2159            )?;
2160            let curr_version: i32 = curr_version.parse()?;
2161            ensure!(curr_version < version, "Db version must be increased");
2162            Self::set_db_version_trans(transaction, version)?;
2163            migration(transaction)?;
2164
2165            Ok(())
2166        })
2167        .await
2168        .with_context(|| format!("execute_migration failed for version {version}"))?;
2169
2170        self.config_cache.write().await.clear();
2171
2172        Ok(())
2173    }
2174}
2175
2176#[cfg(test)]
2177mod migrations_tests;