FennecMigrator.kt 62.3 KB
Newer Older
1
2
3
4
5
6
7
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package mozilla.components.support.migration

import android.content.Context
8
import android.content.Intent
9
import androidx.annotation.VisibleForTesting
10
import androidx.core.content.ContextCompat
11
12
13
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Deferred
14
import kotlinx.coroutines.Dispatchers
15
16
import kotlinx.coroutines.asCoroutineDispatcher
import kotlinx.coroutines.async
17
import kotlinx.coroutines.runBlocking
18
import kotlinx.coroutines.withContext
19
import mozilla.components.browser.search.SearchEngineManager
20
21
22
import mozilla.components.browser.session.SessionManager
import mozilla.components.browser.storage.sync.PlacesBookmarksStorage
import mozilla.components.browser.storage.sync.PlacesHistoryStorage
23
import mozilla.components.concept.engine.Engine
24
25
import mozilla.components.feature.addons.amo.AddonCollectionProvider
import mozilla.components.feature.addons.update.AddonUpdater
26
import mozilla.components.feature.top.sites.PinnedSiteStorage
Grisha Kruglov's avatar
Grisha Kruglov committed
27
import mozilla.components.service.fxa.manager.FxaAccountManager
28
import mozilla.components.service.glean.Glean
29
import mozilla.components.service.sync.logins.SyncableLoginsStorage
30
import mozilla.components.support.base.crash.CrashReporting
31
import mozilla.components.support.base.log.logger.Logger
32
import mozilla.components.support.migration.FennecMigrator.Builder
Grisha Kruglov's avatar
Grisha Kruglov committed
33
import mozilla.components.support.migration.GleanMetrics.MigrationAddons
34
35
import mozilla.components.support.migration.state.MigrationAction
import mozilla.components.support.migration.state.MigrationStore
Grisha Kruglov's avatar
Grisha Kruglov committed
36
37
38
39
40
41
42
43
44
import mozilla.components.support.migration.GleanMetrics.Pings
import mozilla.components.support.migration.GleanMetrics.Migration as MigrationPing
import mozilla.components.support.migration.GleanMetrics.MigrationBookmarks
import mozilla.components.support.migration.GleanMetrics.MigrationFxa
import mozilla.components.support.migration.GleanMetrics.MigrationGecko
import mozilla.components.support.migration.GleanMetrics.MigrationHistory
import mozilla.components.support.migration.GleanMetrics.MigrationLogins
import mozilla.components.support.migration.GleanMetrics.MigrationTelemetryIdentifiers
import mozilla.components.support.migration.GleanMetrics.MigrationOpenTabs
45
import mozilla.components.support.migration.GleanMetrics.MigrationPinnedSites
46
import mozilla.components.support.migration.GleanMetrics.MigrationSearch
Grisha Kruglov's avatar
Grisha Kruglov committed
47
import mozilla.components.support.migration.GleanMetrics.MigrationSettings
48
import java.io.File
49
import java.lang.AssertionError
Grisha Kruglov's avatar
Grisha Kruglov committed
50
51
import java.util.Date
import java.util.UUID
52
import java.util.concurrent.Executors
53
import kotlin.Exception
Grisha Kruglov's avatar
Grisha Kruglov committed
54
import kotlin.IllegalStateException
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
import kotlin.coroutines.CoroutineContext

/**
 * Supported Fennec migrations and their current versions.
 */
sealed class Migration(val currentVersion: Int) {
    /**
     * Migrates history (both "places" and "visits").
     */
    object History : Migration(currentVersion = 1)

    /**
     * Migrates bookmarks. Must run after history was migrated.
     */
    object Bookmarks : Migration(currentVersion = 1)

71
72
73
74
75
    /**
     * Migrates logins.
     */
    object Logins : Migration(currentVersion = 1)

76
77
78
79
    /**
     * Migrates open tabs.
     */
    object OpenTabs : Migration(currentVersion = 1)
Grisha Kruglov's avatar
Grisha Kruglov committed
80
81
82
83
84

    /**
     * Migrates FxA state.
     */
    object FxA : Migration(currentVersion = 1)
85
86
87
88
89

    /**
     * Migrates Gecko(View) internal files.
     */
    object Gecko : Migration(currentVersion = 1)
90
91
92
93

    /**
     * Migrates all Fennec settings backed by SharedPreferences.
     */
94
    object Settings : Migration(currentVersion = 2)
95
96
97
98

    /**
     * Migrates / Disables all currently unsupported Add-ons.
     */
99
    object Addons : Migration(currentVersion = 3)
100
101
102
103

    /**
     * Migrates Fennec's telemetry identifiers.
     */
Grisha Kruglov's avatar
Grisha Kruglov committed
104
    object TelemetryIdentifiers : Migration(currentVersion = 1)
105
106
107
108
109

    /**
     * Migrates Fennec's default search engine.
     */
    object SearchEngine : Migration(currentVersion = 1)
110
111
112
113
114

    /**
     * Migrates Fennec's pinned sites.
     */
    object PinnedSites : Migration(currentVersion = 1)
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
}

/**
 * Describes a [Migration] at a specific version, enforcing in-range version specification.
 *
 * @property migration A [Migration] in question.
 * @property version Version of the [migration], defaulting to the current version.
 */
data class VersionedMigration(val migration: Migration, val version: Int = migration.currentVersion) {
    init {
        require(version <= migration.currentVersion && version >= 1) {
            "Migration version must be between 1 and current version"
        }
    }
}

131
132
133
134
135
136
137
/**
 * Exceptions related to Fennec migrations.
 *
 * See https://github.com/mozilla-mobile/android-components/issues/5095 for stripping any possible PII from these
 * exceptions.
 */
sealed class FennecMigratorException(cause: Exception) : Exception(cause) {
138
139
140
141
142
143
    /**
     * Unexpected exception during high level migration processing.
     * @param cause Original exception which caused the problem.
     */
    class HighLevel(cause: Exception) : FennecMigratorException(cause)

144
145
146
147
148
149
150
151
152
153
154
155
    /**
     * Unexpected exception while migrating history.
     * @param cause Original exception which caused the problem.
     */
    class MigrateHistoryException(cause: Exception) : FennecMigratorException(cause)

    /**
     * Unexpected exception while migrating bookmarks.
     * @param cause Original exception which caused the problem.
     */
    class MigrateBookmarksException(cause: Exception) : FennecMigratorException(cause)

156
157
158
159
160
161
    /**
     * Unexpected exception while migrating logins.
     * @param cause Original exception which caused the problem.
     */
    class MigrateLoginsException(cause: Exception) : FennecMigratorException(cause)

162
163
164
165
166
    /**
     * Unexpected exception while migrating open tabs.
     * @param cause Original exception which caused the problem.
     */
    class MigrateOpenTabsException(cause: Exception) : FennecMigratorException(cause)
167

168
169
170
171
172
    /**
     * Unexpected exception while migrating gecko profile.
     * @param cause Original exception which caused the problem.
     */
    class MigrateGeckoException(cause: Exception) : FennecMigratorException(cause)
173
174
175

    /**
     * Unexpected exception while migrating settings.
176
     * @param cause Original exception which caused the problem.
177
178
     */
    class MigrateSettingsException(cause: Exception) : FennecMigratorException(cause)
179
180
181
182
183
184

    /**
     * Unexpected exception while migrating addons.
     * @param cause Original exception which caused the problem
     */
    class MigrateAddonsException(cause: Exception) : FennecMigratorException(cause)
185

186
187
188
189
190
191
    /**
     * Unexpected exception while migrating FxA.
     * @param cause Original exception which caused the problem
     */
    class MigrateFxaException(cause: Exception) : FennecMigratorException(cause)

192
193
194
195
196
    /**
     * Unexpected exception while migrating telemetry identifiers.
     * @param cause Original exception which caused the problem.
     */
    class TelemetryIdentifierException(cause: Exception) : FennecMigratorException(cause)
197
198
199
200
201
202

    /**
     * Unexpected exception while migrating the default search engine.
     * @param cause Original exception which caused the problem.
     */
    class MigrateSearchEngineException(cause: Exception) : FennecMigratorException(cause)
203
204
205
206
207
208

    /**
     * Unexpected exception while migrating pinned sites.
     * @param cause Original exception which caused the problem.
     */
    class MigratePinnedSitesException(cause: Exception) : FennecMigratorException(cause)
209
210
}

211
212
213
214
215
216
217
218
219
/**
 * Entrypoint for Fennec data migration. See [Builder] for public API.
 *
 * @param context Application context used for accessing the file system.
 * @param migrations Describes ordering and versioning of migrations to run.
 * @param historyStorage An optional instance of [PlacesHistoryStorage] used to store migrated history data.
 * @param bookmarksStorage An optional instance of [PlacesBookmarksStorage] used to store migrated bookmarks data.
 * @param coroutineContext An instance of [CoroutineContext] used for executing async migration tasks.
 */
220
@Suppress("LargeClass", "TooManyFunctions", "LongParameterList")
221
222
class FennecMigrator private constructor(
    private val context: Context,
223
    private val crashReporter: CrashReporting,
224
    private val migrations: List<VersionedMigration>,
225
226
227
    private val historyStorage: Lazy<PlacesHistoryStorage>?,
    private val bookmarksStorage: Lazy<PlacesBookmarksStorage>?,
    private val loginsStorage: Lazy<SyncableLoginsStorage>?,
228
    private val sessionManager: SessionManager?,
229
    private val searchEngineManager: SearchEngineManager?,
230
    private val accountManager: Lazy<FxaAccountManager>?,
231
    private val engine: Engine?,
232
233
    private val addonCollectionProvider: AddonCollectionProvider?,
    private val addonUpdater: AddonUpdater?,
234
    private val profile: FennecProfile?,
Grisha Kruglov's avatar
Grisha Kruglov committed
235
    private val fxaState: File?,
236
    private val browserDbPath: String?,
237
238
    private val signonsDbName: String,
    private val key4DbName: String,
239
    private val coroutineContext: CoroutineContext,
240
    private val pinnedSitesStorage: PinnedSiteStorage?
241
242
243
244
) {
    /**
     * Data migration builder. Allows configuring which migrations to run, their versions and relative order.
     */
245
    @Suppress("TooManyFunctions")
246
    class Builder(private val context: Context, private val crashReporter: CrashReporting) {
247
248
249
        private var historyStorage: Lazy<PlacesHistoryStorage>? = null
        private var bookmarksStorage: Lazy<PlacesBookmarksStorage>? = null
        private var loginsStorage: Lazy<SyncableLoginsStorage>? = null
250
        private var loginsStorageKey: String? = null
251
        private var sessionManager: SessionManager? = null
252
        private var searchEngineManager: SearchEngineManager? = null
253
        private var accountManager: Lazy<FxaAccountManager>? = null
254
        private var engine: Engine? = null
255
256
        private var addonCollectionProvider: AddonCollectionProvider? = null
        private var addonUpdater: AddonUpdater? = null
257
258
259
260
261
262

        private val migrations: MutableList<VersionedMigration> = mutableListOf()

        // Single-thread executor to ensure we don't accidentally parallelize migrations.
        private var coroutineContext: CoroutineContext = Executors.newSingleThreadExecutor().asCoroutineDispatcher()

Grisha Kruglov's avatar
Grisha Kruglov committed
263
        private var fxaState = File("${context.filesDir}", "fxa.account.json")
264
        private var fennecProfile = FennecProfile.findDefault(context, crashReporter)
265
        private var browserDbPath: String? = null
266
267
268
        private var signonsDbName = "signons.sqlite"
        private var key4DbName = "key4.db"
        private var masterPassword = FennecLoginsMigration.DEFAULT_MASTER_PASSWORD
269
        private var pinnedSitesStorage: PinnedSiteStorage? = null
270
271
272
273
274
275
276

        /**
         * Enable history migration.
         *
         * @param storage An instance of [PlacesHistoryStorage], used for storing data.
         * @param version Version of the migration; defaults to the current version.
         */
277
278
279
280
        fun migrateHistory(
            storage: Lazy<PlacesHistoryStorage>,
            version: Int = Migration.History.currentVersion
        ): Builder {
Grisha Kruglov's avatar
Grisha Kruglov committed
281
282
283
            check(migrations.find { it.migration is Migration.FxA } == null) {
                "FxA migration, if desired, must run after history"
            }
284
285
286
287
288
289
290
            historyStorage = storage
            migrations.add(VersionedMigration(Migration.History, version))
            return this
        }

        /**
         * Enable bookmarks migration. Must be called after [migrateHistory].
291
         * Optionally, enable top sites migration, if [pinnedSitesStorage] is specified.
292
         * In Fennec, pinned sites are stored as special type of a bookmark, hence this coupling.
293
294
         *
         * @param storage An instance of [PlacesBookmarksStorage], used for storing data.
295
         * @param pinnedSitesStorage An instance of [PinnedSiteStorage], used for storing pinned sites.
296
297
298
         * @param version Version of the migration; defaults to the current version.
         */
        fun migrateBookmarks(
299
            storage: Lazy<PlacesBookmarksStorage>,
300
            pinnedSitesStorage: PinnedSiteStorage? = null,
301
302
            version: Int = Migration.Bookmarks.currentVersion
        ): Builder {
Grisha Kruglov's avatar
Grisha Kruglov committed
303
304
305
            check(migrations.find { it.migration is Migration.FxA } == null) {
                "FxA migration, if desired, must run after bookmarks"
            }
306
307
308
309
310
            check(migrations.find { it.migration is Migration.History } != null) {
                "To migrate bookmarks, you must first migrate history"
            }
            bookmarksStorage = storage
            migrations.add(VersionedMigration(Migration.Bookmarks, version))
311
312
313
314

            // Allowing enabling pinned sites migration only when bookmarks migration is enabled is a conscious
            // choice. We currently don't have a requirement to only migrate pinned sites, and not bookmarks,
            // and so this is done to keep things a bit simpler.
315
316
            pinnedSitesStorage?.let {
                this.pinnedSitesStorage = it
317
318
319
                migrations.add(VersionedMigration(Migration.PinnedSites, version))
            }

320
321
322
            return this
        }

323
324
325
326
327
328
        /**
         * Enable logins migration.
         *
         * @param storage An instance of [AsyncLoginsStorage], used for storing data.
         */
        fun migrateLogins(
329
            storage: Lazy<SyncableLoginsStorage>,
330
331
332
333
334
335
336
337
338
339
            version: Int = Migration.Logins.currentVersion
        ): Builder {
            check(migrations.find { it.migration is Migration.FxA } == null) {
                "FxA migration, if desired, must run after logins"
            }
            loginsStorage = storage
            migrations.add(VersionedMigration(Migration.Logins, version))
            return this
        }

340
341
342
343
344
345
346
347
348
349
        /**
         * Enables the migration of Gecko internal files.
         */
        fun migrateGecko(
            version: Int = Migration.Gecko.currentVersion
        ): Builder {
            migrations.add(VersionedMigration(Migration.Gecko, version))
            return this
        }

350
351
352
353
354
355
356
357
358
359
360
361
        /**
         * Enable open tabs migration.
         *
         * @param sessionManager An instance of [SessionManager] used for restoring migrated [SessionManager.Snapshot].
         * @param version Version of the migration; defaults to the current version.
         */
        fun migrateOpenTabs(sessionManager: SessionManager, version: Int = Migration.OpenTabs.currentVersion): Builder {
            this.sessionManager = sessionManager
            migrations.add(VersionedMigration(Migration.OpenTabs, version))
            return this
        }

362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
        /**
         * Enable default search engine migration.
         *
         * @param searchEngineManager An instance of [SearchEngineManager] used for restoring the
         * migrated default search engine.
         * @param version Version of the migration; defaults to the current version.
         */
        fun migrateSearchEngine(
            searchEngineManager: SearchEngineManager,
            version: Int = Migration.SearchEngine.currentVersion
        ): Builder {
            this.searchEngineManager = searchEngineManager
            migrations.add(VersionedMigration(Migration.SearchEngine, version))
            return this
        }

Grisha Kruglov's avatar
Grisha Kruglov committed
378
379
380
381
382
383
        /**
         * Enable FxA state migration.
         *
         * @param accountManager An instance of [FxaAccountManager] used for authenticating using a migrated account.
         * @param version Version of the migration; defaults to the current version.
         */
384
        fun migrateFxa(accountManager: Lazy<FxaAccountManager>, version: Int = Migration.FxA.currentVersion): Builder {
Grisha Kruglov's avatar
Grisha Kruglov committed
385
386
387
388
389
            this.accountManager = accountManager
            migrations.add(VersionedMigration(Migration.FxA, version))
            return this
        }

390
391
392
393
394
395
396
397
        /**
         * Enable all Fennec - Fenix common settings migration.
         */
        fun migrateSettings(version: Int = Migration.Settings.currentVersion): Builder {
            migrations.add(VersionedMigration(Migration.Settings, version))
            return this
        }

Grisha Kruglov's avatar
Grisha Kruglov committed
398
399
400
        /**
         * Enable migration of Fennec telemetry identifiers.
         */
401
        fun migrateTelemetryIdentifiers(
Grisha Kruglov's avatar
Grisha Kruglov committed
402
            version: Int = Migration.TelemetryIdentifiers.currentVersion
403
        ): Builder {
Grisha Kruglov's avatar
Grisha Kruglov committed
404
            migrations.add(VersionedMigration(Migration.TelemetryIdentifiers, version))
405
406
407
            return this
        }

408
409
410
411
        /**
         * Enables Add-on migration.
         *
         * @param engine an instance of [Engine] use to query installed add-ons.
412
         * @param addonCollectionProvider an instace of [AddonCollectionProvider] to query supported add-ons.
413
414
         * @param version Version of the migration; defaults to the current version.
         */
415
416
417
418
419
420
        fun migrateAddons(
            engine: Engine,
            addonCollectionProvider: AddonCollectionProvider,
            addonUpdater: AddonUpdater,
            version: Int = Migration.Addons.currentVersion
        ): Builder {
421
            this.engine = engine
422
423
            this.addonCollectionProvider = addonCollectionProvider
            this.addonUpdater = addonUpdater
424
425
426
427
            migrations.add(VersionedMigration(Migration.Addons, version))
            return this
        }

428
429
430
431
432
433
        /**
         * Constructs a [FennecMigrator] based on the current configuration.
         */
        fun build(): FennecMigrator {
            return FennecMigrator(
                context,
Grisha Kruglov's avatar
Grisha Kruglov committed
434
                crashReporter,
435
436
437
                migrations,
                historyStorage,
                bookmarksStorage,
438
                loginsStorage,
439
                sessionManager,
440
                searchEngineManager,
Grisha Kruglov's avatar
Grisha Kruglov committed
441
                accountManager,
442
                engine,
443
444
                addonCollectionProvider,
                addonUpdater,
445
                fennecProfile,
Grisha Kruglov's avatar
Grisha Kruglov committed
446
                fxaState,
447
                browserDbPath ?: fennecProfile?.let { "${it.path}/browser.db" },
448
449
                signonsDbName,
                key4DbName,
450
                coroutineContext,
451
                pinnedSitesStorage
452
453
454
455
456
457
458
459
460
461
462
            )
        }

        // The rest of the setters are useful for unit tests.
        @VisibleForTesting
        internal fun setCoroutineContext(coroutineContext: CoroutineContext): Builder {
            this.coroutineContext = coroutineContext
            return this
        }

        @VisibleForTesting
463
464
        internal fun setBrowserDbPath(name: String): Builder {
            browserDbPath = name
465
466
467
            return this
        }

468
469
470
471
472
473
474
475
476
477
478
479
        @VisibleForTesting
        internal fun setSignonsDbName(name: String): Builder {
            signonsDbName = name
            return this
        }

        @VisibleForTesting
        internal fun setKey4DbName(name: String): Builder {
            key4DbName = name
            return this
        }

480
481
482
483
484
        @VisibleForTesting
        internal fun setProfile(profile: FennecProfile): Builder {
            fennecProfile = profile
            return this
        }
Grisha Kruglov's avatar
Grisha Kruglov committed
485
486
487
488
489
490

        @VisibleForTesting
        internal fun setFxaState(state: File): Builder {
            fxaState = state
            return this
        }
491
492
493
494
495
496
497
498
499
500
501
502
    }

    private val logger = Logger("FennecMigrator")

    // Used to ensure migration runs do not overlap.
    private val migrationLock = Object()

    /**
     * Performs configured data migration. See [Builder] for how to configure a data migration.
     *
     * @return A deferred [MigrationResults], describing which migrations were performed and if they succeeded.
     */
503
504
505
    fun migrateAsync(
        store: MigrationStore
    ): Deferred<MigrationResults> = synchronized(migrationLock) {
506
        val migrationsToRun = getMigrationsToRun()
507
508
509
        // This check is performed in `startMigrationIfNeeded`, but this method may also be called
        // directly so we repeat it here.
        val isFennecInstall = isFennecInstallation()
510
511

        // Short-circuit if there's nothing to do.
512
513
        if (migrationsToRun.isEmpty() || !isFennecInstall) {
            logger.debug("No migrations to run. Fennec install - $isFennecInstall.")
514
515
516
517
518
            val result = CompletableDeferred<MigrationResults>()
            result.complete(emptyMap())
            return result
        }

519
520
521
522
523
524
525
        // If we need to run an FxA migration later, first make sure 'accountManager' is initialized
        // while we're still on the main thread. This is necessary because accountManager can't be
        // initialized on a background thread.
        if (migrationsToRun.any { it.migration is Migration.FxA }) {
            accountManager?.value
        }

526
        return runMigrationsAsync(store, migrationsToRun)
527
528
529
530
531
532
533
534
535
    }

    /**
     * Returns true if there are migrations to run for this installation.
     */
    private fun hasMigrationsToRun(): Boolean {
        return getMigrationsToRun().isNotEmpty()
    }

536
537
538
539
540
541
    @VisibleForTesting
    internal fun isFennecInstallation(): Boolean {
        // We consider presence of 'browser.db' as a proxy for 'this is a fennec install'.
        // Just 'profile' isn't enough, since a profile directory will be created by GV in the same
        // way as it may be already present in Fennec. However, in Fenix we do not have 'browser.db'.
        return browserDbPath?.let { File(it).exists() } ?: false
542
543
    }

544
    /**
545
546
     * If a migration is needed then invoking this method will update the [MigrationStore] and launch
     * the provided [AbstractMigrationService] implementation.
547
     */
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
    fun <T> startMigrationIfNeeded(
        store: MigrationStore,
        service: Class<T>
    ) where T : AbstractMigrationService {
        if (!isFennecInstallation()) {
            // This installation seems to never have been Fennec, so we do not need
            // to migrate anything.
            logger.debug("This is not a Fennec installation. No migration needed.")
            return
        }

        if (!hasMigrationsToRun()) {
            // There are no migrations to run for this installation. This likely means that we are
            // migrated already and there are no updated migrations to run.
            logger.debug("This is a Fennec installation. But there are no migrations to run.")
            return
564
        }
565
566
567
568
569
570
571

        logger.debug("Migration is needed. Updating state and starting service.")

        runBlocking {
            store.dispatch(MigrationAction.Started).join()
        }
        ContextCompat.startForegroundService(context, Intent(context, service))
572
573

        emitStartedFact()
574
575
576
    }

    private fun getMigrationsToRun(): List<VersionedMigration> {
577
578
579
580
        val migrationRecord = MigrationResultsStore(context)
        val migrationHistory = migrationRecord.getCached()

        // Either didn't run before, or ran with an older version than current migration's version.
581
        return migrations.filter { versionedMigration ->
582
583
584
585
586
587
588
589
590
            val pastVersion = migrationHistory?.get(versionedMigration.migration)?.version
            if (pastVersion == null) {
                true
            } else {
                versionedMigration.version > pastVersion
            }
        }
    }

591
    @Suppress("ComplexMethod", "TooGenericExceptionCaught")
592
    private fun runMigrationsAsync(
593
        store: MigrationStore,
594
        migrations: List<VersionedMigration>
Grisha Kruglov's avatar
Grisha Kruglov committed
595
596
597
598
599
    ): Deferred<MigrationResults> = CoroutineScope(coroutineContext).async {

        // Note that we're depending on coroutineContext to be backed by a single-threaded executor, in order to ensure
        // non-overlapping execution of our migrations.

600
        val resultStore = MigrationResultsStore(context)
601
602
603
        val results = mutableMapOf<Migration, MigrationRun>()

        migrations.forEach { versionedMigration ->
Grisha Kruglov's avatar
Grisha Kruglov committed
604
            logger.debug("Executing $versionedMigration")
605

Grisha Kruglov's avatar
Grisha Kruglov committed
606
607
608
609
            val telemetryId = versionedMigration.migration.telemetryIdentifier()
            val migrationVersion = versionedMigration.version
            MigrationPing.migrationVersions[telemetryId].set("$migrationVersion")

610
611
            versionedMigration.migration.metricTotalDuration().start()

612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
            val migrationResult: Result<*> = try {
                when (versionedMigration.migration) {
                    Migration.History -> migrateHistory()
                    Migration.Bookmarks -> migrateBookmarks()
                    Migration.OpenTabs -> migrateOpenTabs()
                    Migration.FxA -> migrateFxA()
                    Migration.Gecko -> migrateGecko(versionedMigration.version)
                    Migration.Logins -> migrateLogins()
                    Migration.Settings -> migrateSharedPrefs()
                    Migration.Addons -> migrateAddons()
                    Migration.TelemetryIdentifiers -> migrateTelemetryIdentifiers()
                    Migration.SearchEngine -> migrateSearchEngine()
                    Migration.PinnedSites -> migratePinnedSites()
                }
            } catch (e: Exception) {
                logger.error("Unexpected error while migrating $versionedMigration", e)
                crashReporter.submitCaughtException(FennecMigratorException.HighLevel(e))
                Result.Failure<Any>(e)
            } finally {
                versionedMigration.migration.metricTotalDuration().stop()
632
633
            }

634
            val migrationRun = when (migrationResult) {
635
636
637
638
639
                is Result.Failure<*> -> {
                    logger.error(
                        "Failed to migrate $versionedMigration",
                        migrationResult.throwables.first()
                    )
Grisha Kruglov's avatar
Grisha Kruglov committed
640

641
                    versionedMigration.migration.metricAnyFailures().set(true)
642
643
644
645
646
647
648
649
650
                    MigrationRun(versionedMigration.version, false)
                }
                is Result.Success<*> -> {
                    logger.debug(
                        "Migrated $versionedMigration"
                    )
                    MigrationRun(versionedMigration.version, true)
                }
            }
651

652
653
654
655
            // Submit migration telemetry that we've gathered up to this point. We do this out of
            // abundance of caution, in case we crash later.
            Pings.migration.submit()

656
657
658
659
660
661
            // Notify MigrationStore about result.
            store.dispatch(MigrationAction.MigrationRunResult(
                versionedMigration.migration,
                migrationRun
            ))

662
663
            // Save result of this migration immediately, so that we keep it even if we crash later
            // in the process and do not rerun this migration version.
664
            resultStore.setOrUpdate(mapOf(versionedMigration.migration to migrationRun))
665
666

            results[versionedMigration.migration] = migrationRun
667
668
669
        }

        results
Grisha Kruglov's avatar
Grisha Kruglov committed
670
    }
671

672
    @SuppressWarnings("TooGenericExceptionCaught", "MagicNumber", "ReturnCount")
673
674
675
    private fun migrateHistory(): Result<Unit> {
        checkNotNull(historyStorage) { "History storage must be configured to migrate history" }

676
        // There's no dbPath without a profile, but if a profile is present we expect dbPath to be also present.
677
        if (profile != null && browserDbPath == null) {
Grisha Kruglov's avatar
Grisha Kruglov committed
678
            crashReporter.submitCaughtException(IllegalStateException("Missing DB path during history migration"))
679
680
        }

681
        if (browserDbPath == null) {
682
            MigrationHistory.failureReason.add(FailureReasonTelemetryCodes.HISTORY_MISSING_DB_PATH.code)
Grisha Kruglov's avatar
Grisha Kruglov committed
683
            return Result.Failure(IllegalStateException("Missing DB path during history migration"))
684
        }
685

Grisha Kruglov's avatar
Grisha Kruglov committed
686
        val migrationMetrics = try {
687
            logger.debug("Migrating history...")
688
            historyStorage.value.importFromFennec(browserDbPath)
689
        } catch (e: Exception) {
690
            crashReporter.submitCaughtException(FennecMigratorException.MigrateHistoryException(e))
691
            MigrationHistory.failureReason.add(FailureReasonTelemetryCodes.HISTORY_RUST_EXCEPTION.code)
Grisha Kruglov's avatar
Grisha Kruglov committed
692
            return Result.Failure(e)
693
        }
Grisha Kruglov's avatar
Grisha Kruglov committed
694
695
696

        // Process migration metrics. Here and elsewhere, we're assuming and hard-coding metrics schema.
        // See application-services repository: https://github.com/mozilla/application-services/commit/a7d5ff1903fb0f904785a1645cb7ae1d6c313f10
697
        return try {
Grisha Kruglov's avatar
Grisha Kruglov committed
698
699
700
701
702
            MigrationHistory.detected.add(migrationMetrics.getInt("num_total"))
            MigrationHistory.migrated["succeeded"].add(migrationMetrics.getInt("num_succeeded"))
            MigrationHistory.migrated["failed"].add(migrationMetrics.getInt("num_failed"))
            // Assuming that 'total_duration' is in milliseconds.
            MigrationHistory.duration.setRawNanos(migrationMetrics.getLong("total_duration") * 1000000)
703
704
705
706

            MigrationHistory.successReason.add(SuccessReasonTelemetryCodes.HISTORY_MIGRATED.code)
            logger.debug("Migrated history.")
            Result.Success(Unit)
Grisha Kruglov's avatar
Grisha Kruglov committed
707
        } catch (e: Exception) {
708
            MigrationHistory.failureReason.add(FailureReasonTelemetryCodes.HISTORY_TELEMETRY_EXCEPTION.code)
Grisha Kruglov's avatar
Grisha Kruglov committed
709
710
711
            crashReporter.submitCaughtException(
                FennecMigratorException.MigrateHistoryException(e)
            )
712
            Result.Failure(e)
Grisha Kruglov's avatar
Grisha Kruglov committed
713
        }
714
715
    }

716
    @SuppressWarnings("TooGenericExceptionCaught", "MagicNumber", "ReturnCount")
717
718
719
    private fun migrateBookmarks(): Result<Unit> {
        checkNotNull(bookmarksStorage) { "Bookmarks storage must be configured to migrate bookmarks" }

720
        // There's no dbPath without a profile, but if a profile is present we expect dbPath to be also present.
721
        if (profile != null && browserDbPath == null) {
Grisha Kruglov's avatar
Grisha Kruglov committed
722
            crashReporter.submitCaughtException(IllegalStateException("Missing DB path during bookmark migration"))
723
724
        }

725
        if (browserDbPath == null) {
726
            MigrationBookmarks.failureReason.add(FailureReasonTelemetryCodes.BOOKMARKS_MISSING_DB_PATH.code)
Grisha Kruglov's avatar
Grisha Kruglov committed
727
            return Result.Failure(IllegalStateException("Missing DB path during bookmark migration"))
728
        }
729

Grisha Kruglov's avatar
Grisha Kruglov committed
730
        val migrationMetrics = try {
731
            logger.debug("Migrating bookmarks...")
732
            bookmarksStorage.value.importFromFennec(browserDbPath)
733
        } catch (e: Exception) {
734
735
736
            crashReporter.submitCaughtException(
                FennecMigratorException.MigrateBookmarksException(e)
            )
737
            MigrationBookmarks.failureReason.add(FailureReasonTelemetryCodes.BOOKMARKS_RUST_EXCEPTION.code)
Grisha Kruglov's avatar
Grisha Kruglov committed
738
739
740
741
742
            return Result.Failure(e)
        }

        // Process migration metrics. Here and elsewhere, we're assuming and hard-coding metrics schema.
        // See application-services repository: https://github.com/mozilla/application-services/commit/b2e2edcc06a04503d493e1733b0d566815feac7c#diff-216f62325632ae6549587b038b21cfe0
743
        return try {
Grisha Kruglov's avatar
Grisha Kruglov committed
744
745
746
747
748
            MigrationBookmarks.detected.add(migrationMetrics.getInt("num_total"))
            MigrationBookmarks.migrated["succeeded"].add(migrationMetrics.getInt("num_succeeded"))
            MigrationBookmarks.migrated["failed"].add(migrationMetrics.getInt("num_failed"))
            // Assuming that 'total_duration' is in milliseconds.
            MigrationBookmarks.duration.setRawNanos(migrationMetrics.getLong("total_duration") * 1000000)
749
750
751
752

            logger.debug("Migrated bookmarks.")
            MigrationBookmarks.successReason.add(SuccessReasonTelemetryCodes.BOOKMARKS_MIGRATED.code)
            Result.Success(Unit)
Grisha Kruglov's avatar
Grisha Kruglov committed
753
        } catch (e: Exception) {
754
            MigrationBookmarks.failureReason.add(FailureReasonTelemetryCodes.BOOKMARKS_TELEMETRY_EXCEPTION.code)
Grisha Kruglov's avatar
Grisha Kruglov committed
755
756
757
            crashReporter.submitCaughtException(
                FennecMigratorException.MigrateBookmarksException(e)
            )
758
            Result.Failure(e)
759
760
761
        }
    }

762
763
764
765
    @Suppress("ComplexMethod", "TooGenericExceptionCaught", "LongMethod", "ReturnCount")
    private suspend fun migrateLogins(): Result<LoginsMigrationResult> {
        if (profile == null) {
            crashReporter.submitCaughtException(IllegalStateException("Missing Profile path"))
766
            MigrationLogins.failureReason.add(FailureReasonTelemetryCodes.LOGINS_MISSING_PROFILE.code)
767
768
769
            return Result.Failure(IllegalStateException("Missing Profile path"))
        }

770
        return try {
771
            logger.debug("Migrating logins...")
772
            val result = FennecLoginsMigration.migrate(
773
774
                crashReporter,
                signonsDbPath = "${profile.path}/$signonsDbName",
775
                key4DbPath = "${profile.path}/$key4DbName",
776
                loginsStorage = loginsStorage!!.value
777
778
            )

779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
            if (result is Result.Failure<LoginsMigrationResult>) {
                val migrationFailureWrapper = result.throwables.first() as LoginMigrationException
                when (val failure = migrationFailureWrapper.failure) {
                    is LoginsMigrationResult.Failure.FailedToCheckMasterPassword -> {
                        logger.error("Failed to check master password: $failure")
                        MigrationLogins.failureReason.add(FailureReasonTelemetryCodes.LOGINS_MP_CHECK.code)
                        // We definitely expect to be able to check our master password, so report a failure.
                        crashReporter.submitCaughtException(migrationFailureWrapper)
                        result
                    }
                    is LoginsMigrationResult.Failure.UnsupportedSignonsDbVersion -> {
                        logger.error("Unsupported logins database version: $failure")
                        MigrationLogins.failureReason.add(FailureReasonTelemetryCodes.LOGINS_UNSUPPORTED_LOGINS_DB.code)
                        MigrationLogins.unsupportedDbVersion.add(failure.version)
                        // We really don't expect anyone to hit this, so let's submit it to Sentry.
                        crashReporter.submitCaughtException(migrationFailureWrapper)
                        result
                    }
                    is LoginsMigrationResult.Failure.UnexpectedLoginsKeyMaterialAlg,
                    is LoginsMigrationResult.Failure.UnexpectedMetadataKeyMaterialAlg -> {
                        logger.error("Encryption failure: $failure")
                        MigrationLogins.failureReason.add(FailureReasonTelemetryCodes.LOGINS_ENCRYPTION.code)
                        // While this may happen in theory, let's keep track of exact reasons.
                        crashReporter.submitCaughtException(migrationFailureWrapper)
                        result
                    }
                    is LoginsMigrationResult.Failure.GetLoginsThrew -> {
                        logger.error("getLogins failure: $failure")
                        MigrationLogins.failureReason.add(FailureReasonTelemetryCodes.LOGINS_GET.code)
                        crashReporter.submitCaughtException(migrationFailureWrapper)
                        result
                    }
                    is LoginsMigrationResult.Failure.RustImportThrew -> {
                        logger.error("Rust import failure: $failure")
                        MigrationLogins.failureReason.add(FailureReasonTelemetryCodes.LOGINS_RUST_IMPORT.code)
                        crashReporter.submitCaughtException(migrationFailureWrapper)
                        result
                    }
817
                }
818
819
820
821
822
823
824
825
            } else {
                val loginMigrationSuccess = result as Result.Success<LoginsMigrationResult>
                when (val success = loginMigrationSuccess.value as LoginsMigrationResult.Success) {
                    is LoginsMigrationResult.Success.MasterPasswordIsSet -> {
                        logger.debug("Could not migrate logins - master password is set")
                        MigrationLogins.successReason.add(SuccessReasonTelemetryCodes.LOGINS_MP_SET.code)
                        result
                    }
826

827
828
                    is LoginsMigrationResult.Success.ImportedLoginRecords -> {
                        logger.debug("""Imported login records! Details:
829
830
831
832
                    Total detected=${success.totalRecordsDetected},
                    failed to process=${success.failedToProcess},
                    failed to import=${success.failedToImport}
                """.trimIndent())
833
834
835
836
837
838
839
                        MigrationLogins.successReason.add(SuccessReasonTelemetryCodes.LOGINS_MIGRATED.code)
                        MigrationLogins.detected.add(success.totalRecordsDetected)
                        MigrationLogins.failureCounts["process"].add(success.failedToProcess)
                        MigrationLogins.failureCounts["import"].add(success.failedToImport)
                        result
                    }
                }
840
            }
841
842
843
844
        } catch (e: Exception) {
            crashReporter.submitCaughtException(FennecMigratorException.MigrateLoginsException(e))
            MigrationLogins.failureReason.add(FailureReasonTelemetryCodes.LOGINS_UNEXPECTED_EXCEPTION.code)
            Result.Failure(e)
845
846
847
        }
    }

848
    @SuppressWarnings("TooGenericExceptionCaught")
849
    private suspend fun migrateOpenTabs(): Result<SessionManager.Snapshot> {
850
        if (profile == null) {
Grisha Kruglov's avatar
Grisha Kruglov committed
851
            crashReporter.submitCaughtException(IllegalStateException("Missing Profile path"))
852
            MigrationOpenTabs.failureReason.add(FailureReasonTelemetryCodes.OPEN_TABS_MISSING_PROFILE.code)
853
854
855
            return Result.Failure(IllegalStateException("Missing Profile path"))
        }

856
857
        logger.debug("Migrating session...")
        val result = try {
858
            FennecSessionMigration.migrate(File(profile.path), crashReporter)
859
860
861
862
863
864
865
866
        } catch (e: Exception) {
            MigrationOpenTabs.failureReason.add(FailureReasonTelemetryCodes.OPEN_TABS_MIGRATE_EXCEPTION.code)
            crashReporter.submitCaughtException(
                FennecMigratorException.MigrateOpenTabsException(e)
            )
            return Result.Failure(e)
        }

867
        return try {
Grisha Kruglov's avatar
Grisha Kruglov committed
868
            if (result is Result.Success<SessionManager.Snapshot>) {
869
                logger.debug("Loading migrated session snapshot...")
Grisha Kruglov's avatar
Grisha Kruglov committed
870
                MigrationOpenTabs.detected.add(result.value.sessions.size)
871
                withContext(Dispatchers.Main) {
Grisha Kruglov's avatar
Grisha Kruglov committed
872
873
874
875
                    sessionManager!!.restore(result.value)
                    // Note that this is assuming that sessionManager starts off empty before the
                    // migration.
                    MigrationOpenTabs.migrated.add(sessionManager.all.size)
876
                    MigrationOpenTabs.successReason.add(SuccessReasonTelemetryCodes.OPEN_TABS_MIGRATED.code)
877
                }
878
879
                result
            } else {
880
                MigrationOpenTabs.failureReason.add(FailureReasonTelemetryCodes.OPEN_TABS_NO_SNAPSHOT.code)
881
                result
882
883
            }
        } catch (e: Exception) {
884
            MigrationOpenTabs.failureReason.add(FailureReasonTelemetryCodes.OPEN_TABS_RESTORE_EXCEPTION.code)
885
886
887
            crashReporter.submitCaughtException(
                FennecMigratorException.MigrateOpenTabsException(e)
            )
888
889
890
            Result.Failure(e)
        }
    }
Grisha Kruglov's avatar
Grisha Kruglov committed
891

892
    @Suppress("ComplexMethod", "LongMethod", "TooGenericExceptionCaught", "NestedBlockDepth")
Grisha Kruglov's avatar
Grisha Kruglov committed
893
    private suspend fun migrateFxA(): Result<FxaMigrationResult> {
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
        return try {
            val result = FennecFxaMigration.migrate(fxaState!!, context, accountManager!!.value)

            if (result is Result.Failure<FxaMigrationResult>) {
                val migrationFailureWrapper = result.throwables.first()
                if (migrationFailureWrapper is FxaMigrationException) {
                    when (val failure = migrationFailureWrapper.failure) {
                        is FxaMigrationResult.Failure.CorruptAccountState -> {
                            logger.error("Detected a corrupt account state: $failure")
                            MigrationFxa.failureReason.add(FailureReasonTelemetryCodes.FXA_CORRUPT_ACCOUNT_STATE.code)
                            result
                        }
                        is FxaMigrationResult.Failure.UnsupportedVersions -> {
                            logger.error("Detected unsupported versions: $failure")
                            MigrationFxa.failureReason.add(FailureReasonTelemetryCodes.FXA_UNSUPPORTED_VERSIONS.code)
                            MigrationFxa.unsupportedAccountVersion.set("${failure.accountVersion}")
                            MigrationFxa.unsupportedPickleVersion.set("${failure.pickleVersion}")
                            MigrationFxa.unsupportedStateVersion.set("${failure.stateVersion}")
                            result
                        }
                        is FxaMigrationResult.Failure.FailedToSignIntoAuthenticatedAccount -> {
                            logger.error("Failed to sign-in into an authenticated account")
                            MigrationFxa.failureReason.add(FailureReasonTelemetryCodes.FXA_SIGN_IN_FAILED.code)
                            crashReporter.submitCaughtException(migrationFailureWrapper)
                            result
                        }
                        is FxaMigrationResult.Failure.CustomServerConfigPresent -> {
                            logger.error("Custom config present: token=${failure.customTokenServer}," +
                                "idp=${failure.customIdpServer}")
                            MigrationFxa.failureReason.add(FailureReasonTelemetryCodes.FXA_CUSTOM_SERVER.code)
                            MigrationFxa.hasCustomIdpServer.set(failure.customIdpServer)
                            MigrationFxa.hasCustomTokenServer.set(failure.customTokenServer)
                            result
                        }
                    }
                } else {
                    logger.error("Unexpected FxA migration exception", migrationFailureWrapper.cause)
                    MigrationFxa.failureReason.add(FailureReasonTelemetryCodes.FXA_MIGRATE_EXCEPTION.code)
Grisha Kruglov's avatar
Grisha Kruglov committed
932
933
                    result
                }
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
            } else {
                val migrationSuccess = result as Result.Success<FxaMigrationResult>
                when (val success = migrationSuccess.value as FxaMigrationResult.Success) {
                    // The rest are all successful migrations.
                    is FxaMigrationResult.Success.NoAccount -> {
                        logger.debug("No Fennec account detected")
                        MigrationFxa.successReason.add(SuccessReasonTelemetryCodes.FXA_NO_ACCOUNT.code)
                        result
                    }
                    is FxaMigrationResult.Success.UnauthenticatedAccount -> {
                        // Here we have an 'email' and a state label.
                        // "Bad auth state" could be a few things - unverified account, bad credentials
                        // detected by Fennec, etc
                        // We could try using the 'email' address as a starting point in the authentication flow.
                        logger.debug("Detected a Fennec account in a bad authentication state: ${success.stateLabel}")
                        MigrationFxa.successReason.add(SuccessReasonTelemetryCodes.FXA_BAD_AUTH.code)
                        MigrationFxa.badAuthState.set(success.stateLabel)
                        result
                    }
                    is FxaMigrationResult.Success.SignedInIntoAuthenticatedAccount -> {
                        logger.debug("Signed-in into a detected Fennec account")
                        MigrationFxa.successReason.add(SuccessReasonTelemetryCodes.FXA_SIGNED_IN.code)
                        result
                    }
                    is FxaMigrationResult.Success.WillAutoRetrySignInLater -> {
                        logger.debug("Will auto-retry Fennec account migration later")
                        MigrationFxa.successReason.add(SuccessReasonTelemetryCodes.FXA_WILL_RETRY.code)
                        result
                    }
963
                }
Grisha Kruglov's avatar
Grisha Kruglov committed
964
            }
965
966
967
968
969
        } catch (e: Exception) {
            logger.error("Unexpected FxA migration exception", e)
            MigrationFxa.failureReason.add(FailureReasonTelemetryCodes.FXA_MIGRATE_EXCEPTION.code)
            crashReporter.submitCaughtException(FennecMigratorException.MigrateFxaException(e))
            Result.Failure(e)
Grisha Kruglov's avatar
Grisha Kruglov committed
970
971
        }
    }
972

973
974
    @Suppress("ComplexMethod", "LongMethod", "TooGenericExceptionCaught", "ReturnCount")
    private fun migrateGecko(migrationVersion: Int): Result<GeckoMigrationResult> {
975
        if (profile == null) {
976
            MigrationGecko.failureReason.add(FailureReasonTelemetryCodes.GECKO_MISSING_PROFILE.code)
977
978
979
980
981
            return Result.Failure(IllegalStateException("Missing Profile path"))
        }

        return try {
            logger.debug("Migrating gecko files...")
982
            val result = GeckoMigration.migrate(profile.path, migrationVersion)
983
            logger.debug("Migrated gecko files.")
984
985
986

            if (result is Result.Failure<GeckoMigrationResult>) {
                val geckoFailureWrapper = result.throwables.first() as GeckoMigrationException
987
                when (val failure = geckoFailureWrapper.failure) {
988
989
990
991
992
993
994
995
996
997
998
999
1000
                    is GeckoMigrationResult.Failure.FailedToDeleteFile -> {
                        logger.error("Failed to delete prefs.js file: $failure")
                        MigrationGecko.failureReason.add(FailureReasonTelemetryCodes.GECKO_FAILED_TO_DELETE_PREFS.code)
                        result
                    }
                    is GeckoMigrationResult.Failure.FailedToWriteBackup -> {
                        logger.error("Failed to write backup of prefs.js: $failure")
                        MigrationGecko.failureReason.add(FailureReasonTelemetryCodes.GECKO_FAILED_TO_WRITE_BACKUP.code)
                        result
                    }
                    is GeckoMigrationResult.Failure.FailedToWritePrefs -> {
                        logger.error("Failed to write transformed prefs.js: $failure")
                        MigrationGecko.failureReason.add(FailureReasonTelemetryCodes.GECKO_FAILED_TO_WRITE_PREFS.code)