Commit df01bdf6 authored by Grisha Kruglov's avatar Grisha Kruglov
Browse files

Basics of the Fennec data migration API

parent d39dd495
......@@ -39,6 +39,9 @@ internal interface Connection : Closeable {
// strange split that doesn't quite map all that well to our internal storage model.
fun syncHistory(syncInfo: SyncAuthInfo)
fun syncBookmarks(syncInfo: SyncAuthInfo)
fun importVisitsFromFennec(dbPath: String)
fun importBookmarksFromFennec(dbPath: String)
}
/**
......@@ -91,6 +94,16 @@ internal object RustPlacesConnection : Connection {
SyncTelemetry.processBookmarksPing(ping)
}
override fun importVisitsFromFennec(dbPath: String) {
check(api != null) { "must call init first" }
api!!.importVisitsFromFennec(dbPath)
}
override fun importBookmarksFromFennec(dbPath: String) {
check(api != null) { "must call init first" }
api!!.importBookmarksFromFennec(dbPath)
}
override fun close() = synchronized(this) {
check(api != null) { "must call init first" }
api!!.close()
......
......@@ -12,6 +12,7 @@ import mozilla.appservices.places.BookmarkSeparator
import mozilla.appservices.places.BookmarkTreeNode
import mozilla.appservices.places.BookmarkUpdateInfo
import mozilla.appservices.places.PlacesApi
import mozilla.appservices.places.PlacesException
import mozilla.components.concept.storage.BookmarkInfo
import mozilla.components.concept.storage.BookmarkNode
import mozilla.components.concept.storage.BookmarkNodeType
......@@ -185,6 +186,17 @@ open class PlacesBookmarksStorage(context: Context) : PlacesStorage(context), Bo
}
}
/**
* Import bookmarks data from Fennec's browser.db file.
* Before running this, first run [PlacesHistoryStorage.importFromFennec] to import history and visits data.
*
* @param dbPath Absolute path to Fennec's browser.db file.
*/
@Throws(PlacesException::class)
fun importFromFennec(dbPath: String) {
places.importBookmarksFromFennec(dbPath)
}
/**
* This should be removed. See: https://github.com/mozilla/application-services/issues/1877
*
......
......@@ -7,6 +7,7 @@ package mozilla.components.browser.storage.sync
import android.content.Context
import kotlinx.coroutines.withContext
import mozilla.appservices.places.PlacesApi
import mozilla.appservices.places.PlacesException
import mozilla.appservices.places.VisitObservation
import mozilla.components.concept.storage.HistoryAutocompleteResult
import mozilla.components.concept.storage.HistoryStorage
......@@ -191,6 +192,16 @@ open class PlacesHistoryStorage(context: Context) : PlacesStorage(context), Hist
}
}
/**
* Import history and visits data from Fennec's browser.db file.
*
* @param dbPath Absolute path to Fennec's browser.db file.
*/
@Throws(PlacesException::class)
fun importFromFennec(dbPath: String) {
places.importVisitsFromFennec(dbPath)
}
/**
* This should be removed. See: https://github.com/mozilla/application-services/issues/1877
*
......
......@@ -22,7 +22,6 @@ import org.junit.Before
import org.junit.Test
import org.junit.runner.RunWith
@RunWith(AndroidJUnit4::class)
class PlacesBookmarksStorageTest {
private lateinit var bookmarks: PlacesBookmarksStorage
......@@ -88,7 +87,7 @@ class PlacesBookmarksStorageTest {
val insertedItem = bookmarks.addItem(BookmarkRoot.Mobile.id, url, "Mozilla", 5)
with (bookmarks.getBookmarksWithUrl(url)) {
with(bookmarks.getBookmarksWithUrl(url)) {
assertEquals(1, this.size)
with(this[0]) {
assertEquals(insertedItem, this.guid)
......@@ -105,7 +104,7 @@ class PlacesBookmarksStorageTest {
bookmarks.updateNode(insertedItem, BookmarkInfo(
parentGuid = folderGuid, title = null, position = -3, url = null
))
with (bookmarks.getBookmarksWithUrl(url)) {
with(bookmarks.getBookmarksWithUrl(url)) {
assertEquals(1, this.size)
with(this[0]) {
assertEquals(insertedItem, this.guid)
......@@ -118,23 +117,23 @@ class PlacesBookmarksStorageTest {
}
val separatorGuid = bookmarks.addSeparator(folderGuid, 1)
with (bookmarks.getTree(folderGuid)!!) {
with(bookmarks.getTree(folderGuid)!!) {
assertEquals(2, this.children!!.size)
assertEquals(BookmarkNodeType.SEPARATOR, this.children!![1].type)
}
assertTrue(bookmarks.deleteNode(separatorGuid))
with (bookmarks.getTree(folderGuid)!!) {
with(bookmarks.getTree(folderGuid)!!) {
assertEquals(1, this.children!!.size)
assertEquals(BookmarkNodeType.ITEM, this.children!![0].type)
}
with (bookmarks.searchBookmarks("mozilla")) {
with(bookmarks.searchBookmarks("mozilla")) {
assertEquals(1, this.size)
assertEquals("http://www.mozilla.org/", this[0].url)
}
with (bookmarks.getBookmark(folderGuid)!!) {
with(bookmarks.getBookmark(folderGuid)!!) {
assertEquals(folderGuid, this.guid)
assertEquals("Test Folder", this.title)
assertEquals(BookmarkRoot.Mobile.id, this.parentGuid)
......@@ -151,8 +150,95 @@ class PlacesBookmarksStorageTest {
} catch (e: PlacesException) {}
}
with (bookmarks.searchBookmarks("mozilla")) {
with(bookmarks.searchBookmarks("mozilla")) {
assertTrue(this.isEmpty())
}
}
@Test
fun `bookmarks import v0 empty`() {
// Doesn't have a schema or a set user_version pragma.
val path = getTestPath("databases/empty-v0.db").absolutePath
try {
bookmarks.importFromFennec(path)
fail("Expected v0 database to be unsupported")
} catch (e: PlacesException) {
// This is a little brittle, but the places library doesn't have a proper error type for this.
assertEquals("Database version 0 is not supported", e.message)
}
}
@Test
fun `bookmarks import v38 populated`() {
// Fennec v38 schema populated with data.
val path = getTestPath("databases/populated-v38.db").absolutePath
try {
bookmarks.importFromFennec(path)
fail("Expected v38 database to be unsupported")
} catch (e: PlacesException) {
// This is a little brittle, but the places library doesn't have a proper error type for this.
assertEquals("Database version 38 is not supported", e.message)
}
}
@Test
fun `bookmarks import v39 populated`() = runBlocking {
val path = getTestPath("databases/populated-v39.db").absolutePath
// Need to import history first before we import bookmarks.
PlacesHistoryStorage(testContext).importFromFennec(path)
bookmarks.importFromFennec(path)
with(bookmarks.getTree(BookmarkRoot.Root.id)!!) {
assertEquals(4, this.children!!.size)
val children = this.children!!.map { it.guid }
assertTrue(BookmarkRoot.Mobile.id in children)
assertTrue(BookmarkRoot.Unfiled.id in children)
assertTrue(BookmarkRoot.Toolbar.id in children)
assertTrue(BookmarkRoot.Menu.id in children)
// Note that we dropped the special "pinned" folder during a migration.
// See https://github.com/mozilla/application-services/issues/1989
}
with(bookmarks.getTree(BookmarkRoot.Mobile.id)!!) {
assertEquals(6, this.children!!.size)
with(this.children!![0]) {
assertEquals("Business & Financial News, Breaking US & International News | Reuters", this.title)
assertEquals("https://mobile.reuters.com/", this.url)
assertEquals("2hazimCy0hhS", this.guid)
assertEquals(BookmarkNodeType.ITEM, this.type)
}
with(this.children!![1]) {
assertEquals("There is a way to protect your privacy. Join Firefox.", this.title)
assertEquals("https://www.mozilla.org/en-US/firefox/accounts/", this.url)
assertEquals("mUcVvqUfJs6r", this.guid)
assertEquals(BookmarkNodeType.ITEM, this.type)
}
with(this.children!![2]) {
assertEquals("Internet for people, not profit — Mozilla", this.title)
assertEquals("https://www.mozilla.org/en-US/", this.url)
assertEquals("tL-ucG5eaoG-", this.guid)
assertEquals(BookmarkNodeType.ITEM, this.type)
}
with(this.children!![3]) {
assertEquals("Firefox: About your browser", this.title)
assertEquals("about:firefox", this.url)
assertEquals("kR_18w0gDLHq", this.guid)
assertEquals(BookmarkNodeType.ITEM, this.type)
}
with(this.children!![4]) {
assertEquals("Firefox: Customize with add-ons", this.title)
assertEquals("https://addons.mozilla.org/android?utm_source=inproduct&utm_medium=default-bookmarks&utm_campaign=mobileandroid", this.url)
assertEquals("bTuLpp58gwqw", this.guid)
assertEquals(BookmarkNodeType.ITEM, this.type)
}
with(this.children!![5]) {
assertEquals("Firefox: Support", this.title)
assertEquals("https://support.mozilla.org/products/mobile?utm_source=inproduct&utm_medium=default-bookmarks&utm_campaign=mobileandroid", this.url)
assertEquals("nbfDW0QSBEKu", this.guid)
assertEquals(BookmarkNodeType.ITEM, this.type)
}
}
}
}
......@@ -26,6 +26,7 @@ import org.junit.Assert.fail
import org.junit.Before
import org.junit.Test
import org.junit.runner.RunWith
import java.io.File
@RunWith(AndroidJUnit4::class)
class PlacesHistoryStorageTest {
......@@ -508,6 +509,14 @@ class PlacesHistoryStorageTest {
return 0L
}
override fun importVisitsFromFennec(dbPath: String) {
fail()
}
override fun importBookmarksFromFennec(dbPath: String) {
fail()
}
override fun close() {
fail()
}
......@@ -545,6 +554,14 @@ class PlacesHistoryStorageTest {
return 0L
}
override fun importVisitsFromFennec(dbPath: String) {
fail()
}
override fun importBookmarksFromFennec(dbPath: String) {
fail()
}
override fun close() {
fail()
}
......@@ -582,6 +599,14 @@ class PlacesHistoryStorageTest {
return 0L
}
override fun importVisitsFromFennec(dbPath: String) {
fail()
}
override fun importBookmarksFromFennec(dbPath: String) {
fail()
}
override fun close() {
fail()
}
......@@ -623,6 +648,14 @@ class PlacesHistoryStorageTest {
return 0L
}
override fun importVisitsFromFennec(dbPath: String) {
fail()
}
override fun importBookmarksFromFennec(dbPath: String) {
fail()
}
override fun close() {
fail()
}
......@@ -631,4 +664,94 @@ class PlacesHistoryStorageTest {
storage.sync(SyncAuthInfo("kid", "token", 123L, "key", "serverUrl"))
fail()
}
@Test
fun `history import v0 empty`() {
// Doesn't have a schema or a set user_version pragma.
val path = getTestPath("databases/empty-v0.db").absolutePath
try {
history.importFromFennec(path)
fail("Expected v0 database to be unsupported")
} catch (e: PlacesException) {
// This is a little brittle, but the places library doesn't have a proper error type for this.
assertEquals("Database version 0 is not supported", e.message)
}
}
@Test
fun `history import v38 populated`() {
// Fennec v38 schema populated with data.
val path = getTestPath("databases/populated-v38.db").absolutePath
try {
history.importFromFennec(path)
fail("Expected v38 database to be unsupported")
} catch (e: PlacesException) {
// This is a little brittle, but the places library doesn't have a proper error type for this.
assertEquals("Database version 38 is not supported", e.message)
}
}
@Test
fun `history import v39 populated`() = runBlocking {
val path = getTestPath("databases/populated-v39.db").absolutePath
var visits = history.getDetailedVisits(0, Long.MAX_VALUE)
assertEquals(0, visits.size)
history.importFromFennec(path)
visits = history.getDetailedVisits(0, Long.MAX_VALUE)
assertEquals(6, visits.size)
assertEquals(listOf(false, true, true, true, true, true, true), history.reader.getVisited(listOf(
"files:///",
"https://news.ycombinator.com/",
"https://news.ycombinator.com/item?id=21224209",
"https://mobile.twitter.com/random_walker/status/1182635589604171776",
"https://www.mozilla.org/en-US/",
"https://www.mozilla.org/en-US/firefox/accounts/",
"https://mobile.reuters.com/"
)))
with(visits[0]) {
assertEquals("Hacker News", this.title)
assertEquals("https://news.ycombinator.com/", this.url)
assertEquals(1570822280639, this.visitTime)
assertEquals(VisitType.LINK, this.visitType)
}
with(visits[1]) {
assertEquals("Why Enterprise Software Sucks | Hacker News", this.title)
assertEquals("https://news.ycombinator.com/item?id=21224209", this.url)
assertEquals(1570822283117, this.visitTime)
assertEquals(VisitType.LINK, this.visitType)
}
with(visits[2]) {
assertEquals("Arvind Narayanan on Twitter: \"My university just announced that it’s dumping Blackboard, and there was much rejoicing. Why is Blackboard universally reviled? There’s a standard story of why \"enterprise software\" sucks. If you’ll bear with me, I think this is best appreciated by talking about… baby clothes!\" / Twitter", this.title)
assertEquals("https://mobile.twitter.com/random_walker/status/1182635589604171776", this.url)
assertEquals(1570822287349, this.visitTime)
assertEquals(VisitType.LINK, this.visitType)
}
with(visits[3]) {
assertEquals("Internet for people, not profit — Mozilla", this.title)
assertEquals("https://www.mozilla.org/en-US/", this.url)
assertEquals(1570830201733, this.visitTime)
assertEquals(VisitType.LINK, this.visitType)
}
with(visits[4]) {
assertEquals("There is a way to protect your privacy. Join Firefox.", this.title)
assertEquals("https://www.mozilla.org/en-US/firefox/accounts/", this.url)
assertEquals(1570830207742, this.visitTime)
assertEquals(VisitType.LINK, this.visitType)
}
with(visits[5]) {
assertEquals("", this.title)
assertEquals("https://mobile.reuters.com/", this.url)
assertEquals(1570830217562, this.visitTime)
assertEquals(VisitType.LINK, this.visitType)
}
}
}
fun getTestPath(path: String): File {
return PlacesHistoryStorage::class.java.classLoader!!
.getResource(path).file
.let { File(it) }
}
\ No newline at end of file
......@@ -23,10 +23,29 @@ android {
}
}
configurations {
// There's an interaction between Gradle's resolution of dependencies with different types
// (@jar, @aar) for `implementation` and `testImplementation` and with Android Studio's built-in
// JUnit test runner. The runtime classpath in the built-in JUnit test runner gets the
// dependency from the `implementation`, which is type @aar, and therefore the JNA dependency
// doesn't provide the JNI dispatch libraries in the correct Java resource directories. I think
// what's happening is that @aar type in `implementation` resolves to the @jar type in
// `testImplementation`, and that it wins the dependency resolution battle.
//
// A workaround is to add a new configuration which depends on the @jar type and to reference
// the underlying JAR file directly in `testImplementation`. This JAR file doesn't resolve to
// the @aar type in `implementation`. This works when invoked via `gradle`, but also sets the
// correct runtime classpath when invoked with Android Studio's built-in JUnit test runner.
// Success!
jnaForTest
}
dependencies {
implementation project(':concept-engine')
implementation project(':browser-session')
implementation project(':browser-storage-sync')
implementation Dependencies.kotlin_stdlib
implementation Dependencies.kotlin_coroutines
......@@ -38,6 +57,10 @@ dependencies {
testImplementation Dependencies.testing_coroutines
testImplementation Dependencies.testing_robolectric
testImplementation Dependencies.testing_mockito
jnaForTest Dependencies.thirdparty_jna
testImplementation files(configurations.jnaForTest.copyRecursive().files)
testImplementation Dependencies.mozilla_full_megazord_forUnitTests
}
apply from: '../../../publish.gradle'
......
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package mozilla.components.support.migration
import android.content.Context
import androidx.annotation.VisibleForTesting
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Deferred
import kotlinx.coroutines.asCoroutineDispatcher
import kotlinx.coroutines.async
import mozilla.components.browser.session.SessionManager
import mozilla.components.browser.storage.sync.PlacesBookmarksStorage
import mozilla.components.browser.storage.sync.PlacesHistoryStorage
import mozilla.components.support.base.log.logger.Logger
import java.io.File
import java.lang.Exception
import java.lang.IllegalStateException
import java.util.concurrent.Executors
import kotlin.coroutines.CoroutineContext
/**
* Supported Fennec migrations and their current versions.
*/
sealed class Migration(val currentVersion: Int) {
/**
* Migrates history (both "places" and "visits").
*/
object History : Migration(currentVersion = 1)
/**
* Migrates bookmarks. Must run after history was migrated.
*/
object Bookmarks : Migration(currentVersion = 1)
/**
* Migrates open tabs.
*/
object OpenTabs : Migration(currentVersion = 1)
}
/**
* Describes a [Migration] at a specific version, enforcing in-range version specification.
*
* @property migration A [Migration] in question.
* @property version Version of the [migration], defaulting to the current version.
*/
data class VersionedMigration(val migration: Migration, val version: Int = migration.currentVersion) {
init {
require(version <= migration.currentVersion && version >= 1) {
"Migration version must be between 1 and current version"
}
}
}
/**
* Entrypoint for Fennec data migration. See [Builder] for public API.
*
* @param context Application context used for accessing the file system.
* @param migrations Describes ordering and versioning of migrations to run.
* @param historyStorage An optional instance of [PlacesHistoryStorage] used to store migrated history data.
* @param bookmarksStorage An optional instance of [PlacesBookmarksStorage] used to store migrated bookmarks data.
* @param coroutineContext An instance of [CoroutineContext] used for executing async migration tasks.
*/
class FennecMigrator private constructor(
private val context: Context,
private val migrations: List<VersionedMigration>,
private val historyStorage: PlacesHistoryStorage?,
private val bookmarksStorage: PlacesBookmarksStorage?,
private val sessionManager: SessionManager?,
private val profile: FennecProfile?,
private val browserDbName: String,
private val coroutineContext: CoroutineContext
) {
/**
* Data migration builder. Allows configuring which migrations to run, their versions and relative order.
*/
class Builder(private val context: Context) {
private var historyStorage: PlacesHistoryStorage? = null
private var bookmarksStorage: PlacesBookmarksStorage? = null
private var sessionManager: SessionManager? = null
private val migrations: MutableList<VersionedMigration> = mutableListOf()
// Single-thread executor to ensure we don't accidentally parallelize migrations.
private var coroutineContext: CoroutineContext = Executors.newSingleThreadExecutor().asCoroutineDispatcher()
private var fennecProfile = FennecProfile.findDefault(context)
private var browserDbName = "browser.db"
/**
* Enable history migration.
*
* @param storage An instance of [PlacesHistoryStorage], used for storing data.
* @param version Version of the migration; defaults to the current version.
*/
fun migrateHistory(storage: PlacesHistoryStorage, version: Int = Migration.History.currentVersion): Builder {
historyStorage = storage
migrations.add(VersionedMigration(Migration.History, version))
return this
}
/**
* Enable bookmarks migration. Must be called after [migrateHistory].
*
* @param storage An instance of [PlacesBookmarksStorage], used for storing data.
* @param version Version of the migration; defaults to the current version.
*/
fun migrateBookmarks(
storage: PlacesBookmarksStorage,
version: Int = Migration.Bookmarks.currentVersion