Issue #1968 - Add FetchDownloadManager

components/browser/session/src/main/java/mozilla/components/browser/session/Download.kt

@@ -15,6 +15,7 @@ import android.os.Environment
  * @property contentLength The file size reported by the server.
  * @property userAgent The user agent to be used for the download.
  * @property destinationDirectory The matching destination directory for this type of download.
+ * @property referrerUrl The site that linked to this download.
  */
 data class Download(
     val url: String,
@@ -22,5 +23,6 @@ data class Download(
     val contentType: String? = null,
     val contentLength: Long? = null,
     val userAgent: String? = null,
-    val destinationDirectory: String = Environment.DIRECTORY_DOWNLOADS
+    val destinationDirectory: String = Environment.DIRECTORY_DOWNLOADS,
+    val referrerUrl: String? = null
 )

components/concept/fetch/src/main/java/mozilla/components/concept/fetch/Headers.kt

@@ -4,8 +4,6 @@
 
 package mozilla.components.concept.fetch
 
-import java.lang.IllegalArgumentException
-
 /**
  * A collection of HTTP [Headers] (immutable) of a [Request] or [Response].
  */
@@ -52,7 +50,11 @@ interface Headers : Iterable<Header> {
      * @see [Headers.Values]
      */
     object Names {
+        const val CONTENT_DISPOSITION = "Content-Disposition"
+        const val CONTENT_LENGTH = "Content-Length"
         const val CONTENT_TYPE = "Content-Type"
+        const val COOKIE = "Cookie"
+        const val REFERRER = "Referer"
         const val USER_AGENT = "User-Agent"
     }
 
@@ -83,12 +85,13 @@ data class Header(
 /**
  * A collection of HTTP [Headers] (mutable) of a [Request] or [Response].
  */
-class MutableHeaders(
-    vararg pairs: Pair<String, String>
-) : Headers, MutableIterable<Header> {
-    private val headers: MutableList<Header> = pairs.map {
-            (name, value) -> Header(name, value)
-    }.toMutableList()
+class MutableHeaders(headers: List<Header>) : Headers, MutableIterable<Header> {
+
+    private val headers = headers.toMutableList()
+
+    constructor(vararg pairs: Pair<String, String>) : this(
+        pairs.map { (name, value) -> Header(name, value) }.toMutableList()
+    )
 
     /**
      * Gets the [Header] at the specified [index].
@@ -151,4 +154,8 @@ class MutableHeaders(
 
         return append(name, value)
     }
+
+    override fun equals(other: Any?) = other is MutableHeaders && headers == other.headers
+
+    override fun hashCode() = headers.hashCode()
 }

components/feature/downloads/build.gradle

@@ -30,10 +30,12 @@ dependencies {
     implementation project(':support-utils')
 
     implementation Dependencies.androidx_core_ktx
+    implementation Dependencies.kotlin_coroutines
     implementation Dependencies.kotlin_stdlib
 
     testImplementation Dependencies.androidx_test_core
     testImplementation Dependencies.androidx_test_junit
+    testImplementation Dependencies.kotlin_coroutines_test
     testImplementation Dependencies.testing_robolectric
     testImplementation Dependencies.testing_mockito
     testImplementation project(':support-test')

components/feature/downloads/src/main/AndroidManifest.xml

@@ -7,4 +7,5 @@
     package="mozilla.components.feature.downloads">
 
     <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
-</manifest>
\ No newline at end of file
+    <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
+</manifest>

components/feature/downloads/src/main/java/mozilla/components/feature/downloads/AbstractFetchDownloadService.kt

+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package mozilla.components.feature.downloads
+
+import android.annotation.TargetApi
+import android.app.DownloadManager.ACTION_DOWNLOAD_COMPLETE
+import android.app.DownloadManager.EXTRA_DOWNLOAD_ID
+import android.app.Notification
+import android.app.NotificationChannel
+import android.app.NotificationManager
+import android.content.Context
+import android.content.Intent
+import android.net.Uri
+import android.os.Build
+import android.os.Build.VERSION.SDK_INT
+import android.os.Environment
+import android.os.IBinder
+import androidx.core.app.NotificationCompat
+import androidx.core.content.getSystemService
+import androidx.core.net.toUri
+import androidx.localbroadcastmanager.content.LocalBroadcastManager
+import kotlinx.coroutines.Dispatchers.IO
+import kotlinx.coroutines.withContext
+import mozilla.components.browser.session.Download
+import mozilla.components.concept.fetch.Client
+import mozilla.components.concept.fetch.Header
+import mozilla.components.concept.fetch.Headers.Names.CONTENT_DISPOSITION
+import mozilla.components.concept.fetch.Headers.Names.CONTENT_LENGTH
+import mozilla.components.concept.fetch.Headers.Names.CONTENT_TYPE
+import mozilla.components.concept.fetch.Headers.Names.REFERRER
+import mozilla.components.concept.fetch.MutableHeaders
+import mozilla.components.concept.fetch.Request
+import mozilla.components.concept.fetch.Response
+import mozilla.components.feature.downloads.ext.getDownloadExtra
+import mozilla.components.feature.downloads.manager.SystemDownloadManager
+import mozilla.components.feature.downloads.manager.getFileName
+import mozilla.components.support.base.ids.NotificationIds
+import java.io.File
+import java.io.FileOutputStream
+import java.io.OutputStream
+
+/**
+ * Service that performs downloads through a fetch [Client] rather than through the native
+ * Android download manager.
+ *
+ * To use this service, you must create a subclass in your application and it to the manifest.
+ *
+ * @param broadcastManager Override the [LocalBroadcastManager] instance.
+ */
+abstract class AbstractFetchDownloadService(
+    broadcastManager: LocalBroadcastManager? = null
+) : CoroutineService() {
+
+    protected abstract val httpClient: Client
+    private val broadcastManager = broadcastManager ?: LocalBroadcastManager.getInstance(this)
+
+    override fun onCreate() {
+        startForeground(
+            NotificationIds.getIdForTag(this, ONGOING_DOWNLOAD_NOTIFICATION_TAG),
+            buildNotification()
+        )
+        super.onCreate()
+    }
+
+    override fun onBind(intent: Intent?): IBinder? = null
+
+    override suspend fun onStartCommand(intent: Intent?, flags: Int) {
+        val download = intent?.getDownloadExtra() ?: return
+        performDownload(download)
+
+        val downloadID = intent.getLongExtra(EXTRA_DOWNLOAD_ID, -1)
+        sendDownloadCompleteBroadcast(downloadID)
+    }
+
+    private suspend fun performDownload(download: Download) = withContext(IO) {
+        val headers = listOf(
+            CONTENT_TYPE to download.contentType,
+            CONTENT_LENGTH to download.contentLength?.toString(),
+            REFERRER to download.referrerUrl
+        ).mapNotNull { (name, value) ->
+            if (value.isNullOrBlank()) null else Header(name, value)
+        }
+
+        val request = Request(download.url, headers = MutableHeaders(headers))
+
+        val response = httpClient.fetch(request)
+        val filename = download.getFileName(response.headers[CONTENT_DISPOSITION])
+
+        response.body.useStream { inStream ->
+            useFileStream(download, response, filename) { outStream ->
+                inStream.copyTo(outStream)
+            }
+        }
+    }
+
+    /**
+     * Informs [mozilla.components.feature.downloads.manager.FetchDownloadManager] that a download
+     * has been completed.
+     */
+    private fun sendDownloadCompleteBroadcast(downloadID: Long) {
+        val intent = Intent(ACTION_DOWNLOAD_COMPLETE)
+        intent.putExtra(EXTRA_DOWNLOAD_ID, downloadID)
+        broadcastManager.sendBroadcast(intent)
+    }
+
+    /**
+     * Creates an output stream on the local filesystem, then informs the system that a download
+     * is complete after [block] is run.
+     *
+     * Encapsulates different behaviour depending on the SDK version.
+     */
+    @Suppress("LongMethod")
+    internal fun useFileStream(
+        download: Download,
+        response: Response,
+        filename: String,
+        block: (OutputStream) -> Unit
+    ) {
+        /*if (SDK_INT >= Build.VERSION_CODES.Q) {
+            val values = ContentValues().apply {
+                put(MediaStore.Downloads.DISPLAY_NAME, filename)
+                put(MediaStore.Downloads.MIME_TYPE, download.contentType)
+                put(MediaStore.Downloads.SIZE, download.contentLength)
+                put(MediaStore.Downloads.IS_PENDING, 1)
+            }
+
+            val resolver = applicationContext.contentResolver
+            val collection = MediaStore.Downloads.getContentUri(MediaStore.VOLUME_EXTERNAL_PRIMARY)
+            val item = resolver.insert(collection, values)
+
+            val pfd = resolver.openFileDescriptor(item!!, "w")
+            ParcelFileDescriptor.AutoCloseOutputStream(pfd).use(block)
+
+            values.clear()
+            values.put(MediaStore.Downloads.IS_PENDING, 0)
+            resolver.update(item, values, null, null)
+        } else {*/
+        val dir = Environment.getExternalStoragePublicDirectory(download.destinationDirectory)
+        val file = File(dir, filename)
+        FileOutputStream(file).use(block)
+
+        val contentType = response.headers[CONTENT_TYPE]
+        val contentLength = response.headers[CONTENT_LENGTH]?.toLongOrNull()
+
+        addCompletedDownload(
+            title = filename,
+            description = filename,
+            isMediaScannerScannable = true,
+            mimeType = contentType ?: download.contentType ?: "*/*",
+            path = file.absolutePath,
+            length = contentLength ?: download.contentLength ?: file.length(),
+            showNotification = true,
+            uri = download.url.toUri(),
+            referer = download.referrerUrl?.toUri()
+        )
+        // }
+    }
+
+    /**
+     * Wraps around [android.app.DownloadManager.addCompletedDownload] and calls the correct
+     * method depending on the SDK version.
+     *
+     * Deprecated in Android Q, use MediaStore on that version.
+     */
+    @TargetApi(Build.VERSION_CODES.P)
+    @Suppress("Deprecation", "LongParameterList", "LongMethod")
+    private fun addCompletedDownload(
+        title: String,
+        description: String,
+        isMediaScannerScannable: Boolean,
+        mimeType: String,
+        path: String,
+        length: Long,
+        showNotification: Boolean,
+        uri: Uri,
+        referer: Uri?
+    ) = getSystemService<SystemDownloadManager>()!!.run {
+        if (SDK_INT >= Build.VERSION_CODES.N) {
+            addCompletedDownload(
+                title,
+                description,
+                isMediaScannerScannable,
+                mimeType,
+                path,
+                length,
+                showNotification,
+                uri,
+                referer
+            )
+        } else {
+            addCompletedDownload(
+                title,
+                description,
+                isMediaScannerScannable,
+                mimeType,
+                path,
+                length,
+                showNotification
+            )
+        }
+    }
+
+    /**
+     * Build the notification to be displayed while the service is active.
+     */
+    private fun buildNotification(): Notification {
+        val channelId = ensureChannelExists(this)
+
+        return NotificationCompat.Builder(this, channelId)
+            .setSmallIcon(android.R.drawable.stat_sys_download)
+            .setContentTitle(getString(R.string.mozac_feature_downloads_ongoing_notification_title))
+            .setContentText(getString(R.string.mozac_feature_downloads_ongoing_notification_text))
+            .setCategory(NotificationCompat.CATEGORY_PROGRESS)
+            .setProgress(1, 0, true)
+            .setOngoing(true)
+            .build()
+    }
+
+    /**
+     * Make sure a notification channel for download notification exists.
+     *
+     * Returns the channel id to be used for download notifications.
+     */
+    private fun ensureChannelExists(context: Context): String {
+        if (SDK_INT >= Build.VERSION_CODES.O) {
+            val notificationManager: NotificationManager = context.getSystemService()!!
+
+            val channel = NotificationChannel(
+                NOTIFICATION_CHANNEL_ID,
+                context.getString(R.string.mozac_feature_downloads_notification_channel),
+                NotificationManager.IMPORTANCE_DEFAULT
+            )
+
+            notificationManager.createNotificationChannel(channel)
+        }
+
+        return NOTIFICATION_CHANNEL_ID
+    }
+
+    companion object {
+        private const val NOTIFICATION_CHANNEL_ID = "Downloads"
+        private const val ONGOING_DOWNLOAD_NOTIFICATION_TAG = "OngoingDownload"
+    }
+}

components/feature/downloads/src/main/java/mozilla/components/feature/downloads/CoroutineService.kt

+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package mozilla.components.feature.downloads
+
+import android.app.Service
+import android.content.Intent
+import androidx.annotation.CallSuper
+import androidx.annotation.VisibleForTesting
+import androidx.annotation.VisibleForTesting.PROTECTED
+import kotlinx.coroutines.CoroutineDispatcher
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.cancel
+import kotlinx.coroutines.launch
+
+/**
+ * Service that runs suspend functions in parallel.
+ * When all jobs are completed, the service is stopped automatically.
+ */
+abstract class CoroutineService(
+    jobDispatcher: CoroutineDispatcher = Dispatchers.IO
+) : Service() {
+
+    private val scope = CoroutineScope(jobDispatcher)
+    private val runningJobs = mutableSetOf<Job>()
+
+    /**
+     * Called by every time a client explicitly starts the service by calling
+     * [android.content.Context.startService], providing the arguments it supplied.
+     * Do not call this method directly.
+     *
+     * @param intent The Intent supplied to [android.content.Context.startService], as given.
+     * This may be null if the service is being restarted after its process has gone away.
+     * @param flags Additional data about this start request.
+     */
+    @VisibleForTesting(otherwise = PROTECTED)
+    internal abstract suspend fun onStartCommand(intent: Intent?, flags: Int)
+
+    /**
+     * Starts a job using [onStartCommand] then stops the service once all jobs are complete.
+     */
+    final override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
+        val job = scope.launch { onStartCommand(intent, flags) }
+        synchronized(runningJobs) {
+            runningJobs.add(job)
+        }
+        job.invokeOnCompletion { cleanupJob(job) }
+        return START_REDELIVER_INTENT
+    }
+
+    /**
+     * Stops all jobs when the service is destroyed.
+     */
+    @CallSuper
+    override fun onDestroy() {
+        scope.cancel()
+    }
+
+    private fun cleanupJob(job: Job) = synchronized(runningJobs) {
+        runningJobs.remove(job)
+        if (runningJobs.isEmpty()) {
+            stopSelf()
+        }
+    }
+}

components/feature/downloads/src/main/java/mozilla/components/feature/downloads/DownloadDialogFragment.kt

 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
 package mozilla.components.feature.downloads
 
 import android.os.Bundle

components/feature/downloads/src/main/java/mozilla/components/feature/downloads/DownloadsFeature.kt

@@ -4,10 +4,9 @@
 
 package mozilla.components.feature.downloads
 
-import android.Manifest.permission.INTERNET
-import android.Manifest.permission.WRITE_EXTERNAL_STORAGE
 import android.annotation.SuppressLint
 import android.content.Context
+import android.widget.Toast
 import androidx.annotation.VisibleForTesting
 import androidx.annotation.VisibleForTesting.PRIVATE
 import androidx.fragment.app.FragmentManager
@@ -19,10 +18,12 @@ import mozilla.components.feature.downloads.DownloadDialogFragment.Companion.FRA
 import mozilla.components.feature.downloads.manager.AndroidDownloadManager
 import mozilla.components.feature.downloads.manager.DownloadManager
 import mozilla.components.feature.downloads.manager.OnDownloadCompleted
+import mozilla.components.feature.downloads.manager.noop
 import mozilla.components.support.base.feature.LifecycleAwareFeature
 import mozilla.components.support.base.feature.OnNeedToRequestPermissions
 import mozilla.components.support.base.feature.PermissionsFeature
 import mozilla.components.support.base.observer.Consumable
+import mozilla.components.support.ktx.android.content.appName
 import mozilla.components.support.ktx.android.content.isPermissionGranted
 
 /**
@@ -46,8 +47,8 @@ import mozilla.components.support.ktx.android.content.isPermissionGranted
 class DownloadsFeature(
     private val applicationContext: Context,
     override var onNeedToRequestPermissions: OnNeedToRequestPermissions = { },
-    var onDownloadCompleted: OnDownloadCompleted = { _, _ -> },
-    private val downloadManager: DownloadManager = AndroidDownloadManager(applicationContext, onDownloadCompleted),
+    onDownloadCompleted: OnDownloadCompleted = noop,
+    private val downloadManager: DownloadManager = AndroidDownloadManager(applicationContext),
     sessionManager: SessionManager,
     private val sessionId: String? = null,
     private val fragmentManager: FragmentManager? = null,
@@ -55,6 +56,14 @@ class DownloadsFeature(
     internal var dialog: DownloadDialogFragment = SimpleDownloadDialogFragment.newInstance()
 ) : SelectionAwareSessionObserver(sessionManager), LifecycleAwareFeature, PermissionsFeature {
 
+    var onDownloadCompleted: OnDownloadCompleted
+        get() = downloadManager.onDownloadCompleted
+        set(value) { downloadManager.onDownloadCompleted = value }
+
+    init {
+        this.onDownloadCompleted = onDownloadCompleted
+    }
+
     /**
      * Starts observing downloads on the selected session and sends them to the [DownloadManager]
      * to be processed.
@@ -72,7 +81,7 @@ class DownloadsFeature(
      */
     override fun stop() {
         super.stop()
-        downloadManager.unregisterListener()
+        downloadManager.unregisterListeners()
     }
 
     /**
@@ -80,16 +89,25 @@ class DownloadsFeature(
      */
     @SuppressLint("MissingPermission")
     override fun onDownload(session: Session, download: Download): Boolean {
-        return if (applicationContext.isPermissionGranted(INTERNET, WRITE_EXTERNAL_STORAGE)) {
+        return if (applicationContext.isPermissionGranted(downloadManager.permissions.asIterable())) {
             if (fragmentManager != null) {
                 showDialog(download, session)
                 false
             } else {
-                downloadManager.download(download)
-                true
+                startDownload(download)
             }
         } else {
-            onNeedToRequestPermissions(arrayOf(INTERNET, WRITE_EXTERNAL_STORAGE))
+            onNeedToRequestPermissions(downloadManager.permissions)
+            false
+        }
+    }
+
+    private fun startDownload(download: Download): Boolean {
+        val id = downloadManager.download(download)
+        return if (id != null) {
+            true
+        } else {
+            showUnSupportFileErrorMessage()
             false
         }
     }
@@ -99,24 +117,30 @@ class DownloadsFeature(
      * either trigger or clear the pending download.
      */
     override fun onPermissionsResult(permissions: Array<String>, grantResults: IntArray) {
-        if (applicationContext.isPermissionGranted(INTERNET, WRITE_EXTERNAL_STORAGE)) {
+        if (applicationContext.isPermissionGranted(downloadManager.permissions.asIterable())) {
             activeSession?.let { session ->
-                session.download.consume {
-                    onDownload(session, it)
-                }
+                session.download.consume { onDownload(session, it) }
             }
         } else {
             activeSession?.download = Consumable.empty()
         }
     }
 
+    private fun showUnSupportFileErrorMessage() {
+        val text = applicationContext.getString(
+            R.string.mozac_feature_downloads_file_not_supported2,
+            applicationContext.appName
+        )
+
+        Toast.makeText(applicationContext, text, Toast.LENGTH_LONG).show()
+    }
+
     @SuppressLint("MissingPermission")
     private fun showDialog(download: Download, session: Session) {
         dialog.setDownload(download)
 
         dialog.onStartDownload = {
-            downloadManager.download(download)
-            session.download.consume { true }
+            session.download.consume(this::startDownload)
         }
 
         if (!isAlreadyADialogCreated()) {
@@ -129,13 +153,10 @@ class DownloadsFeature(
     }
 
     private fun reAttachOnStartDownloadListener(previousDialog: DownloadDialogFragment?) {
-        previousDialog?.apply {
-            this@DownloadsFeature.dialog = this
+        previousDialog?.let {
+            dialog = it
             activeSession?.let { session ->
-                session.download.consume {
-                    onDownload(session, it)
-                    false
-                }
+                session.download.consume { download -> onDownload(session, download) }
             }
         }
     }

components/feature/downloads/src/main/java/mozilla/components/feature/downloads/ext/Intent.kt

+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package mozilla.components.feature.downloads.ext
+
+import android.content.Intent
+import androidx.core.os.bundleOf