1516527: Add labeled metrics

components/service/glean/build.gradle

@@ -14,7 +14,7 @@ apply plugin: 'kotlin-android'
  * created during unit testing.
  * This uses a specific version of the schema identified by a git commit hash.
  */
-String GLEAN_PING_SCHEMA_GIT_HASH = "8939bcb"
+String GLEAN_PING_SCHEMA_GIT_HASH = "04e203c"
 String GLEAN_PING_SCHEMA_URL = "https://raw.githubusercontent.com/mozilla-services/mozilla-pipeline-schemas/$GLEAN_PING_SCHEMA_GIT_HASH/schemas/glean/baseline/baseline.1.schema.json"
 
 android {

components/service/glean/src/main/java/mozilla/components/service/glean/Glean.kt

@@ -24,6 +24,8 @@ import mozilla.components.service.glean.storages.ExperimentsStorageEngine
 import mozilla.components.service.glean.storages.StorageEngineManager
 import mozilla.components.service.glean.ping.BaselinePing
 import mozilla.components.service.glean.scheduler.MetricsPingScheduler
+import mozilla.components.service.glean.storages.StringsStorageEngine
+import mozilla.components.service.glean.storages.UuidsStorageEngine
 import mozilla.components.support.base.log.logger.Logger
 import java.io.File
 
@@ -189,6 +191,13 @@ open class GleanInternalAPI internal constructor () {
      * Initialize the core metrics internally managed by Glean (e.g. client id).
      */
     private fun initializeCoreMetrics(applicationContext: Context) {
+        // Since all of the ping_info properties are required, we can't
+        // use the normal metrics API to set them, since those work
+        // asynchronously, and there is a race condition between when they
+        // are set and the possible sending of the first ping upon startup.
+        // Therefore, this uses the lower-level internal storage engine API
+        // to set these metrics, which is synchronous.
+
         val gleanDataDir = File(applicationContext.applicationInfo.dataDir, Glean.GLEAN_DATA_DIR)
 
         // Make sure the data directory exists and is writable.
@@ -206,17 +215,22 @@ open class GleanInternalAPI internal constructor () {
         // one-time only metrics.
         val firstRunDetector = FileFirstRunDetector(gleanDataDir)
         if (firstRunDetector.isFirstRun()) {
-            GleanInternalMetrics.clientId.generateAndSet()
+            val uuid = UUID.randomUUID()
+            UuidsStorageEngine.record(GleanInternalMetrics.clientId, uuid)
         }
 
         try {
             val packageInfo = applicationContext.packageManager.getPackageInfo(
                 applicationContext.packageName, 0
             )
-            GleanInternalMetrics.appBuild.set(packageInfo.versionCode.toString())
-            packageInfo.versionName?.let {
-                GleanInternalMetrics.appDisplayVersion.set(it)
-            }
+            StringsStorageEngine.record(
+                GleanInternalMetrics.appBuild,
+                packageInfo.versionCode.toString()
+            )
+            StringsStorageEngine.record(
+                GleanInternalMetrics.appDisplayVersion,
+                packageInfo.versionName?.let { it } ?: "Unknown"
+            )
         } catch (e: PackageManager.NameNotFoundException) {
             logger.error("Could not get own package info, unable to report build id and display version")
             throw AssertionError("Could not get own package info, aborting init")

components/service/glean/src/main/java/mozilla/components/service/glean/LabeledMetricType.kt

+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package mozilla.components.service.glean
+
+import mozilla.components.support.base.log.logger.Logger
+
+/**
+ * This implements the developer facing API for labeled metrics.
+ *
+ * Instances of this class type are automatically generated by the parsers at build time,
+ * allowing developers to record values that were previously registered in the metrics.yaml file.
+ *
+ * Unlike most metric types, LabeledMetricType does not have its own corresponding storage engine,
+ * but records metrics for the underlying metric type T in the storage engine for that type.  The
+ * only difference is that labeled metrics are stored with the special key `$category.$name/$label`.
+ * The |StorageEngineManager.collect| method knows how to pull these special values back out of the
+ * individual storage engines and rearrange them correctly in the ping.
+ */
+
+data class LabeledMetricType<T>(
+    override val disabled: Boolean,
+    override val category: String,
+    override val lifetime: Lifetime,
+    override val name: String,
+    override val sendInPings: List<String>,
+    val subMetric: T,
+    val labels: Set<String>? = null
+) : CommonMetricData {
+
+    override val defaultStorageDestinations: List<String> = (
+        subMetric as CommonMetricData).defaultStorageDestinations
+
+    private val logger = Logger("glean/LabeledMetricType")
+
+    companion object {
+        private const val MAX_LABELS = 16
+        private const val OTHER_LABEL = "__other__"
+        private val labelRegex = Regex("^[a-z_][a-z0-9_]{0,29}$")
+    }
+
+    private val seenLabels: MutableSet<String> = mutableSetOf()
+
+    /**
+     * Handles the label in the case where labels are predefined.
+     *
+     * If the given label is not in the predefined set of labels, returns [OTHER_LABEL], otherwise
+     * returns the label verbatim.
+     *
+     * @param label The label, as specified by the user
+     * @return adjusted label, possibly set to [OTHER_LABEL]
+     */
+    private fun getFinalStaticLabel(label: String): String {
+        return if (labels!!.contains(label)) label else OTHER_LABEL
+    }
+
+    /**
+     * Handles the label in the case where labels aren't predefined.
+     *
+     * If we've already seen more than [MAX_LABELS] unique labels, returns [OTHER_LABEL].
+     *
+     * Also validates any unseen labels to make sure they are snake_case and under 30 characters.
+     * If not, returns [OTHER_LABEL].
+     *
+     * @param label The label, as specified by the user
+     * @return adjusted label, possibly set to [OTHER_LABEL]
+     */
+    private fun getFinalDynamicLabel(label: String): String {
+        if (!seenLabels.contains(label)) {
+            if (seenLabels.size >= MAX_LABELS) {
+                return OTHER_LABEL
+            } else {
+                // Labels must be snake_case.
+                if (!labelRegex.matches(label)) {
+                    logger.error("Labels must be snake_case and < 30 characters. Got '$label'")
+                    return OTHER_LABEL
+                }
+                seenLabels.add(label)
+            }
+        }
+        return label
+    }
+
+    /**
+     * Get a copy of the subMetric with the name changed to the given `newName`.
+     *
+     * @param newName The new name for the metric.
+     * @return A copy of subMetric with the new name.
+     * @throws IllegalStateException If this metric type does not support labels.
+     */
+    internal fun getMetricWithNewName(newName: String): T {
+        // function is "internal" so we can mock it in testing
+
+        // Every metric that supports labels needs an entry here
+        return when (subMetric) {
+            is BooleanMetricType -> subMetric.copy(name = newName) as T
+            is CounterMetricType -> subMetric.copy(name = newName) as T
+            is StringListMetricType -> subMetric.copy(name = newName) as T
+            is StringMetricType -> subMetric.copy(name = newName) as T
+            is TimespanMetricType -> subMetric.copy(name = newName) as T
+            is UuidMetricType -> subMetric.copy(name = newName) as T
+            else -> throw IllegalStateException(
+                "Can not create a labeled version of this metric type"
+            )
+        }
+    }
+
+    /**
+     * Get the specific metric for a given label.
+     *
+     * If a set of acceptable labels were specified in the metrics.yaml file,
+     * and the given label is not in the set, it will be recorded under the
+     * special [OTHER_LABEL].
+     *
+     * If a set of acceptable labels was not specified in the metrics.yaml file,
+     * only the first 16 unique labels will be used. After that, any additional
+     * labels will be recorded under the special [OTHER_LABEL] label.
+     *
+     * Labels must be snake_case and less than 30 characters. If an invalid label
+     * is used, the metric will be recorded in the special [OTHER_LABEL] label.
+     *
+     * @param label The label
+     * @return The specific metric for that label
+     */
+    operator fun get(label: String): T {
+        val actualLabel = labels?.let {
+            getFinalStaticLabel(label)
+        } ?: run {
+            getFinalDynamicLabel(label)
+        }
+
+        return getMetricWithNewName("$name/$actualLabel")
+    }
+}

components/service/glean/src/main/java/mozilla/components/service/glean/storages/StorageEngineManager.kt

@@ -6,7 +6,9 @@ package mozilla.components.service.glean.storages
 
 import android.content.Context
 import android.support.annotation.VisibleForTesting
+import org.json.JSONArray
 import org.json.JSONObject
+import mozilla.components.support.ktx.android.org.json.getOrPutJSONObject
 
 /**
  * This singleton is the one interface to all the available storage engines:
@@ -31,6 +33,56 @@ internal class StorageEngineManager(
         }
     }
 
+    /**
+     * Splits a labeled metric back into its name/label parts.
+     *
+     * If not a labeled metric, the second part of the Pair will be null.
+     *
+     * @param key The key for the metric value in the flattened storage engine
+     * @return A pair (metricName, label).  label is null if key is not
+     * from a labeled metric.
+     */
+    private fun parseLabeledMetric(key: String): Pair<String, String?> {
+        val divider = key.indexOf('/', 1)
+        if (divider >= 0) {
+            return Pair(
+                key.substring(0, divider),
+                key.substring(divider + 1)
+            )
+        } else {
+            return Pair(key, null)
+        }
+    }
+
+    /**
+     * Reorganizes the flat storage of metrics into labeled and unlabeled categories as
+     * they appear in the ping.
+     *
+     * The unlabeled metrics go under the `sectionName` key, and the labeled metrics go
+     * under the `labeled_$sectionName` key.
+     *
+     * @param sectionName The name of the metric section (the name of the metric type)
+     * @param dst The destination JSONObject in the ping
+     * @param engineData The flat object of metrics from the storage engine
+     */
+    private fun separateLabeledAndUnlabeledMetrics(
+        sectionName: String,
+        dst: JSONObject,
+        engineData: JSONObject
+    ) {
+        for (key in engineData.keys()) {
+            val parts = parseLabeledMetric(key)
+            parts.second?.let {
+                val labeledSection = dst.getOrPutJSONObject("labeled_$sectionName") { JSONObject() }
+                val labeledMetric = labeledSection.getOrPutJSONObject(parts.first) { JSONObject() }
+                labeledMetric.put(it, engineData.get(key))
+            } ?: run {
+                val section = dst.getOrPutJSONObject(sectionName) { JSONObject() }
+                section.put(key, engineData.get(key))
+            }
+        }
+    }
+
     /**
      * Collect the recorded data for the requested storage.
      *
@@ -44,10 +96,15 @@ internal class StorageEngineManager(
         val metricsSection = JSONObject()
         for ((sectionName, engine) in storageEngines) {
             val engineData = engine.getSnapshotAsJSON(storeName, clearStore = true)
-            if (engine.sendAsTopLevelField) {
-                jsonPing.put(sectionName, engineData)
-            } else {
-                metricsSection.put(sectionName, engineData)
+            val dst = if (engine.sendAsTopLevelField) jsonPing else metricsSection
+            // Most storage engines return a JSONObject mapping metric names
+            // to metric values, and these can include labeled metrics that we
+            // need to separate out.  The EventsStorageEngine just returns an
+            // array of events, which are never "labeled".
+            if (engineData is JSONObject) {
+                separateLabeledAndUnlabeledMetrics(sectionName, dst, engineData)
+            } else if (engineData is JSONArray) {
+                dst.put(sectionName, engineData)
             }
         }
         if (metricsSection.length() != 0) {

components/service/glean/src/test/java/mozilla/components/service/glean/LabeledMetricTypeTest.kt

+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package mozilla.components.service.glean
+
+import android.content.SharedPreferences
+import mozilla.components.service.glean.storages.BooleansStorageEngine
+import mozilla.components.service.glean.storages.CountersStorageEngine
+import mozilla.components.service.glean.storages.GenericScalarStorageEngine
+import mozilla.components.service.glean.storages.StringListsStorageEngine
+import mozilla.components.service.glean.storages.StringsStorageEngine
+import mozilla.components.service.glean.storages.TimespansStorageEngine
+import mozilla.components.service.glean.storages.UuidsStorageEngine
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertNull
+import java.util.UUID
+import mozilla.components.support.base.log.logger.Logger
+import org.robolectric.RobolectricTestRunner
+
+@RunWith(RobolectricTestRunner::class)
+class LabeledMetricTypeTest {
+    private class MockScalarStorageEngine(
+        override val logger: Logger = Logger("test")
+    ) : GenericScalarStorageEngine<Int>() {
+        override fun deserializeSingleMetric(metricName: String, value: Any?): Int? {
+            if (value is String) {
+                return value.toIntOrNull()
+            }
+
+            return value as? Int?
+        }
+
+        override fun serializeSingleMetric(
+            userPreferences: SharedPreferences.Editor?,
+            storeName: String,
+            value: Int,
+            extraSerializationData: Any?
+        ) {
+            userPreferences?.putInt(storeName, value)
+        }
+
+        fun record(
+            metricData: CommonMetricData,
+            value: Int
+        ) {
+            super.recordScalar(metricData, value)
+        }
+    }
+
+    private data class GenericMetricType(
+        override val disabled: Boolean,
+        override val category: String,
+        override val lifetime: Lifetime,
+        override val name: String,
+        override val sendInPings: List<String>
+    ) : CommonMetricData {
+        override val defaultStorageDestinations: List<String> = listOf("metrics")
+    }
+
+    @Before
+    fun setup() {
+        resetGlean()
+    }
+
+    @After
+    fun resetGlobalState() {
+        Glean.setUploadEnabled(true)
+    }
+
+    @Test
+    fun `test labeled counter type`() {
+        CountersStorageEngine.clearAllStores()
+
+        val counterMetric = CounterMetricType(
+            disabled = false,
+            category = "telemetry",
+            lifetime = Lifetime.Application,
+            name = "labeled_counter_metric",
+            sendInPings = listOf("default")
+        )
+
+        val labeledCounterMetric = LabeledMetricType<CounterMetricType>(
+            disabled = false,
+            category = "telemetry",
+            lifetime = Lifetime.Application,
+            name = "labeled_counter_metric",
+            sendInPings = listOf("default"),
+            subMetric = counterMetric
+        )
+
+        CountersStorageEngine.record(labeledCounterMetric["label1"], 1)
+        CountersStorageEngine.record(labeledCounterMetric["label2"], 2)
+
+        // Record a regular non-labeled counter. This isn't normally
+        // possible with the generated code because the subMetric is private,
+        // but it's useful to test here that it works.
+        CountersStorageEngine.record(counterMetric, 3)
+
+        val snapshot = CountersStorageEngine.getSnapshot(storeName = "metrics", clearStore = false)
+
+        assertEquals(3, snapshot!!.size)
+        assertEquals(1, snapshot.get("telemetry.labeled_counter_metric/label1"))
+        assertEquals(2, snapshot.get("telemetry.labeled_counter_metric/label2"))
+        assertEquals(3, snapshot.get("telemetry.labeled_counter_metric"))
+
+        val json = collectAndCheckPingSchema("metrics").getJSONObject("metrics")!!
+        // Do the same checks again on the JSON structure
+        assertEquals(
+            1,
+            json.getJSONObject("labeled_counter")!!
+                .getJSONObject("telemetry.labeled_counter_metric")!!
+                .get("label1")
+        )
+        assertEquals(
+            2,
+            json.getJSONObject("labeled_counter")!!
+                .getJSONObject("telemetry.labeled_counter_metric")!!
+                .get("label2")
+        )
+        assertEquals(
+            3,
+            json.getJSONObject("counter")!!
+                .get("telemetry.labeled_counter_metric")
+        )
+    }
+
+    @Test
+    fun `test __other__ label with predefined labels`() {
+        CountersStorageEngine.clearAllStores()
+
+        val counterMetric = CounterMetricType(
+            disabled = false,
+            category = "telemetry",
+            lifetime = Lifetime.Application,
+            name = "labeled_counter_metric",
+            sendInPings = listOf("default")
+        )
+
+        val labeledCounterMetric = LabeledMetricType<CounterMetricType>(
+            disabled = false,
+            category = "telemetry",
+            lifetime = Lifetime.Application,
+            name = "labeled_counter_metric",
+            sendInPings = listOf("default"),
+            subMetric = counterMetric,
+            labels = setOf("foo", "bar", "baz")
+        )
+
+        CountersStorageEngine.record(labeledCounterMetric["foo"], 1)
+        CountersStorageEngine.record(labeledCounterMetric["foo"], 1)
+        CountersStorageEngine.record(labeledCounterMetric["bar"], 1)
+        CountersStorageEngine.record(labeledCounterMetric["not_there"], 1)
+        CountersStorageEngine.record(labeledCounterMetric["also_not_there"], 1)
+        CountersStorageEngine.record(labeledCounterMetric["not_me"], 1)
+
+        val snapshot = CountersStorageEngine.getSnapshot(storeName = "metrics", clearStore = false)
+
+        assertEquals(3, snapshot!!.size)
+        assertEquals(2, snapshot.get("telemetry.labeled_counter_metric/foo"))
+        assertEquals(1, snapshot.get("telemetry.labeled_counter_metric/bar"))
+        assertNull(snapshot.get("telemetry.labeled_counter_metric/baz"))
+        assertEquals(3, snapshot.get("telemetry.labeled_counter_metric/__other__"))
+
+        val json = collectAndCheckPingSchema("metrics").getJSONObject("metrics")!!
+        // Do the same checks again on the JSON structure
+        assertEquals(
+            2,
+            json.getJSONObject("labeled_counter")!!
+                .getJSONObject("telemetry.labeled_counter_metric")
+                .get("foo")
+        )
+        assertEquals(
+            1,
+            json.getJSONObject("labeled_counter")!!
+                .getJSONObject("telemetry.labeled_counter_metric")
+                .get("bar")
+        )
+        assertEquals(
+            3,
+            json.getJSONObject("labeled_counter")!!
+                .getJSONObject("telemetry.labeled_counter_metric")
+                .get("__other__")
+        )
+    }
+
+    @Test
+    fun `test __other__ label without predefined labels`() {
+        CountersStorageEngine.clearAllStores()
+
+        val counterMetric = CounterMetricType(
+            disabled = false,
+            category = "telemetry",
+            lifetime = Lifetime.Application,
+            name = "labeled_counter_metric",
+            sendInPings = listOf("default")
+        )
+
+        val labeledCounterMetric = LabeledMetricType<CounterMetricType>(
+            disabled = false,
+            category = "telemetry",
+            lifetime = Lifetime.Application,
+            name = "labeled_counter_metric",
+            sendInPings = listOf("default"),
+            subMetric = counterMetric
+        )
+
+        for (i in 0..20) {
+            CountersStorageEngine.record(labeledCounterMetric["label_$i"], 1)
+        }
+        // Go back and record in one of the real labels again
+        CountersStorageEngine.record(labeledCounterMetric["label_0"], 1)
+
+        val snapshot = CountersStorageEngine.getSnapshot(storeName = "metrics", clearStore = false)
+
+        assertEquals(17, snapshot!!.size)
+        assertEquals(2, snapshot.get("telemetry.labeled_counter_metric/label_0"))
+        for (i in 1..15) {
+            assertEquals(1, snapshot.get("telemetry.labeled_counter_metric/label_$i"))
+        }
+        assertEquals(5, snapshot.get("telemetry.labeled_counter_metric/__other__"))
+
+        val json = collectAndCheckPingSchema("metrics").getJSONObject("metrics")!!
+        // Do the same checks again on the JSON structure
+        assertEquals(
+            2,
+            json.getJSONObject("labeled_counter")!!
+                .getJSONObject("telemetry.labeled_counter_metric")!!
+                .get("label_0")
+        )
+        for (i in 1..15) {
+            assertEquals(
+                1,
+                json.getJSONObject("labeled_counter")!!
+                    .getJSONObject("telemetry.labeled_counter_metric")!!
+                    .get("label_$i")
+            )
+        }
+        assertEquals(
+            5,
+            json.getJSONObject("labeled_counter")!!
+                .getJSONObject("telemetry.labeled_counter_metric")!!
+                .get("__other__")
+        )
+    }
+
+    @Test
+    fun `Ensure non-snake_case labels go to __other__`() {
+        CountersStorageEngine.clearAllStores()
+
+        val counterMetric = CounterMetricType(
+            disabled = false,
+            category = "telemetry",
+            lifetime = Lifetime.Application,
+            name = "labeled_counter_metric",
+            sendInPings = listOf("default")
+        )
+
+        val labeledCounterMetric = LabeledMetricType<CounterMetricType>(
+            disabled = false,
+            category = "telemetry",
+            lifetime = Lifetime.Application,
+            name = "labeled_counter_metric",
+            sendInPings = listOf("default"),
+            subMetric = counterMetric
+        )
+
+        CountersStorageEngine.record(labeledCounterMetric["notSnakeCase"], 1)
+        CountersStorageEngine.record(labeledCounterMetric[""], 1)
+        CountersStorageEngine.record(labeledCounterMetric["with/slash"], 1)
+        CountersStorageEngine.record(
+            labeledCounterMetric["this_string_has_more_than_thirty_characters"],
+            1
+        )
+
+        val snapshot = CountersStorageEngine.getSnapshot(storeName = "metrics", clearStore = false)
+
+