Commit 6068bd7d authored by travis79's avatar travis79
Browse files

Experiments: add matcher for min/max versions

- Add experiments matcher functionality for min and max versions
- Add VersionString utility class to aid in comparing versions
- Update tests to add coverage for min/max matchers
parent d5568f27
......@@ -64,8 +64,8 @@ For any technical tests, we do have a Kinto dev server available, which can be f
The admin interface is [here](https://kinto.dev.mozaws.net/v1/admin/). For setting up a testing setup we can:
- [Create a collection in the main bucket](https://kinto.dev.mozaws.net/v1/admin/#/buckets/main/collections/create).
- The *collection id* should be `mobile-experiments`.
- The *JSON schema* should have [this content](https://gist.github.com/georgf/cbf4c145ca0e202ad0d25a5a83e06f38/#file-json-schema-json).
- The *UI schema* should have [this content](https://gist.github.com/georgf/cbf4c145ca0e202ad0d25a5a83e06f38/#file-ui-schema-json).
- The *JSON schema* should have [this content](https://gist.github.com/travis79/c112d803dfcd84cb5f854f5b22bfcd0f#file-json-schema-json).
- The *UI schema* should have [this content](https://gist.github.com/travis79/c112d803dfcd84cb5f854f5b22bfcd0f#file-ui-schema-json).
- The *Records list columns* should contain `id` and `description`.
- Click *Create collection*
- In the [`mobile-experiments` record list](https://kinto.dev.mozaws.net/v1/admin/#/buckets/main/collections/mobile-experiments/records), create new entries for experiments as needed.
......@@ -178,12 +178,14 @@ The experiments records in Kinto contain the following properties:
| branches[i].name | string | x | The name of that branch. | `"control"` or `"red-button"` |
| branches[i].ratio | number | x | The weight to randomly distribute enrolled clients among the branches. | |
| match | object | x | Object containing the filter parameters to match specific user groups. | |
| match.app_id | regex | | The app ID (package name) | "^org.mozilla.firefox_beta${'$'}|^org.mozilla.firefox${'$'}" |
| match.device_manufacturer | regex | | The Android device manufacturer. | |
| match.device_model | regex | | The Android device model. | |
| match.locale_country | | | The default locales country. | "USA|DEU" |
| match.locale_language | | | The default locales language. | "eng|zho|deu" |
| match.app_display_version | | | The application version. | |
| match.app_id | regex | | The app ID (package name) | "^org.mozilla.firefox_beta${'$'}|^org.mozilla.firefox${'$'}" |
| match.device_manufacturer | regex | | The Android device manufacturer. | |
| match.device_model | regex | | The Android device model. | |
| match.locale_country | | | The default locales country. | "USA|DEU" |
| match.locale_language | | | The default locales language. | "eng|zho|deu" |
| match.app_display_version | regex | | The application version. | `"1.0.0"` |
| match.app_min_version | string | | The minimum application version, inclusive. | `"1.0.1"` |
| match.app_max_version | string | | The maximum application version, inclusive. | `"1.4.2"` |
| match.regions | array | | Array of strings. Not currently supported. Custom regions, different from the one from the default locale (like a GeoIP, or something similar). | `["USA", "GBR"]` |
| match.debug_tags | array | | Array of strings. Debug tags to match only specific client for QA of experiments launch & targeting. | `["john-test-1"]` |
......
......@@ -80,6 +80,14 @@ internal data class Experiment(
* App version, as a regex
*/
val appDisplayVersion: String?,
/**
* App minimum version, expected dotted numeric version E.g. 1.0.2, or 67.0.1
*/
val appMinVersion: String?,
/**
* App maximum version, expected dotted numeric version E.g. 1.0.2, or 67.0.1
*/
val appMaxVersion: String?,
/**
* Locale language, as a regex.
*/
......
......@@ -9,6 +9,7 @@ import android.content.Context
import android.os.Looper
import androidx.annotation.VisibleForTesting
import android.text.TextUtils
import mozilla.components.service.experiments.util.VersionString
import mozilla.components.support.base.log.logger.Logger
import java.util.zip.CRC32
......@@ -86,6 +87,7 @@ internal class ExperimentEvaluator(
private fun matches(context: Context, experiment: Experiment): Boolean {
val match = experiment.match
val region = valuesProvider.getRegion(context)
val version = valuesProvider.getVersion(context) ?: ""
val matchesRegion = (region == null) ||
match.regions.isNullOrEmpty() ||
match.regions.any { it == region }
......@@ -93,10 +95,14 @@ internal class ExperimentEvaluator(
val matchesTag = (tag == null) ||
match.debugTags.isNullOrEmpty() ||
match.debugTags.any { it == tag }
val matchesMinVersion = match.appMinVersion.isNullOrEmpty() ||
VersionString(match.appMinVersion) <= VersionString(version)
val matchesMaxVersion = match.appMaxVersion.isNullOrEmpty() ||
VersionString(match.appMaxVersion) >= VersionString(version)
return matchesRegion && matchesTag &&
return matchesRegion && matchesTag && matchesMinVersion && matchesMaxVersion &&
matchesExperiment(match.appId, valuesProvider.getAppId(context)) &&
matchesExperiment(match.appDisplayVersion, valuesProvider.getVersion(context)) &&
matchesExperiment(match.appDisplayVersion, version) &&
matchesExperiment(match.localeLanguage, valuesProvider.getLanguage(context)) &&
matchesExperiment(match.localeCountry, valuesProvider.getCountry(context)) &&
matchesExperiment(match.deviceManufacturer, valuesProvider.getManufacturer(context)) &&
......
......@@ -46,6 +46,8 @@ internal class JSONExperimentParser {
val matcher = Experiment.Matcher(
appId = matchObject.tryGetString(MATCH_APP_ID_KEY),
appDisplayVersion = matchObject.tryGetString(MATCH_APP_DISPLAY_VERSION_KEY),
appMinVersion = matchObject.tryGetString(MATCH_APP_MIN_VERSION_KEY),
appMaxVersion = matchObject.tryGetString(MATCH_APP_MAX_VERSION_KEY),
localeLanguage = matchObject.tryGetString(MATCH_LOCALE_LANGUAGE_KEY),
localeCountry = matchObject.tryGetString(MATCH_LOCALE_COUNTRY_KEY),
deviceManufacturer = matchObject.tryGetString(MATCH_DEVICE_MANUFACTURER_KEY),
......@@ -104,6 +106,8 @@ internal class JSONExperimentParser {
val matchObject = JSONObject()
matchObject.putIfNotNull(MATCH_APP_ID_KEY, experiment.match.appId)
matchObject.putIfNotNull(MATCH_APP_DISPLAY_VERSION_KEY, experiment.match.appDisplayVersion)
matchObject.putIfNotNull(MATCH_APP_MIN_VERSION_KEY, experiment.match.appMinVersion)
matchObject.putIfNotNull(MATCH_APP_MAX_VERSION_KEY, experiment.match.appMaxVersion)
matchObject.putIfNotNull(MATCH_LOCALE_COUNTRY_KEY, experiment.match.localeCountry)
matchObject.putIfNotNull(MATCH_LOCALE_LANGUAGE_KEY, experiment.match.localeLanguage)
matchObject.putIfNotNull(MATCH_DEVICE_MANUFACTURER_KEY, experiment.match.deviceManufacturer)
......@@ -130,6 +134,8 @@ internal class JSONExperimentParser {
private const val MATCH_KEY = "match"
private const val MATCH_APP_ID_KEY = "app_id"
private const val MATCH_APP_DISPLAY_VERSION_KEY = "app_display_version"
private const val MATCH_APP_MIN_VERSION_KEY = "app_min_version"
private const val MATCH_APP_MAX_VERSION_KEY = "app_max_version"
private const val MATCH_LOCALE_COUNTRY_KEY = "locale_country"
private const val MATCH_LOCALE_LANGUAGE_KEY = "locale_language"
private const val MATCH_DEVICE_MANUFACTURER_KEY = "device_manufacturer"
......
package mozilla.components.service.experiments.util
import java.lang.IllegalArgumentException
import kotlin.math.max
internal class VersionString(
val version: String
) : Comparable<VersionString> {
companion object {
// Validates that the version matches expected version formatting
// This matches any number of digits separated by dots.
// Valid examples:
// 1.0.1
// 1.0
// 10.4.123545
// Invalid examples:
// a.b.c
// Not.a.version
// 12.1.1A
val VERSION_REGEX = "[0-9]+(\\.[0-9]+)*".toRegex()
}
@Suppress("ComplexMethod")
override fun compareTo(other: VersionString): Int {
if (!version.matches(VERSION_REGEX)) {
throw IllegalArgumentException("Unexpected format in VersionString")
}
if (!other.version.matches(VERSION_REGEX)) {
throw IllegalArgumentException("Unexpected format in VersionString")
}
val thisParts = version.split(".")
val otherParts = other.version.split(".")
val length = max(thisParts.count(), otherParts.count())
for (i in 0..length) {
// Get the current part, or zero if there isn't a part to compare to for thisPart
val thisPart = if (i < thisParts.count()) {
thisParts[i].toInt()
} else {
0
}
// Get the current part, or zero if there isn't a part to compare to for otherPart
val otherPart = if (i < otherParts.count()) {
otherParts[i].toInt()
} else {
0
}
if (thisPart < otherPart) {
return -1
}
if (thisPart > otherPart) {
return 1
}
}
return 0
}
override fun equals(other: Any?): Boolean {
if (other == null) {
return false
}
if (this.hashCode() == other.hashCode()) {
return true
}
if (this.javaClass != other.javaClass) {
return false
}
return this.compareTo(other as VersionString) == 0
}
override fun hashCode(): Int {
return version.hashCode()
}
}
......@@ -251,6 +251,150 @@ class ExperimentEvaluatorTest {
assertNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
}
@Test
fun `evaluate appMinVersion`() {
testReset(appId = "test.appId", versionName = "1.0.0")
val experiment = createDefaultExperiment(
id = "testexperiment",
match = createDefaultMatcher(
localeLanguage = "eng",
appId = "test.appId",
regions = listOf("USA", "GBR"),
appMinVersion = "1.0.0",
deviceManufacturer = "unknown",
deviceModel = "robolectric",
localeCountry = "USA"
),
buckets = Experiment.Buckets(20, 70),
lastModified = currentTime
)
val evaluator = ExperimentEvaluator()
assertNotNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "1"
assertNotNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "1.0"
assertNotNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "1.0.1"
assertNotNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "1.1.18"
assertNotNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "1.1.18.43123"
assertNotNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "1.1.18.34.2345"
assertNotNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "0.1.18"
assertNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "0.9.9"
assertNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
}
@Test
fun `evaluate appMaxVersion`() {
testReset(appId = "test.appId", versionName = "2.0.0")
val experiment = createDefaultExperiment(
id = "testexperiment",
match = createDefaultMatcher(
localeLanguage = "eng",
appId = "test.appId",
regions = listOf("USA", "GBR"),
appMaxVersion = "2.0.0",
deviceManufacturer = "unknown",
deviceModel = "robolectric",
localeCountry = "USA"
),
buckets = Experiment.Buckets(20, 70),
lastModified = currentTime
)
val evaluator = ExperimentEvaluator()
assertNotNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "1"
assertNotNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "1.9"
assertNotNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "1.9.9999"
assertNotNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "1.1.18"
assertNotNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "2.0.1"
assertNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "2.0.1.2.3"
assertNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "67.1.12345678"
assertNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
}
@Test
fun `evaluate a min and max version together`() {
testReset(appId = "test.appId")
val experiment = createDefaultExperiment(
id = "testexperiment",
match = createDefaultMatcher(
localeLanguage = "eng",
appId = "test.appId",
regions = listOf("USA", "GBR"),
appMinVersion = "1.0.0",
appMaxVersion = "2.0.0",
deviceManufacturer = "unknown",
deviceModel = "robolectric",
localeCountry = "USA"
),
buckets = Experiment.Buckets(20, 70),
lastModified = currentTime
)
val evaluator = ExperimentEvaluator()
packageInfo.versionName = "0.1"
assertNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "0.0.1"
assertNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "0.9.9"
assertNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "0.9.9.9.9.9"
assertNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "1.0.0"
assertNotNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "1.0.1"
assertNotNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "1.9.9999"
assertNotNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "1.9.9999.999.99"
assertNotNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "2.0.1"
assertNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
packageInfo.versionName = "2.1.18"
assertNull(evaluator.evaluate(context, ExperimentDescriptor("testexperiment"), listOf(experiment), 20))
}
@Test
fun `evaluate deviceModel`() {
testReset(appId = "test.appId", versionName = "test.version")
......
......@@ -28,6 +28,8 @@ class JSONExperimentParserTest {
appId = "sample-appId",
regions = listOf("US"),
appDisplayVersion = "1.0",
appMinVersion = "0.1.0",
appMaxVersion = "1.1.0",
deviceManufacturer = "manufacturer",
deviceModel = "device",
localeCountry = "country",
......@@ -67,6 +69,8 @@ class JSONExperimentParserTest {
assertEquals("sample-appId", match.getString("app_id"))
assertEquals("es|en", match.getString("locale_language"))
assertEquals("1.0", match.getString("app_display_version"))
assertEquals("0.1.0", match.getString("app_min_version"))
assertEquals("1.1.0", match.getString("app_max_version"))
assertEquals("manufacturer", match.getString("device_manufacturer"))
assertEquals("device", match.getString("device_model"))
assertEquals("country", match.getString("locale_country"))
......@@ -89,6 +93,8 @@ class JSONExperimentParserTest {
null,
null,
null,
null,
null,
null
)
)
......@@ -131,7 +137,9 @@ class JSONExperimentParserTest {
"US"
],
"app_id": "sample-appId",
"locale_language": "es|en"
"locale_language": "es|en",
"app_min_version": "1.0.0",
"app_max_version": "1.1.0"
},
"last_modified": $currentTime
}
......@@ -143,7 +151,9 @@ class JSONExperimentParserTest {
match = createDefaultMatcher(
localeLanguage = "es|en",
appId = "sample-appId",
regions = listOf("US")
regions = listOf("US"),
appMinVersion = "1.0.0",
appMaxVersion = "1.1.0"
),
buckets = Experiment.Buckets(0, 20),
lastModified = currentTime
......
......@@ -64,6 +64,8 @@ internal fun getWorkInfoByTag(tag: String): WorkInfo? {
internal fun createDefaultMatcher(
appId: String? = null,
appDisplayVersion: String? = null,
appMinVersion: String? = null,
appMaxVersion: String? = null,
localeLanguage: String? = null,
localeCountry: String? = null,
deviceManufacturer: String? = null,
......@@ -74,6 +76,8 @@ internal fun createDefaultMatcher(
return Experiment.Matcher(
appId,
appDisplayVersion,
appMinVersion,
appMaxVersion,
localeLanguage,
localeCountry,
deviceManufacturer,
......
......@@ -188,6 +188,8 @@ class ExperimentsDebugActivityTest {
match = Experiment.Matcher(
appId = null,
appDisplayVersion = null,
appMinVersion = null,
appMaxVersion = null,
debugTags = null,
deviceManufacturer = null,
deviceModel = null,
......@@ -208,6 +210,8 @@ class ExperimentsDebugActivityTest {
match = Experiment.Matcher(
appId = null,
appDisplayVersion = null,
appMinVersion = null,
appMaxVersion = null,
debugTags = null,
deviceManufacturer = null,
deviceModel = null,
......
package mozilla.components.service.experiments.util
import org.junit.Assert
import org.junit.Test
class VersionStringTest {
@Test
fun compareTo() {
// Comparing two exact versions returns 0
Assert.assertEquals(0, VersionString("1.0.0").compareTo(VersionString("1.0.0")))
// Comparing with left padded zeroes returns 0
Assert.assertEquals(0, VersionString("1.0.0").compareTo(VersionString("1.00.00")))
Assert.assertEquals(0, VersionString("1.1.1").compareTo(VersionString("1.01.0001")))
// Comparing to extra dotted sections with zero returns 0
Assert.assertEquals(0, VersionString("1").compareTo(VersionString("1.0.000")))
Assert.assertEquals(0, VersionString("1.1").compareTo(VersionString("1.1.000.0")))
// Comparing a higher version to a lower version returns 1
Assert.assertEquals(1, VersionString("1.1.0").compareTo(VersionString("1.0.0")))
Assert.assertEquals(1, VersionString("1.1.54321").compareTo(VersionString("1.1.12345")))
Assert.assertEquals(1, VersionString("51.1.0").compareTo(VersionString("50.1.0")))
Assert.assertEquals(1, VersionString("1.1.0").compareTo(VersionString("1.0.1")))
// Comparing a lower version to a higher version returns -1
Assert.assertEquals(-1, VersionString("1.0.0").compareTo(VersionString("1.1.0")))
Assert.assertEquals(-1, VersionString("1.1.12345").compareTo(VersionString("1.1.54321")))
Assert.assertEquals(-1, VersionString("50.1.0").compareTo(VersionString("51.1.0")))
Assert.assertEquals(-1, VersionString("1.0.1").compareTo(VersionString("1.1.0")))
}
@Test(expected = IllegalArgumentException::class)
fun `invalid VersionString's throw IllegalArgumentException`() {
VersionString("Invalid").compareTo(VersionString("Also_invalid"))
}
@Test
fun equals() {
Assert.assertEquals(VersionString("1.0.0"), VersionString("1.0.0"))
Assert.assertNotEquals(VersionString("1.0.0"), VersionString("1.0.1"))
}
}
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment