Unverified Commit d146e756 authored by Severin Rudie's avatar Severin Rudie Committed by GitHub
Browse files

3088 filter locales 2nd try (#7839)

* For #3088: filter incomplete locales in release builds

* For 3088: update locale filtering to not error on nightly builds
parent 0b1a6736
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
This script takes the list of release locales defined in locales.py
and removes all non-release translations from the application's
resource folder.
This script is run before building a release version so that
those builds only contain locales we actually want to ship.
"""
import os
import re
import shutil
from locales import RELEASE_LOCALES
LANGUAGE_REGEX = re.compile('^[a-z]{2,3}$')
LANGUAGE_REGION_REGEX = re.compile('^([a-z]{2})-r([A-Z]{2})$')
# Get all resource directories that start with "values-" (and remove the "values-" prefix)
resources_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'app', 'src', 'main', 'res')
locale_dirs = [localeDir.replace('values-', '') for localeDir in os.listdir(resources_dir)
if os.path.isdir(os.path.join(resources_dir, localeDir))
and localeDir.startswith('values-')]
# Remove everything that doesn't look like it's a resource folder for a specific language (e.g. sw600dp)
locale_dirs = filter(lambda x: LANGUAGE_REGEX.match(x) or LANGUAGE_REGION_REGEX.match(x), locale_dirs)
# Android prefixes regions with an "r" (de-rDE). Remove the prefix.
locale_dirs = [item.replace('-r','-') for item in locale_dirs]
# Now determine the list of locales that are not in our release list
locales_to_remove = list(set(locale_dirs) - set(RELEASE_LOCALES))
print "RELEASE LOCALES:", ", ".join(RELEASE_LOCALES)
print "APP LOCALES:", ", ".join(locale_dirs)
print "REMOVE:", ", ".join(locales_to_remove) if len(locales_to_remove) > 0 else "-Nothing-"
# Remove unneeded resource folders
for locale in locales_to_remove:
# Build resource folder names from locale: de -> values-de, de-DE -> vlaues-de-rDE
parts = locale.split("-")
folder = "values-" + (parts[0] if len(parts) == 1 else parts[0] + "-r" + parts[1])
path = os.path.join(resources_dir, folder)
print "* Removing: ", path
# This modifies source, which could cause problems if we ever start caching source
shutil.rmtree(path)
\ No newline at end of file
#!/usr/bin/python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import re
OPEN_LOCALES = "locales = ["
CLOSE_LOCALES = "]"
def trim_to_locale(str):
match = re.search('\s*"([a-z]+-?[A-Z]*)",\s*', str)
if not match:
raise Exception("Failed parsing locale found in l10n.toml: " + str)
return match.group(1)
# This file is a dumb parser that converts values from '/l10n.toml' to be easily consumed from
# Python.
#
# 'l10n.toml' has a very simple structure, and it is reasonable to believe that this (very basic)
# algorithm will continue to work as it is changed.
#
# Alternatives to custom parsing that were considered:
# - Using standard library module --- none exists to parse TOML
# - Importing a TOML parsing module --- introduces an additional build step, complexity, and
# security risk
# - Vendoring a TOML module --- large amount of code given the use case. Introduces a security
# risk
def get_release_locales():
with open(r"l10n.toml") as f:
file = f.read().splitlines()
locales_opened = False
locales_closed = False
found_locales = []
for line in file:
if line == OPEN_LOCALES:
locales_opened = True
elif line == CLOSE_LOCALES:
locales_closed = True
break
elif locales_opened:
found_locales.append(trim_to_locale(line))
if locales_opened == False:
raise Exception("Could not find `locales` open in l10n.toml")
if locales_closed == False:
raise Exception("Could not find `locales` close in l10n.toml")
return found_locales
RELEASE_LOCALES = get_release_locales()
...@@ -100,6 +100,7 @@ jobs: ...@@ -100,6 +100,7 @@ jobs:
release-type: beta release-type: beta
include-release-version: true include-release-version: true
include-shippable-secrets: true include-shippable-secrets: true
filter-incomplete-translations: true
run: run:
geckoview-engine: geckoBeta geckoview-engine: geckoBeta
gradle-build-type: fenixBeta gradle-build-type: fenixBeta
...@@ -112,6 +113,7 @@ jobs: ...@@ -112,6 +113,7 @@ jobs:
release-type: production release-type: production
include-release-version: true include-release-version: true
include-shippable-secrets: true include-shippable-secrets: true
filter-incomplete-translations: true
run: run:
geckoview-engine: geckoBeta geckoview-engine: geckoBeta
gradle-build-type: fenixProduction gradle-build-type: fenixProduction
...@@ -134,9 +136,11 @@ jobs: ...@@ -134,9 +136,11 @@ jobs:
attributes: attributes:
# TODO replace `nightly: true` by `release-type: beta` once the data migration # TODO replace `nightly: true` by `release-type: beta` once the data migration
# testing is over # testing is over
# TODO when making the above change, uncomment `filter-incomplete-translations: true`
nightly: true nightly: true
include-release-version: true include-release-version: true
include-shippable-secrets: true include-shippable-secrets: true
# filter-incomplete-translations: true
run: run:
geckoview-engine: geckoBeta geckoview-engine: geckoBeta
gradle-build-type: fennecBeta gradle-build-type: fennecBeta
...@@ -147,9 +151,11 @@ jobs: ...@@ -147,9 +151,11 @@ jobs:
attributes: attributes:
# TODO replace `nightly: true` by `release-type: production` once the data migration # TODO replace `nightly: true` by `release-type: production` once the data migration
# testing is over # testing is over
# TODO when making the above change, uncomment `filter-incomplete-translations: true`
nightly: true nightly: true
include-release-version: true include-release-version: true
include-shippable-secrets: true include-shippable-secrets: true
# filter-incomplete-translations: true
run: run:
geckoview-engine: geckoBeta geckoview-engine: geckoBeta
gradle-build-type: fennecProduction gradle-build-type: fennecProduction
......
...@@ -125,3 +125,13 @@ def add_artifacts(config, tasks): ...@@ -125,3 +125,13 @@ def add_artifacts(config, tasks):
apks[apk["abi"]] = apk_name apks[apk["abi"]] = apk_name
yield task yield task
@transforms.add
def filter_incomplete_translation(config, tasks):
for task in tasks:
if task.pop("filter-incomplete-translations", False):
# filter-release-translations modifies source, which could cause problems if we ever start caching source
pre_gradlew = task["run"].setdefault("pre-gradlew", [])
pre_gradlew.append(["python", "automation/taskcluster/l10n/filter-release-translations.py"])
yield task
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment