Unverified Commit 93122b61 authored by Severin Rudie's avatar Severin Rudie Committed by GitHub
Browse files

3088 filter incomplete locales on release builds (#7581) 

* For #3088: add translation pruning Py scripts from Focus

The only change made was updating "Focus/Klar" to "Fenix"

* For 3088: call translation pruning scripts during release builds

* For 3088: fix filter-release-translations.py

Moving it into Fenix introduced one additional layer of nesting, which was not accounted for in a relative path.  🤦

* For 3088: comment about the risks of modifying source

* For 3088: updated locales.py to read from l10n.toml

Also removed some previously unused properties from locales.py

* For 3088: added error handling to script in locales.py

* For 3088: updated locale parser per review comments
parent 90409838

automation/taskcluster/l10n/filter-release-translations.py

0 → 100644
+50 −0
Original line number Diff line number Diff line 
# This Source Code Form is subject to the terms of the Mozilla Public

# License, v. 2.0. If a copy of the MPL was not distributed with this

# file, You can obtain one at http://mozilla.org/MPL/2.0/.



"""

This script takes the list of release locales defined in locales.py

and removes all non-release translations from the application's

resource folder.

This script is run before building a release version so that

those builds only contain locales we actually want to ship.

"""



import os

import re

import shutil



from locales import RELEASE_LOCALES



LANGUAGE_REGEX = re.compile('^[a-z]{2,3}$')

LANGUAGE_REGION_REGEX = re.compile('^([a-z]{2})-r([A-Z]{2})$')



# Get all resource directories that start with "values-" (and remove the "values-" prefix)

resources_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'app', 'src', 'main', 'res')

locale_dirs = [localeDir.replace('values-', '') for localeDir in os.listdir(resources_dir)

               if os.path.isdir(os.path.join(resources_dir, localeDir))

               and localeDir.startswith('values-')]



# Remove everything that doesn't look like it's a resource folder for a specific language (e.g. sw600dp)

locale_dirs = filter(lambda x: LANGUAGE_REGEX.match(x) or LANGUAGE_REGION_REGEX.match(x), locale_dirs)



# Android prefixes regions with an "r" (de-rDE). Remove the prefix.

locale_dirs = [item.replace('-r','-') for item in locale_dirs]



# Now determine the list of locales that are not in our release list

locales_to_remove = list(set(locale_dirs) - set(RELEASE_LOCALES))



print "RELEASE LOCALES:", ", ".join(RELEASE_LOCALES)

print "APP LOCALES:", ", ".join(locale_dirs)

print "REMOVE:", ", ".join(locales_to_remove) if len(locales_to_remove) > 0 else "-Nothing-"



# Remove unneeded resource folders

for locale in locales_to_remove:

    # Build resource folder names from locale: de -> values-de, de-DE -> vlaues-de-rDE

    parts = locale.split("-")

    folder = "values-" + (parts[0] if len(parts) == 1 else parts[0] + "-r" + parts[1])

    path = os.path.join(resources_dir, folder)



    print "* Removing: ", path

    # This modifies source, which could cause problems if we ever start caching source

    shutil.rmtree(path)
 
 No newline at end of file

automation/taskcluster/l10n/locales.py

0 → 100644
+59 −0
Original line number Diff line number Diff line 
#!/usr/bin/python



#  This Source Code Form is subject to the terms of the Mozilla Public

# License, v. 2.0. If a copy of the MPL was not distributed with this

# file, You can obtain one at http://mozilla.org/MPL/2.0/.



from __future__ import absolute_import, print_function, unicode_literals



import re



OPEN_LOCALES = "locales = ["

CLOSE_LOCALES = "]"



def trim_to_locale(str):

    match = re.search('\s*"([a-z]+-?[A-Z]*)",\s*', str)

    if not match:

        raise Exception("Failed parsing locale found in l10n.toml: " + str)

    return match.group(1)





# This file is a dumb parser that converts values from '/l10n.toml' to be easily consumed from

# Python.

#

# 'l10n.toml' has a very simple structure, and it is reasonable to believe that this (very basic)

# algorithm will continue to work as it is changed.

#

# Alternatives to custom parsing that were considered:

# - Using standard library module --- none exists to parse TOML

# - Importing a TOML parsing module --- introduces an additional build step, complexity, and

#   security risk

# - Vendoring a TOML module --- large amount of code given the use case. Introduces a security

#   risk

def get_release_locales():

    with open(r"l10n.toml") as f:

        file = f.read().splitlines()



    locales_opened = False

    locales_closed = False



    found_locales = []



    for line in file:

        if line == OPEN_LOCALES:

            locales_opened = True

        elif line == CLOSE_LOCALES:

            locales_closed = True

            break

        elif locales_opened:

            found_locales.append(trim_to_locale(line))



    if locales_opened == False:

        raise Exception("Could not find `locales` open in l10n.toml")

    if locales_closed == False:

        raise Exception("Could not find `locales` close in l10n.toml")



    return found_locales





RELEASE_LOCALES = get_release_locales()

taskcluster/ci/build/kind.yml

+4 −0
Original line number Diff line number Diff line
@@ -100,6 +100,7 @@ jobs: 
            release-type: beta

        include-release-version: true

        include-shippable-secrets: true

        filter-incomplete-translations: true

        run:

            geckoview-engine: geckoBeta

            gradle-build-type: fenixBeta
@@ -112,6 +113,7 @@ jobs: 
            release-type: production

        include-release-version: true

        include-shippable-secrets: true

        filter-incomplete-translations: true

        run:

            geckoview-engine: geckoBeta

            gradle-build-type: fenixProduction
@@ -137,6 +139,7 @@ jobs: 
            nightly: true

        include-release-version: true

        include-shippable-secrets: true

        filter-incomplete-translations: true

        run:

            geckoview-engine: geckoBeta

            gradle-build-type: fennecBeta
@@ -150,6 +153,7 @@ jobs: 
            nightly: true

        include-release-version: true

        include-shippable-secrets: true

        filter-incomplete-translations: true

        run:

            geckoview-engine: geckoBeta

            gradle-build-type: fennecProduction

taskcluster/fenix_taskgraph/transforms/build.py

+9 −0
Original line number Diff line number Diff line
@@ -125,3 +125,12 @@ def add_artifacts(config, tasks): 
                apks[apk["abi"]] = apk_name



        yield task





@transforms.add

def filter_incomplete_translation(config, tasks):

    for task in tasks:

        if task.pop("filter-incomplete-translations", False):

            # filter-release-translations modifies source, which could cause problems if we ever start caching source

            task["run"]["pre-gradlew"].append(["python", "automation/taskcluster/l10n/filter-release-translations.py"])

        yield task