add page with status per language, with release information

    - echo "making stats for Tor Browser"

    - pip3 install httpx markdown

    - python3 bin/check-torbrowser-translations.py $WEBLATE_TOKEN

    - python3 bin/status-per-language.py $WEBLATE_TOKEN

    - cat ci-templates/stats-header.html tb-stats.html ci-templates/stats-footer.html > public/stats.html

    - cp ci-templates/style.css public/

#!/usr/bin/env python3

# this script is based/copied from

# Micah Lee's script for OnionShare

# translations at

# https://github.com/onionshare/onionshare/blob/main/docs/check-weblate.py

# modified by emmapeel for the Tor Project - 2023

import sys

import httpx

import asyncio

import markdown

import datetime

import configparser

resources_config = configparser.ConfigParser()

resources_config.read("./ci-templates/stats/resources-config.ini")

api_token = None

languages = {}

ignored_languages = "am kab"

component_translations = {}

dt = datetime.datetime.now()

date_built = dt.strftime("%A, %d. %B %Y %I:%M%p")

missing_translations = {}

try:

    CONFIG_FILE = sys.argv[2]

except IndexError:

    CONFIG_FILE = "./ci-templates/stats/resources-config.ini"

# def get_published_langs(lektorconfig):

# gets a list of all locales configured on a particular Lektor website

# needs a lektor i18n config file, such as the one available at

# https://gitlab.torproject.org/tpo/web/l10n/community/-/blob/main/configs/i18n.ini

#    mylangs = []

#    with open(lektorconfig, "rt") as config_file:

#        for config_option in config_file.readlines():

#            if config_option.startswith("translations"):

#                mylangs = [

#                    name.strip()

#                    for name in config_option.split("=")[1].split(",")

#                ]

#    return mylangs

async def api(path):

    global languages

    url = f"https://hosted.weblate.org{path}"

    # Wait a bit before each API call, to avoid hammering the server and

    # getting temporarily blocked

    await asyncio.sleep(1)

    async with httpx.AsyncClient() as client:

        r = await client.get(

            url, headers={"Authorization": f"Token {api_token}"}, timeout=60

        )

    if r.status_code == 200:

        print(f"GET {url}")

        return r.json()

    else:

        print(f"GET {url} | error {r.status_code}")

        return None

async def add_stats_to_component(obj, component):

    # process a json result object line and

    # adds the translation percentage to the results

    global component_translations

    for res in obj:

        if res["language"]["code"] != "en":

            languages[res["language"]["code"]]["population"] = res["language"][

                "population"

            ]

            component_translations[component][res["language"]["code"]] = {

                "total": res["total"],

                "total_words": res["total_words"],

                "translated": res["translated"],

                "translated_percent": res["translated_percent"],

                "fuzzy": res["fuzzy"],

                "fuzzy_percent": res["fuzzy_percent"],

                "failing_checks": res["failing_checks"],

                "have_suggestion": res["have_suggestion"],

                "have_comment": res["have_comment"],

                "translate_url": res["translate_url"],

                "last_change": res["last_change"],

                "is_released": False,

            }

async def get_translations(component):

    global component_translations

    obj = await api(f"/api/components/tor/{component}/translations/")

    await add_stats_to_component(obj["results"], component)

    while obj["next"] != None:

        newpage = obj["next"].split("/")[-1]

        obj = await api(

            f"/api/components/tor/{component}/translations/{newpage}"

        )

        await add_stats_to_component(obj["results"], component)

async def get_all_components():

    global component_translations

    obj = await api(f"/api/projects/tor/components/")

    # for res in obj["results"][1:3]:

    for res in obj["results"]:

        if res["slug"] not in component_translations:

            component_translations[res["slug"]] = {"name": res["name"]}

        await get_translations(res["slug"])

    while obj["next"] != None:

        newpage = obj["next"].split("/")[-1]

        obj = await api(f"/api/projects/tor/components/{newpage}")

        await get_translations(res["slug"])

        # await get_translations(res["slug"][1:3])

async def one_lang_page(lang_code):

    out = []

    for component in component_translations:

        if (

            lang_code in component_translations[component]

            and lang_code != "en"

        ):

            if (

                component_translations[component][lang_code]["is_released"]

                == True

            ):

                released = "*"

            else:

                released = "-"

            langstats = component_translations[component][lang_code]

            # we make a markdown table line for each locale, and then we sort them

            suggestions = (

                " | 0 "

                if langstats["have_suggestion"] == 0

                else f" | [{ langstats['have_suggestion'] }]"

                + f"({ langstats['translate_url'] }"

                + "?q=has%3Asuggestion&sort_by=-priority%2Cposition&checksum=#suggestions) "

            )

            comments = (

                " | 0 "

                if langstats["have_comment"] == 0

                else f" | [{ langstats['have_comment'] }]({ langstats['translate_url'] }"

                + "?q=has%3Acomment&sort_by=-priority%2Cposition&checksum=#comments) "

            )

            failing = (

                " | 0 "

                if langstats["failing_checks"] == 0

                else f" | [{ langstats['failing_checks'] }]({ langstats['translate_url'] }"

                + "?q=has%3Acheck&sort_by=-priority%2Cposition&checksum=#translations) "

            )

            tr_line = (

                f"|{ released }"

                + f"| { int(langstats['translated_percent']) }%"

                + f"| [{ component_translations[component]['name'] } ]({ component }.html)"

                + f"| { langstats['translated'] }/{ langstats['total'] }"

                + str(failing)

                + str(suggestions)

                + str(comments)

                + f" | { langstats['last_change'] } "

                + f" |"

            )

            out.append(tr_line)

        else:

            if lang_code not in missing_translations:

                missing_translations[lang_code] = []

            missing_translations[lang_code].append(component)

    out.sort()

    with open(f"lang-stats-{ lang_code }.md", "w") as output_file:

        output_file.write(

            f"\n# Translation status for {languages[lang_code]['name']}"

        )

        output_file.write(f"\nbuilt on {date_built}\n")

        output_file.write("\n[TOC]")

        output_file.write("\n### Translations")

        output_file.write(

            "\n|Released?| %|Component | Translated| Failing| Suggestions| Comments|Last change|"

        )

        output_file.write("\n|---:|---:|---------|---:|---:|---:|---:|---|\n")

        output_file.write("\n".join(out))

        output_file.write("\n")

        if lang_code in missing_translations:

            output_file.write("\n### Missing translations\n")

            for missing_component in missing_translations[lang_code]:

                output_file.write(f"\n - { missing_component }")

    markdown.markdownFromFile(

        input=f"lang-stats-{ lang_code }.md",

        extensions=[

            "markdown.extensions.tables:TableExtension",

            "md_in_html",

            "toc",

        ],

        output=f"{ lang_code }.html",

    )

    filenames = [

        "./ci-templates/stats-header.html",

        f"./{ lang_code }.html",

        "./ci-templates/stats-footer.html",

    ]

    with open(f"./public/{ lang_code }.html", "w") as outfile:

        for fname in filenames:

            with open(fname) as infile:

                for line in infile:

                    outfile.write(line)

def release_locale(locale, component):

    global component_translations

    if component not in component_translations:

        component_translations[component] = {}

    if locale not in component_translations[component]:

        component_translations[component][locale] = {}

    component_translations[component][locale]["is_released"] = True

async def add_releases():

    global component_translations

    # here we add the release info per locale

    for component in resources_config.keys():

        if "released" in resources_config[component]:

            locales = resources_config[component]["released"].split(", ")

            if "components" in resources_config[component].keys():

                for subcomponent in resources_config[component][

                    "components"

                ].split(", "):

                    for locale in locales:

                        release_locale(locale, subcomponent)

            else:

                for locale in locales:

                    try:

                        release_locale(locale, component)

                    except:

                        print(

                            f"WARNING\nWARNING!\nProblems with the component { component } { locale }!!!!"

                        )

                        print(component_translations[component][locale])

                        print(

                            f"please look at {component}, has problems with release/components"

                        )

        else:

            print(

                f"component { component } has no release info in "

                + "ci-templates/stats/resource_config.ini, please add!"

            )

async def main():

    global api_token, languages

    from markdown.extensions.tables import TableExtension

    if len(sys.argv) != 2:

        print(f"Usage: {sys.argv[0]} API_KEY")

        print(

            "You can find your personal API key at: https://hosted.weblate.org/accounts/profile/#api"

        )

        return

    api_token = sys.argv[1]

    # Get the list of languages in the OnionShare project

    res = await api("/api/projects/tor/languages/")

    for obj in res:

        if obj["code"] != "en":

            languages[obj["code"]] = obj

    # start building the page

    await get_all_components()

    await add_releases()

    with open("lang-stats.md", "w") as output_file:

        output_file.write("\n## Translation status per language\n")

        output_file.write(f"\nbuilt on {date_built}\n")

        output_file.write(

            "\n| Locale | Translated | % | Suggestions | Reviewed | Failing |\n"

        )

        output_file.write(

            "| ------ | ----------:|---:|-----------:| --------:| -------:|\n"

        )

        # Get the Tor Browser translations for each language and component

        for locale in languages:

            # we add a link and quick info

            output_file.write(

                f"| [{ languages[locale]['name'] }]({ locale }.html) "

                + f"| { languages[locale]['translated'] }/{ languages[locale]['total'] } "

                + f"| { languages[locale]['translated_percent'] }% "

                + f"| [{ languages[locale]['suggestions'] }]({ languages[locale]['translate_url'] }"

                + "?q=has%3Asuggestion&sort_by=-priority%2Cposition&checksum=#suggestions) "

                + f"| { languages[locale]['approved'] } "

                + f"| [{ languages[locale]['failing'] }]({ languages[locale]['translate_url'] }"

                + "?q=has%3Acheck&sort_by=-priority%2Cposition&checksum=) |\n"

            )

            await one_lang_page(locale)

    # convert markdown to html

    markdown.markdownFromFile(

        input="lang-stats.md",

        extensions=[

            "markdown.extensions.tables:TableExtension",

            "md_in_html",

            "toc",

        ],

        output="lang-stats.html",

    )

if __name__ == "__main__":

    asyncio.run(main())

# this file has configurations for the gitlabCI to

# generate stats, available at

#  https://tpo.pages.torproject.net/community/l10n/stats.html

# gathered by emmapeel GPL 3 2023 for the Tor Project

# used by bin/status-per-language.py

# and bin/check-torbrowser-translations.py (soon)

[community-portal]

released = es, ru, tr

[tpo-web]

released = ca, de, es, fa, fr, hu, is, it, ja, ka, km, ko, pt_BR, ru, sq, sw, tr, vi, zh_Hans, zh-Hant

[support-portal]

released = de, es, fa, fr, id, it, ko, pt_BR, ro, ru, sw, tr, uk, zh_Hans

[tor-browser-user-manual]

released = ar, bn, ca, de, el, es, fa, fi, fr, ga, he, hu, id, is, it, ka, km, ko, lt, mk, my, pl, pt_BR, pt_PT, ro, ru, sq, sw, th, tr, uk, vi, zh_Hans, zh_Hant

[tor-browser]

released = ar, ca, cs, da, de, el, es, fa, fi, fr, ga, he, hu, id, is, it, ja, ka, ko, lt, mk, ms, my, nb_NO, nl, pl, pt_BR, ro, ru, sq, sv, th, tr, uk, vi, zh_Hans, zh_Hant

components = tb-aboutdialogdtd, tb-abouttbupdatedtd, tb-abouttordtd, tb-android, tb-branddtd, tb-brandproperties, tor-browser-brandingbrand-ftl, tb-browseronboardingproperties, tb-cryptosafetypromptproperties, tb-languagenotification, tb-newidentityproperties, tb-onboardingproperties, tb-onionlocationproperties, tb-rulesetsproperties, tb-securitylevelproperties, tb-settingsproperties, tb-torconnectproperties, tb-torbuttondtd, tb-torbuttonproperties, tb-torlauncherproperties

[snowflake]

released = cs, de, el, es, fa, fi, fr, it, ja, ka, pl, pt_BR, ro, ru, sq, sv, sw, tr, uk, vi, zh_Hans

components = snowflake-badge, snowflake-website

[tor-check]

released = es, fi, fr, ga, he, hu, id, mk, th, tr, zh_Hans