From c369ce36f0ec0ea6aa7aa810d9148508565f4b74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20Knau=C3=9F?= <sknauss@kde.org> Date: Fri, 3 Jan 2025 23:03:27 +0100 Subject: [PATCH 1/4] WeblatePermission: Handle user renames correctly. Create a userdb_cache that ships a username -> id link to check renames. --- files/config/weblate_permissions.yaml | 1 + .../scripts/tests/data/permissions_stage1.yml | 1 + .../scripts/tests/data/permissions_stage2.yml | 1 + .../scripts/tests/data/permissions_stage3.yml | 1 + files/scripts/tests/data/superuser.yml | 1 + .../scripts/tests/test_weblate_permissions.py | 26 +++++++++ files/scripts/weblate_permissions.py | 58 ++++++++++++++++++- 7 files changed, 88 insertions(+), 1 deletion(-) diff --git a/files/config/weblate_permissions.yaml b/files/config/weblate_permissions.yaml index f7a4ef1..775239d 100644 --- a/files/config/weblate_permissions.yaml +++ b/files/config/weblate_permissions.yaml @@ -361,3 +361,4 @@ superusers: - drebs - hefee - intrigeri +userdb_cache: /var/lib/weblate/userdb_cache.yml diff --git a/files/scripts/tests/data/permissions_stage1.yml b/files/scripts/tests/data/permissions_stage1.yml index 874bbd5..aac1809 100644 --- a/files/scripts/tests/data/permissions_stage1.yml +++ b/files/scripts/tests/data/permissions_stage1.yml @@ -15,3 +15,4 @@ roles: - suggestion.add users: superusers: +userdb_cache: /tmp/userdb_cache.yml diff --git a/files/scripts/tests/data/permissions_stage2.yml b/files/scripts/tests/data/permissions_stage2.yml index 469c199..3fa4cb2 100644 --- a/files/scripts/tests/data/permissions_stage2.yml +++ b/files/scripts/tests/data/permissions_stage2.yml @@ -19,3 +19,4 @@ users: groups: - Users superusers: +userdb_cache: /tmp/userdb_cache.yml diff --git a/files/scripts/tests/data/permissions_stage3.yml b/files/scripts/tests/data/permissions_stage3.yml index c82d985..e386942 100644 --- a/files/scripts/tests/data/permissions_stage3.yml +++ b/files/scripts/tests/data/permissions_stage3.yml @@ -6,3 +6,4 @@ roles: - __deleted users: superusers: +userdb_cache: /tmp/userdb_cache.yml diff --git a/files/scripts/tests/data/superuser.yml b/files/scripts/tests/data/superuser.yml index a87abf4..8563f3e 100644 --- a/files/scripts/tests/data/superuser.yml +++ b/files/scripts/tests/data/superuser.yml @@ -13,3 +13,4 @@ users: - Viewers superusers: - User1 +userdb_cache: /tmp/userdb_cache.yml diff --git a/files/scripts/tests/test_weblate_permissions.py b/files/scripts/tests/test_weblate_permissions.py index 311b471..e9d8d9a 100644 --- a/files/scripts/tests/test_weblate_permissions.py +++ b/files/scripts/tests/test_weblate_permissions.py @@ -41,6 +41,8 @@ class testTP(unittest.TestCase): for i in User.objects.all(): if i.username not in self.existing_user: i.delete() + userdb_cache = pathlib.Path("/tmp/userdb_cache.yml") + userdb_cache.unlink(missing_ok=True) def test_create_roles_and_groups(self): group = Group.objects.filter(name="Test Group").first() @@ -388,3 +390,27 @@ class testTP(unittest.TestCase): ' - Users\n' ' - Viewers', ]) + + def test_rename_user(self): + user2 = User.objects.create(username="User2") + e = tp.Config([self.datapath/'superuser.yml']) + with self.assertLogs(level="DEBUG") as cm: + tp.weblate_permission(e, True) + user1 = User.objects.filter(username="User1").first() + self.assertEqual(user1.is_superuser, True) + + user2.delete() + user1.username = "NewUsername" + user1.save() + user_takeover = User.objects.create(username="User1") + with self.assertLogs(level="DEBUG") as cm: + tp.weblate_permission(e, True) + + print(cm.output) + self.assertEqual(cm.output[-3:], [ + 'WARNING:weblate_permissions:The referenced user User1 renamed itself to NewUsername. Update the config accordingly.', + 'WARNING:weblate_permissions:Referenced user User2 does not exist. Create the account now or delete user from the configuration.', + 'ERROR:weblate_permissions:Stop to set permissions - Update the configuration first.', + ]) + self.assertEqual(User.objects.filter(username="User1").first().is_superuser, False) + self.assertEqual(User.objects.filter(username="NewUsername").first().is_superuser, True) # As the script did stop diff --git a/files/scripts/weblate_permissions.py b/files/scripts/weblate_permissions.py index 36e06ae..0c1899a 100644 --- a/files/scripts/weblate_permissions.py +++ b/files/scripts/weblate_permissions.py @@ -124,9 +124,12 @@ maintenance without the script interfering in the work. To run the script in `--enforceMaintenance` option. ''' +import datetime +import itertools import logging import logging.config import operator +import pathlib import yaml import tailsWeblate # Needed because of the side effect to setup the django application from weblate.auth.models import Group, Language, Permission, Project, Role, User @@ -450,6 +453,59 @@ class WeblatePermission: check(a, self.groups, "More groups found, than defined.", logger.warning) def fix_users(self): + # Usernames are unique, but changeable in Weblate. + # The problem is that using user ids in the config is not very readable for admistrators. + # To solve this we keep a userdb_cach it is a dict username -> id, to detect user renames. + + changed = False # userdb_cache changed (needs to be saved) + config_warning = False # detected a warning + userdb_path = pathlib.Path(self.config.userdb_cache) + usernames = set(itertools.chain.from_iterable((self.config.users, self.config.superusers))) + if userdb_path.exists(): + with userdb_path.open() as f: + userdb_cache = yaml.safe_load(f) + + # Cleanup userdb cache (delete all unknown entries) + to_del = set(userdb_cache.keys()) - usernames + changed = bool(to_del) + for username in to_del: + del userdb_cache[username] + else: + userdb_cache = {} + + for username in sorted(usernames): + if username == "__default": + continue + try: + user_db = User.objects.get(username=username) + except User.DoesNotExist: + logger.warning(f"Referenced user {username} does not exist. Create the account now or delete user from the configuration.") + config_warning = True + continue + user_cache = userdb_cache.get(username, None) + if user_cache is None: + userdb_cache[user_db.username] = { + 'id':user_db.id, + 'email': user_db.email, + 'added': datetime.datetime.now().isoformat(), + } + changed = True + continue + elif user_db.id != user_cache['id']: + new_user = User.objects.get(id=user_cache['id']) + logger.warning(f"The referenced user {username} renamed itself to {new_user.username}. Update the config accordingly.") + config_warning = True + + if config_warning: + logger.error("Stop to set permissions - Update the configuration first.") + if self.enforce: + return + elif changed: + with userdb_path.open('w') as f: + yaml.dump(userdb_cache, f, default_flow_style=False) + + default_user = self.config.users['__default'] + for i in sorted(User.objects.all(),key=lambda i:i.username.lower()): changed = False @@ -457,7 +513,7 @@ class WeblatePermission: # Ignore inactive users, if they are not listed explictitly continue - e = self.config.users.get(i.username, self.config.users['__default']) + e = self.config.users.get(i.username, default_user) dbquery = DBQuery(i.groups, 'name', Group.objects) if not self.attribute(dbquery)(e['groups'], f"Group mismatch for user({i.username})", self.logfunc): -- GitLab From 7ac5b76bb334d40fad117913cf0a41c2c36a3d16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20Knau=C3=9F?= <hefee@debian.org> Date: Fri, 10 Jan 2025 15:49:58 +0100 Subject: [PATCH 2/4] Refactor UserCache into own class. --- files/scripts/weblate_permissions.py | 126 ++++++++++++++++++--------- 1 file changed, 83 insertions(+), 43 deletions(-) diff --git a/files/scripts/weblate_permissions.py b/files/scripts/weblate_permissions.py index 0c1899a..370c46d 100644 --- a/files/scripts/weblate_permissions.py +++ b/files/scripts/weblate_permissions.py @@ -131,6 +131,7 @@ import logging.config import operator import pathlib import yaml +from typing import List import tailsWeblate # Needed because of the side effect to setup the django application from weblate.auth.models import Group, Language, Permission, Project, Role, User @@ -234,6 +235,80 @@ class DBQuery: return True +class UserCache: + """ + Usernames are unique, but changeable in Weblate. + The problem is that using user ids in the config is not very readable for admistrators. + To solve this we keep a userdb_cach it is a dict username -> id, to detect user renames. + """ + def __init__(self, path: pathlib.Path): + + self.changed = False # userdb_cache changed (needs to be saved) + self.warning_detected = False # detected a warning + self.path = pathlib.Path(path) + self._read() + + def _read(self): + if self.path.exists(): + with self.path.open() as f: + self.cache = yaml.safe_load(f) + else: + self.cache = {} + + def _save(self): + with self.path.open('w') as f: + yaml.dump(self.cache, f, default_flow_style=False) + self.changed = False + + def save_if_changed(self): + if self.changed: + self._save() + + def add(self, user): + self.cache[user.username] = { + 'id':user.id, + 'email': user.email, + 'added': datetime.datetime.now().isoformat(), + } + self.changed = True + + def delete(self, username): + del self.cache[username] + self.changed = True + + def cleanup_cache(self, usernames:List[str]): + to_del = set(self.cache.keys()) - usernames + for username in to_del: + self.delete(username) + + def check(self, usernames:List[str]): + for username in usernames: + if username == "__default": + continue + + user_cache = self.cache.get(username, None) + user_db = None + try: + user_db = User.objects.get(username=username) + except User.DoesNotExist: + if user_cache is None: + logger.warning(f"Referenced user {username} does not exist. Create the account now or delete user from the configuration.") + self.warning_detected = True + continue + + if user_cache is None: + self.add(user_db) + continue + elif user_db is None or user_db.id != user_cache['id']: + try: + new_user = User.objects.get(id=user_cache['id']) + except User.DoesNotExist: + logger.warning(f"Referenced user {username} is deleted. Create the account again or delete user from the configuration.") + else: + logger.warning(f"The referenced user {username} renamed itself to {new_user.username}. Update the config accordingly.") + self.warning_detected = True + + class WeblatePermission: def __init__(self, config, enforce): @@ -453,56 +528,21 @@ class WeblatePermission: check(a, self.groups, "More groups found, than defined.", logger.warning) def fix_users(self): + # Usernames are unique, but changeable in Weblate. - # The problem is that using user ids in the config is not very readable for admistrators. - # To solve this we keep a userdb_cach it is a dict username -> id, to detect user renames. + # The UserCache class keeps track of username <-> id. - changed = False # userdb_cache changed (needs to be saved) - config_warning = False # detected a warning - userdb_path = pathlib.Path(self.config.userdb_cache) + user_cache = UserCache(self.config.userdb_cache) usernames = set(itertools.chain.from_iterable((self.config.users, self.config.superusers))) - if userdb_path.exists(): - with userdb_path.open() as f: - userdb_cache = yaml.safe_load(f) - - # Cleanup userdb cache (delete all unknown entries) - to_del = set(userdb_cache.keys()) - usernames - changed = bool(to_del) - for username in to_del: - del userdb_cache[username] - else: - userdb_cache = {} + user_cache.cleanup_cache(usernames) + user_cache.check(sorted(usernames)) - for username in sorted(usernames): - if username == "__default": - continue - try: - user_db = User.objects.get(username=username) - except User.DoesNotExist: - logger.warning(f"Referenced user {username} does not exist. Create the account now or delete user from the configuration.") - config_warning = True - continue - user_cache = userdb_cache.get(username, None) - if user_cache is None: - userdb_cache[user_db.username] = { - 'id':user_db.id, - 'email': user_db.email, - 'added': datetime.datetime.now().isoformat(), - } - changed = True - continue - elif user_db.id != user_cache['id']: - new_user = User.objects.get(id=user_cache['id']) - logger.warning(f"The referenced user {username} renamed itself to {new_user.username}. Update the config accordingly.") - config_warning = True - - if config_warning: + if user_cache.warning_detected: logger.error("Stop to set permissions - Update the configuration first.") if self.enforce: return - elif changed: - with userdb_path.open('w') as f: - yaml.dump(userdb_cache, f, default_flow_style=False) + else: + user_cache.save_if_changed() default_user = self.config.users['__default'] -- GitLab From 15c1975c0814796b81b3f14a7d37733aced6646c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20Knau=C3=9F?= <hefee@debian.org> Date: Fri, 10 Jan 2025 15:50:58 +0100 Subject: [PATCH 3/4] Add simple test for rename without overtake. --- .../scripts/tests/test_weblate_permissions.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/files/scripts/tests/test_weblate_permissions.py b/files/scripts/tests/test_weblate_permissions.py index e9d8d9a..ddfabc3 100644 --- a/files/scripts/tests/test_weblate_permissions.py +++ b/files/scripts/tests/test_weblate_permissions.py @@ -399,6 +399,26 @@ class testTP(unittest.TestCase): user1 = User.objects.filter(username="User1").first() self.assertEqual(user1.is_superuser, True) + user1.username = "NewUsername" + user1.save() + with self.assertLogs(level="DEBUG") as cm: + tp.weblate_permission(e, True) + + print(cm.output) + self.assertEqual(cm.output[-2:], [ + 'WARNING:weblate_permissions:The referenced user User1 renamed itself to NewUsername. Update the config accordingly.', + 'ERROR:weblate_permissions:Stop to set permissions - Update the configuration first.', + ]) + self.assertEqual(User.objects.filter(username="NewUsername").first().is_superuser, True) # As the script did stop + + def test_rename_user_overtake(self): + user2 = User.objects.create(username="User2") + e = tp.Config([self.datapath/'superuser.yml']) + with self.assertLogs(level="DEBUG") as cm: + tp.weblate_permission(e, True) + user1 = User.objects.filter(username="User1").first() + self.assertEqual(user1.is_superuser, True) + user2.delete() user1.username = "NewUsername" user1.save() @@ -409,7 +429,7 @@ class testTP(unittest.TestCase): print(cm.output) self.assertEqual(cm.output[-3:], [ 'WARNING:weblate_permissions:The referenced user User1 renamed itself to NewUsername. Update the config accordingly.', - 'WARNING:weblate_permissions:Referenced user User2 does not exist. Create the account now or delete user from the configuration.', + 'WARNING:weblate_permissions:Referenced user User2 is deleted. Create the account again or delete user from the configuration.', 'ERROR:weblate_permissions:Stop to set permissions - Update the configuration first.', ]) self.assertEqual(User.objects.filter(username="User1").first().is_superuser, False) -- GitLab From 1ea4ff7f042a907a9d060a5bcb131e2b87d73143 Mon Sep 17 00:00:00 2001 From: hefee <tails@sandroknauss.de> Date: Fri, 17 Jan 2025 23:34:21 +0000 Subject: [PATCH 4/4] refactoring & suggestions by zen --- .../scripts/tests/test_weblate_permissions.py | 52 +++++++++++++++---- files/scripts/weblate_permissions.py | 48 ++++++++--------- 2 files changed, 64 insertions(+), 36 deletions(-) diff --git a/files/scripts/tests/test_weblate_permissions.py b/files/scripts/tests/test_weblate_permissions.py index ddfabc3..2727bd2 100644 --- a/files/scripts/tests/test_weblate_permissions.py +++ b/files/scripts/tests/test_weblate_permissions.py @@ -391,8 +391,8 @@ class testTP(unittest.TestCase): ' - Viewers', ]) - def test_rename_user(self): - user2 = User.objects.create(username="User2") + def test_renaming_user_causes_permission_enforcement_error(self): + User.objects.create(username="User2") # referenced by superuser.yml e = tp.Config([self.datapath/'superuser.yml']) with self.assertLogs(level="DEBUG") as cm: tp.weblate_permission(e, True) @@ -405,14 +405,25 @@ class testTP(unittest.TestCase): tp.weblate_permission(e, True) print(cm.output) - self.assertEqual(cm.output[-2:], [ + self.assertEqual(cm.output[-4:], [ 'WARNING:weblate_permissions:The referenced user User1 renamed itself to NewUsername. Update the config accordingly.', - 'ERROR:weblate_permissions:Stop to set permissions - Update the configuration first.', + 'INFO:weblate_permissions.audit:Group mismatch for user(NewUsername)\n +++ expected\n --- actual\n - Users\n - Viewers', + 'INFO:weblate_permissions.audit:Wrong superuser status for NewUsername (True != False)', + 'INFO:weblate_permissions.audit:Save user(NewUsername)', ]) - self.assertEqual(User.objects.filter(username="NewUsername").first().is_superuser, True) # As the script did stop + self.assertEqual(User.objects.filter(username="NewUsername").first().is_superuser, False) - def test_rename_user_overtake(self): - user2 = User.objects.create(username="User2") + with self.assertLogs(level="DEBUG") as cm: + tp.weblate_permission(e, True) + + print(cm.output) + self.assertEqual(cm.output[-1:], [ + 'WARNING:weblate_permissions:The referenced user User1 renamed itself to NewUsername. Update the config accordingly.', + ]) + self.assertEqual(User.objects.filter(username="NewUsername").first().is_superuser, False) + + def test_prevent_overtake_permissions_of_renamed_user(self): + user2 = User.objects.create(username="User2") # referenced by superuser.yml e = tp.Config([self.datapath/'superuser.yml']) with self.assertLogs(level="DEBUG") as cm: tp.weblate_permission(e, True) @@ -423,14 +434,33 @@ class testTP(unittest.TestCase): user1.username = "NewUsername" user1.save() user_takeover = User.objects.create(username="User1") + with self.assertLogs(level="DEBUG") as cm: + tp.weblate_permission(e, True) + + print(cm.output[-8:]) + self.assertEqual(cm.output[-8:], [ + 'WARNING:weblate_permissions:User1 was taken over by someone else.', + 'WARNING:weblate_permissions:The referenced user User1 renamed itself to NewUsername. Update the config accordingly.', + 'WARNING:weblate_permissions:The Referenced user User2 was deleted. Create the account again or delete user from the configuration.', + 'INFO:weblate_permissions.audit:Group mismatch for user(NewUsername)\n +++ expected\n --- actual\n - Users\n - Viewers', + 'INFO:weblate_permissions.audit:Wrong superuser status for NewUsername (True != False)', + 'INFO:weblate_permissions.audit:Save user(NewUsername)', + 'INFO:weblate_permissions.audit:Wrong superuser status for User1 (False != True)', + 'INFO:weblate_permissions:User1 is marked (the issues are listed above): skip to save user.', + ]) + self.assertEqual(User.objects.filter(username="User1").first().is_superuser, False) + self.assertEqual(User.objects.filter(username="NewUsername").first().is_superuser, False) + with self.assertLogs(level="DEBUG") as cm: tp.weblate_permission(e, True) print(cm.output) - self.assertEqual(cm.output[-3:], [ + self.assertEqual(cm.output[-5:], [ + 'WARNING:weblate_permissions:User1 was taken over by someone else.', 'WARNING:weblate_permissions:The referenced user User1 renamed itself to NewUsername. Update the config accordingly.', - 'WARNING:weblate_permissions:Referenced user User2 is deleted. Create the account again or delete user from the configuration.', - 'ERROR:weblate_permissions:Stop to set permissions - Update the configuration first.', + 'WARNING:weblate_permissions:The Referenced user User2 was deleted. Create the account again or delete user from the configuration.', + 'INFO:weblate_permissions.audit:Wrong superuser status for User1 (False != True)', + 'INFO:weblate_permissions:User1 is marked (the issues are listed above): skip to save user.', ]) self.assertEqual(User.objects.filter(username="User1").first().is_superuser, False) - self.assertEqual(User.objects.filter(username="NewUsername").first().is_superuser, True) # As the script did stop + self.assertEqual(User.objects.filter(username="NewUsername").first().is_superuser, False) diff --git a/files/scripts/weblate_permissions.py b/files/scripts/weblate_permissions.py index 370c46d..742655b 100644 --- a/files/scripts/weblate_permissions.py +++ b/files/scripts/weblate_permissions.py @@ -239,12 +239,11 @@ class UserCache: """ Usernames are unique, but changeable in Weblate. The problem is that using user ids in the config is not very readable for admistrators. - To solve this we keep a userdb_cach it is a dict username -> id, to detect user renames. + To solve this we keep a userdb_cache, which is a dict mapping username to ids, to detect user renames. """ def __init__(self, path: pathlib.Path): - - self.changed = False # userdb_cache changed (needs to be saved) - self.warning_detected = False # detected a warning + self.changed = False # userdb_cache changed (needs to be saved) + self.warning_users:List[str] = [] # detected a warning for users self.path = pathlib.Path(path) self._read() @@ -287,26 +286,27 @@ class UserCache: continue user_cache = self.cache.get(username, None) - user_db = None try: user_db = User.objects.get(username=username) except User.DoesNotExist: - if user_cache is None: - logger.warning(f"Referenced user {username} does not exist. Create the account now or delete user from the configuration.") - self.warning_detected = True - continue + user_db = None if user_cache is None: - self.add(user_db) - continue + if user_db is None: + logger.warning(f"Referenced user {username} does not exist. Create the account now or delete user from the configuration.") + self.warning_users.append(username) + else: + self.add(user_db) elif user_db is None or user_db.id != user_cache['id']: + if user_db: + logger.warning(f"{username} was taken over by someone else.") try: - new_user = User.objects.get(id=user_cache['id']) + renamed_user = User.objects.get(id=user_cache['id']) except User.DoesNotExist: - logger.warning(f"Referenced user {username} is deleted. Create the account again or delete user from the configuration.") + logger.warning(f"The Referenced user {username} was deleted. Create the account again or delete user from the configuration.") else: - logger.warning(f"The referenced user {username} renamed itself to {new_user.username}. Update the config accordingly.") - self.warning_detected = True + logger.warning(f"The referenced user {username} renamed itself to {renamed_user.username}. Update the config accordingly.") + self.warning_users.append(username) class WeblatePermission: @@ -528,23 +528,17 @@ class WeblatePermission: check(a, self.groups, "More groups found, than defined.", logger.warning) def fix_users(self): + default_user = self.config.users['__default'] + usernames = set(itertools.chain.from_iterable((self.config.users, self.config.superusers))) # Usernames are unique, but changeable in Weblate. # The UserCache class keeps track of username <-> id. - user_cache = UserCache(self.config.userdb_cache) - usernames = set(itertools.chain.from_iterable((self.config.users, self.config.superusers))) user_cache.cleanup_cache(usernames) user_cache.check(sorted(usernames)) + marked_users = user_cache.warning_users - if user_cache.warning_detected: - logger.error("Stop to set permissions - Update the configuration first.") - if self.enforce: - return - else: - user_cache.save_if_changed() - - default_user = self.config.users['__default'] + user_cache.save_if_changed() for i in sorted(User.objects.all(),key=lambda i:i.username.lower()): changed = False @@ -562,7 +556,11 @@ class WeblatePermission: changed = True if not self.check_func(i,'is_active', e.get('is_active'), f"Wrong is_active status for {i.username}", self.logfunc, False): changed = True + if changed and self.enforce: + if i.username in marked_users: + logger.info(f"{i.username} is marked (the issues are listed above): skip to save user.") + continue auditlogger.info(f"Save user({i.username})") i.save() -- GitLab