From 2315e811fa1f418a832ca9b8f732aeb23c18e4b7 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 30 Nov 2020 10:58:50 +0100 Subject: [PATCH 01/40] [wip] Import users with a CSV --- data/actionsmap/yunohost.yml | 17 +++++++++++++++++ src/yunohost/user.py | 11 +++++++++++ 2 files changed, 28 insertions(+) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index b2f5a349b..075e429ec 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -206,6 +206,23 @@ user: arguments: username: help: Username or email to get information + + ### user_import() + import: + action_help: Import several users from CSV + api: POST /users/import + arguments: + csv: + help: "CSV file with columns username, email, quota, groups(separated by coma) and optionally password" + type: open + -u: + full: --update + help: Update all existing users contained in the csv file (by default those users are ignored) + action: store_true + -d: + full: --delete + help: Delete all existing users that are not contained in the csv file (by default those users are ignored) + action: store_true subcategories: group: diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 266c2774c..7b920b8a9 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -566,6 +566,17 @@ def user_info(username): return result_dict +def user_import(csv, update=False, delete=False): + """ + Import users from CSV + + Keyword argument: + csv -- CSV file with columns username, email, quota, groups and optionnally password + + """ + logger.warning(type(csv)) + return {} + # # Group subcategory # From fdc2337e0f9ccc624ac1d293d98f5714ee8a844e Mon Sep 17 00:00:00 2001 From: ljf Date: Thu, 3 Dec 2020 18:04:09 +0100 Subject: [PATCH 02/40] [wip] Import users by csv --- src/yunohost/user.py | 95 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 3 deletions(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 7b920b8a9..a9010c060 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -566,7 +566,8 @@ def user_info(username): return result_dict -def user_import(csv, update=False, delete=False): +@is_unit_operation() +def user_import(operation_logger, csv, update=False, delete=False): """ Import users from CSV @@ -574,8 +575,96 @@ def user_import(csv, update=False, delete=False): csv -- CSV file with columns username, email, quota, groups and optionnally password """ - logger.warning(type(csv)) - return {} + import csv # CSV are needed only in this function + + # Prepare what should be done + actions = { + 'created': [], + 'updated': [], + 'deleted': [] + } + is_well_formatted = True + + existing_users = user_list()['users'].keys() + reader = csv.DictReader(csv, delimiter=';', quotechar='"') + for user in reader: + if user['username']:#TODO better check + logger.error(m18n.n('user_import_bad_line', line=reader.line_num)) + is_well_formatted = False + continue + + if user['username'] not in existing_users: + actions['created'].append(user) + else: + if update: + actions['updated'].append(user) + existing_users.remove(user['username']) + + if delete: + for user in existing_users: + actions['deleted'].append(user) + + if not is_well_formatted: + raise YunohostError('user_import_bad_file') + + total = len(actions['created'] + actions['updated'] + actions['deleted']) + + # Apply creation, update and deletion operation + result = { + 'created': 0, + 'updated': 0, + 'deleted': 0, + 'errors': 0 + } + + if total == 0: + logger.info(m18n.n('nothing_to_do')) + return + + def on_failure(user, exception): + result['errors'] += 1 + logger.error(user + ': ' + str(exception)) + + operation_logger.start() + for user in actions['created']: + try: + user_create(operation_logger, user['username'], + user['firstname'], user['lastname'], + user['domain'], user['password'], + user['mailbox_quota'], user['mail']) + result['created'] += 1 + except Exception as e: + on_failure(user['username'], e) + + if update: + for user in actions['updated']: + try: + user_update(operation_logger, user['username'], + user['firstname'], user['lastname'], + user['mail'], user['password'], + mailbox_quota=user['mailbox_quota']) + result['updated'] += 1 + except Exception as e: + on_failure(user['username'], e) + + if delete: + for user in actions['deleted']: + try: + user_delete(operation_logger, user, purge=True) + result['deleted'] += 1 + except Exception as e: + on_failure(user, e) + + if result['errors']: + msg = m18n.n('user_import_partial_failed') + if result['created'] + result['updated'] + result['deleted'] == 0: + msg = m18n.n('user_import_failed') + logger.error(msg) + operation_logger.error(msg) + else: + logger.success(m18n.n('user_import_success')) + operation_logger.success() + return result # # Group subcategory From 2ae0ec46f44a55eb98dbdca90711ab6708912604 Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 8 Dec 2020 16:47:28 +0100 Subject: [PATCH 03/40] [wip] Import users from csv --- src/yunohost/user.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index a9010c060..7745ec56a 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -588,7 +588,7 @@ def user_import(operation_logger, csv, update=False, delete=False): existing_users = user_list()['users'].keys() reader = csv.DictReader(csv, delimiter=';', quotechar='"') for user in reader: - if user['username']:#TODO better check + if re.match(r'^[a-z0-9_]+$', user['username']:#TODO better check logger.error(m18n.n('user_import_bad_line', line=reader.line_num)) is_well_formatted = False continue @@ -636,6 +636,7 @@ def user_import(operation_logger, csv, update=False, delete=False): except Exception as e: on_failure(user['username'], e) +<<<<<<< Updated upstream if update: for user in actions['updated']: try: @@ -654,6 +655,24 @@ def user_import(operation_logger, csv, update=False, delete=False): result['deleted'] += 1 except Exception as e: on_failure(user, e) +======= + for user in actions['updated']: + try: + user_update(operation_logger, user['username'], + user['firstname'], user['lastname'], + user['mail'], user['password'], + mailbox_quota=user['mailbox_quota']) + result['updated'] += 1 + except Exception as e: + on_failure(user['username'], e) + + for user in actions['deleted']: + try: + user_delete(operation_logger, user, purge=True) + result['deleted'] += 1 + except Exception as e: + on_failure(user, e) +>>>>>>> Stashed changes if result['errors']: msg = m18n.n('user_import_partial_failed') From 3e047c4b943881c1e8a14311006773cc583893bb Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 13 Dec 2020 03:24:18 +0100 Subject: [PATCH 04/40] [fix] CSV import --- data/actionsmap/yunohost.yml | 2 +- locales/en.json | 5 + src/yunohost/log.py | 3 + src/yunohost/user.py | 265 ++++++++++++++++++++++------------- 4 files changed, 173 insertions(+), 102 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 075e429ec..a3ff431e7 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -212,7 +212,7 @@ user: action_help: Import several users from CSV api: POST /users/import arguments: - csv: + csvfile: help: "CSV file with columns username, email, quota, groups(separated by coma) and optionally password" type: open -u: diff --git a/locales/en.json b/locales/en.json index 938a38e20..367183a8a 100644 --- a/locales/en.json +++ b/locales/en.json @@ -400,6 +400,7 @@ "log_regen_conf": "Regenerate system configurations '{}'", "log_user_create": "Add '{}' user", "log_user_delete": "Delete '{}' user", + "log_user_import": "Import users", "log_user_group_create": "Create '{}' group", "log_user_group_delete": "Delete '{}' group", "log_user_group_update": "Update '{}' group", @@ -630,6 +631,10 @@ "user_unknown": "Unknown user: {user:s}", "user_update_failed": "Could not update user {user}: {error}", "user_updated": "User info changed", + "user_import_bad_line": "Incorrect line {line}: {details} ", + "user_import_partial_failed": "The users import operation partially failed", + "user_import_failed": "The users import operation completely failed", + "user_import_success": "Users have been imported", "yunohost_already_installed": "YunoHost is already installed", "yunohost_configured": "YunoHost is now configured", "yunohost_installing": "Installing YunoHost...", diff --git a/src/yunohost/log.py b/src/yunohost/log.py index f8da40002..9ea2c2024 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -371,6 +371,9 @@ def is_unit_operation( for field in exclude: if field in context: context.pop(field, None) + for field, value in context.items(): + if isinstance(value, file): + context[field] = value.name operation_logger = OperationLogger(op_key, related_to, args=context) try: diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 7745ec56a..0489a34fa 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -99,6 +99,7 @@ def user_create( password, mailbox_quota="0", mail=None, + imported=False ): from yunohost.domain import domain_list, _get_maindomain @@ -167,7 +168,8 @@ def user_create( if mail in aliases: raise YunohostValidationError("mail_unavailable") - operation_logger.start() + if not imported: + operation_logger.start() # Get random UID/GID all_uid = {str(x.pw_uid) for x in pwd.getpwall()} @@ -247,13 +249,14 @@ def user_create( hook_callback("post_user_create", args=[username, mail], env=env_dict) # TODO: Send a welcome mail to user - logger.success(m18n.n("user_created")) + if not imported: + logger.success(m18n.n('user_created')) return {"fullname": fullname, "username": username, "mail": mail} -@is_unit_operation([("username", "user")]) -def user_delete(operation_logger, username, purge=False): +@is_unit_operation([('username', 'user')]) +def user_delete(operation_logger, username, purge=False, imported=False): """ Delete user @@ -268,7 +271,8 @@ def user_delete(operation_logger, username, purge=False): if username not in user_list()["users"]: raise YunohostValidationError("user_unknown", user=username) - operation_logger.start() + if not imported: + operation_logger.start() user_group_update("all_users", remove=username, force=True, sync_perm=False) for group, infos in user_group_list()["groups"].items(): @@ -295,13 +299,13 @@ def user_delete(operation_logger, username, purge=False): subprocess.call(["nscd", "-i", "passwd"]) if purge: - subprocess.call(["rm", "-rf", "/home/{0}".format(username)]) - subprocess.call(["rm", "-rf", "/var/mail/{0}".format(username)]) + subprocess.call(['rm', '-rf', '/home/{0}'.format(username)]) + subprocess.call(['rm', '-rf', '/var/mail/{0}'.format(username)]) - hook_callback("post_user_delete", args=[username, purge]) - - logger.success(m18n.n("user_deleted")) + hook_callback('post_user_delete', args=[username, purge]) + if not imported: + logger.success(m18n.n('user_deleted')) @is_unit_operation([("username", "user")], exclude=["change_password"]) def user_update( @@ -316,6 +320,7 @@ def user_update( add_mailalias=None, remove_mailalias=None, mailbox_quota=None, + imported=False ): """ Update user informations @@ -394,34 +399,38 @@ def user_update( "admin@" + main_domain, "webmaster@" + main_domain, "postmaster@" + main_domain, + 'abuse@' + main_domain, ] - try: - ldap.validate_uniqueness({"mail": mail}) - except Exception as e: - raise YunohostValidationError("user_update_failed", user=username, error=e) - if mail[mail.find("@") + 1 :] not in domains: - raise YunohostValidationError( - "mail_domain_unknown", domain=mail[mail.find("@") + 1 :] - ) + if mail in user['mail']: + user['mail'].remove(mail) + else: + try: + ldap.validate_uniqueness({'mail': mail}) + except Exception as e: + raise YunohostError('user_update_failed', user=username, error=e) + if mail[mail.find('@') + 1:] not in domains: + raise YunohostError('mail_domain_unknown', domain=mail[mail.find('@') + 1:]) if mail in aliases: raise YunohostValidationError("mail_unavailable") - del user["mail"][0] - new_attr_dict["mail"] = [mail] + user["mail"] + new_attr_dict['mail'] = [mail] + user['mail'][1:] if add_mailalias: if not isinstance(add_mailalias, list): add_mailalias = [add_mailalias] for mail in add_mailalias: - try: - ldap.validate_uniqueness({"mail": mail}) - except Exception as e: - raise YunohostValidationError( - "user_update_failed", user=username, error=e - ) - if mail[mail.find("@") + 1 :] not in domains: - raise YunohostValidationError( - "mail_domain_unknown", domain=mail[mail.find("@") + 1 :] + if mail in user["mail"]: + user["mail"].remove(mail) + else: + try: + ldap.validate_uniqueness({"mail": mail}) + except Exception as e: + raise YunohostError( + "user_update_failed", user=username, error=e + ) + if mail[mail.find("@") + 1:] not in domains: + raise YunohostError( + "mail_domain_unknown", domain=mail[mail.find("@") + 1:] ) user["mail"].append(mail) new_attr_dict["mail"] = user["mail"] @@ -465,7 +474,8 @@ def user_update( new_attr_dict["mailuserquota"] = [mailbox_quota] env_dict["YNH_USER_MAILQUOTA"] = mailbox_quota - operation_logger.start() + if not imported: + operation_logger.start() try: ldap.update("uid=%s,ou=users" % username, new_attr_dict) @@ -475,9 +485,10 @@ def user_update( # Trigger post_user_update hooks hook_callback("post_user_update", env=env_dict) - logger.success(m18n.n("user_updated")) - app_ssowatconf() - return user_info(username) + if not imported: + app_ssowatconf() + logger.success(m18n.n('user_updated')) + return user_info(username) def user_info(username): @@ -507,11 +518,13 @@ def user_info(username): raise YunohostValidationError("user_unknown", user=username) result_dict = { - "username": user["uid"][0], - "fullname": user["cn"][0], - "firstname": user["givenName"][0], - "lastname": user["sn"][0], - "mail": user["mail"][0], + 'username': user['uid'][0], + 'fullname': user['cn'][0], + 'firstname': user['givenName'][0], + 'lastname': user['sn'][0], + 'mail': user['mail'][0], + 'mail-aliases': [], + 'mail-forward': [] } if len(user["mail"]) > 1: @@ -567,7 +580,7 @@ def user_info(username): @is_unit_operation() -def user_import(operation_logger, csv, update=False, delete=False): +def user_import(operation_logger, csvfile, update=False, delete=False): """ Import users from CSV @@ -576,24 +589,51 @@ def user_import(operation_logger, csv, update=False, delete=False): """ import csv # CSV are needed only in this function - - # Prepare what should be done + from moulinette.utils.text import random_ascii + from yunohost.permission import permission_sync_to_user + from yunohost.app import app_ssowatconf + # Pre-validate data and prepare what should be done actions = { 'created': [], 'updated': [], 'deleted': [] } is_well_formatted = True - + validators = { + 'username': r'^[a-z0-9_]+$', + 'firstname': r'^([^\W\d_]{2,30}[ ,.\'-]{0,3})+$', #FIXME Merge first and lastname and support more name (arabish, chinese...) + 'lastname': r'^([^\W\d_]{2,30}[ ,.\'-]{0,3})+$', + 'password': r'^|(.{3,})$', + 'mailbox_quota': r'^(\d+[bkMGT])|0$', + 'mail': r'^([\w.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}))$', + 'alias': r'^|([\w.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}),?)+$', + 'forward': r'^|([\w\+.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}),?)+$', + 'groups': r'^|([a-z0-9_]+(,?[a-z0-9_]+)*)$' + } + def to_list(str_list): + return str_list.split(',') if str_list else [] existing_users = user_list()['users'].keys() - reader = csv.DictReader(csv, delimiter=';', quotechar='"') + reader = csv.DictReader(csvfile, delimiter=';', quotechar='"') for user in reader: - if re.match(r'^[a-z0-9_]+$', user['username']:#TODO better check - logger.error(m18n.n('user_import_bad_line', line=reader.line_num)) + format_errors = [key + ':' + user[key] + for key, validator in validators.items() + if not re.match(validator, user[key])] + if format_errors: + logger.error(m18n.n('user_import_bad_line', + line=reader.line_num, + details=', '.join(format_errors))) is_well_formatted = False continue + user['groups'] = to_list(user['groups']) + user['alias'] = to_list(user['alias']) + user['forward'] = to_list(user['forward']) + user['domain'] = user['mail'].split('@')[1] if user['username'] not in existing_users: + # Generate password if not exists + # This could be used when reset password will be merged + if not user['password']: + user['password'] = random_ascii(70) actions['created'].append(user) else: if update: @@ -609,6 +649,10 @@ def user_import(operation_logger, csv, update=False, delete=False): total = len(actions['created'] + actions['updated'] + actions['deleted']) + if total == 0: + logger.info(m18n.n('nothing_to_do')) + return + # Apply creation, update and deletion operation result = { 'created': 0, @@ -617,62 +661,71 @@ def user_import(operation_logger, csv, update=False, delete=False): 'errors': 0 } - if total == 0: - logger.info(m18n.n('nothing_to_do')) - return - def on_failure(user, exception): result['errors'] += 1 logger.error(user + ': ' + str(exception)) + def update(user, created=False): + remove_alias = None + remove_forward = None + if not created: + info = user_info(user['username']) + user['mail'] = None if info['mail'] == user['mail'] else user['mail'] + remove_alias = list(set(info['mail-aliases']) - set(user['alias'])) + remove_forward = list(set(info['mail-forward']) - set(user['forward'])) + user['alias'] = list(set(user['alias']) - set(info['mail-aliases'])) + user['forward'] = list(set(user['forward']) - set(info['mail-forward'])) + for group, infos in user_group_list()["groups"].items(): + if group == "all_users": + continue + # If the user is in this group (and it's not the primary group), + # remove the member from the group + if user['username'] != group and user['username'] in infos["members"]: + user_group_update(group, remove=user['username'], sync_perm=False, imported=True) + + user_update(user['username'], + user['firstname'], user['lastname'], + user['mail'], user['password'], + mailbox_quota=user['mailbox_quota'], + mail=user['mail'], add_mailalias=user['alias'], + remove_mailalias=remove_alias, + remove_mailforward=remove_forward, + add_mailforward=user['forward'], imported=True) + + for group in user['groups']: + user_group_update(group, add=user['username'], sync_perm=False, imported=True) + operation_logger.start() - for user in actions['created']: - try: - user_create(operation_logger, user['username'], - user['firstname'], user['lastname'], - user['domain'], user['password'], - user['mailbox_quota'], user['mail']) - result['created'] += 1 - except Exception as e: - on_failure(user['username'], e) - -<<<<<<< Updated upstream - if update: - for user in actions['updated']: - try: - user_update(operation_logger, user['username'], - user['firstname'], user['lastname'], - user['mail'], user['password'], - mailbox_quota=user['mailbox_quota']) - result['updated'] += 1 - except Exception as e: - on_failure(user['username'], e) - - if delete: - for user in actions['deleted']: - try: - user_delete(operation_logger, user, purge=True) - result['deleted'] += 1 - except Exception as e: - on_failure(user, e) -======= - for user in actions['updated']: - try: - user_update(operation_logger, user['username'], - user['firstname'], user['lastname'], - user['mail'], user['password'], - mailbox_quota=user['mailbox_quota']) - result['updated'] += 1 - except Exception as e: - on_failure(user['username'], e) - + # We do delete and update before to avoid mail uniqueness issues for user in actions['deleted']: try: - user_delete(operation_logger, user, purge=True) + user_delete(user, purge=True, imported=True) result['deleted'] += 1 - except Exception as e: + except YunohostError as e: on_failure(user, e) ->>>>>>> Stashed changes + + for user in actions['updated']: + try: + update(user) + result['updated'] += 1 + except YunohostError as e: + on_failure(user['username'], e) + + for user in actions['created']: + try: + user_create(user['username'], + user['firstname'], user['lastname'], + user['domain'], user['password'], + user['mailbox_quota'], imported=True) + update(user, created=True) + result['created'] += 1 + except YunohostError as e: + on_failure(user['username'], e) + + + + permission_sync_to_user() + app_ssowatconf() if result['errors']: msg = m18n.n('user_import_partial_failed') @@ -685,6 +738,7 @@ def user_import(operation_logger, csv, update=False, delete=False): operation_logger.success() return result + # # Group subcategory # @@ -857,9 +911,15 @@ def user_group_delete(operation_logger, groupname, force=False, sync_perm=True): logger.debug(m18n.n("group_deleted", group=groupname)) -@is_unit_operation([("groupname", "group")]) +@is_unit_operation([('groupname', 'group')]) def user_group_update( - operation_logger, groupname, add=None, remove=None, force=False, sync_perm=True + operation_logger, + groupname, + add=None, + remove=None, + force=False, + sync_perm=True, + imported=False ): """ Update user informations @@ -929,7 +989,8 @@ def user_group_update( ] if set(new_group) != set(current_group): - operation_logger.start() + if not imported: + operation_logger.start() ldap = _get_ldap_interface() try: ldap.update( @@ -939,14 +1000,16 @@ def user_group_update( except Exception as e: raise YunohostError("group_update_failed", group=groupname, error=e) - if groupname != "all_users": - logger.success(m18n.n("group_updated", group=groupname)) - else: - logger.debug(m18n.n("group_updated", group=groupname)) - if sync_perm: permission_sync_to_user() - return user_group_info(groupname) + + if not imported: + if groupname != "all_users": + logger.success(m18n.n("group_updated", group=groupname)) + else: + logger.debug(m18n.n("group_updated", group=groupname)) + + return user_group_info(groupname) def user_group_info(groupname): From 9e2f4a56f33ded262d6beb2dcf260e194807f905 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 20 Dec 2020 23:13:22 +0100 Subject: [PATCH 05/40] [enh] Add export feature and refactor user_list --- data/actionsmap/yunohost.yml | 7 +- locales/en.json | 5 +- src/yunohost/user.py | 164 +++++++++++++++++++++++++---------- 3 files changed, 129 insertions(+), 47 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index a3ff431e7..a5fdf5872 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -67,7 +67,7 @@ user: api: GET /users arguments: --fields: - help: fields to fetch + help: fields to fetch (username, fullname, mail, mail-alias, mail-forward, mailbox-quota, groups, shell, home-path) nargs: "+" ### user_create() @@ -207,6 +207,11 @@ user: username: help: Username or email to get information + ### user_export() + export: + action_help: Export users into CSV + api: GET /users/export + ### user_import() import: action_help: Import several users from CSV diff --git a/locales/en.json b/locales/en.json index 367183a8a..6a092d108 100644 --- a/locales/en.json +++ b/locales/en.json @@ -631,9 +631,12 @@ "user_unknown": "Unknown user: {user:s}", "user_update_failed": "Could not update user {user}: {error}", "user_updated": "User info changed", - "user_import_bad_line": "Incorrect line {line}: {details} ", + "user_import_bad_line": "Incorrect line {line}: {details}", + "user_import_bad_file": "Your CSV file is not correctly formatted it will be ignored to avoid potential data loss", + "user_import_missing_column": "The column {column} is missing", "user_import_partial_failed": "The users import operation partially failed", "user_import_failed": "The users import operation completely failed", + "user_import_nothing_to_do": "No user needs to be imported", "user_import_success": "Users have been imported", "yunohost_already_installed": "YunoHost is already installed", "yunohost_configured": "YunoHost is now configured", diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 0489a34fa..0fae9cf43 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -48,27 +48,48 @@ def user_list(fields=None): from yunohost.utils.ldap import _get_ldap_interface - user_attrs = { - "uid": "username", - "cn": "fullname", - "mail": "mail", - "maildrop": "mail-forward", - "homeDirectory": "home_path", - "mailuserquota": "mailbox-quota", + ldap_attrs = { + 'username': 'uid', + 'password': 'uid', + 'fullname': 'cn', + 'firstname': 'givenName', + 'lastname': 'sn', + 'mail': 'mail', + 'mail-alias': 'mail', + 'mail-forward': 'maildrop', + 'mailbox-quota': 'mailuserquota', + 'groups': 'memberOf', + 'shell': 'loginShell', + 'home-path': 'homeDirectory' } - attrs = ["uid"] + def display_default(values, _): + return values[0] if len(values) == 1 else values + + display = { + 'password': lambda values, user: '', + 'mail': lambda values, user: display_default(values[:1], user), + 'mail-alias': lambda values, _: values[1:], + 'mail-forward': lambda values, user: [forward for forward in values if forward != user['uid'][0]], + 'groups': lambda values, user: [ + group[3:].split(',')[0] + for group in values + if not group.startswith('cn=all_users,') and + not group.startswith('cn=' + user['uid'][0] + ',')], + 'shell': lambda values, _: len(values) > 0 and values[0].strip() == "/bin/false" + } + + attrs = set(['uid']) users = {} - if fields: - keys = user_attrs.keys() - for attr in fields: - if attr in keys: - attrs.append(attr) - else: - raise YunohostError("field_invalid", attr) - else: - attrs = ["uid", "cn", "mail", "mailuserquota"] + if not fields: + fields = ['username', 'fullname', 'mail', 'mailbox-quota', 'shell'] + + for field in fields: + if field in ldap_attrs: + attrs|=set([ldap_attrs[field]]) + else: + raise YunohostError('field_invalid', field) ldap = _get_ldap_interface() result = ldap.search( @@ -79,12 +100,13 @@ def user_list(fields=None): for user in result: entry = {} - for attr, values in user.items(): - if values: - entry[user_attrs[attr]] = values[0] + for field in fields: + values = [] + if ldap_attrs[field] in user: + values = user[ldap_attrs[field]] + entry[field] = display.get(field, display_default)(values, user) - uid = entry[user_attrs["uid"]] - users[uid] = entry + users[entry['username']] = entry return {"users": users} @@ -579,13 +601,49 @@ def user_info(username): return result_dict +def user_export(): + """ + Export users into CSV + + Keyword argument: + csv -- CSV file with columns username;firstname;lastname;password;mailbox_quota;mail;alias;forward;groups + + """ + import csv # CSV are needed only in this function + from io import BytesIO + fieldnames = [u'username', u'firstname', u'lastname', u'password', u'mailbox-quota', u'mail', u'mail-alias', u'mail-forward', u'groups'] + with BytesIO() as csv_io: + writer = csv.DictWriter(csv_io, fieldnames, delimiter=';', quotechar='"') + writer.writeheader() + users = user_list(fieldnames)['users'] + for username, user in users.items(): + user['mail-alias'] = ','.join(user['mail-alias']) + user['mail-forward'] = ','.join(user['mail-forward']) + user['groups'] = ','.join(user['groups']) + writer.writerow(user) + + body = csv_io.getvalue() + if msettings.get('interface') == 'api': + # We return a raw bottle HTTPresponse (instead of serializable data like + # list/dict, ...), which is gonna be picked and used directly by moulinette + from bottle import LocalResponse + response = LocalResponse(body=body, + headers={ + "Content-Disposition": "attachment; filename='users.csv'", + "Content-Type": "text/csv", + } + ) + return response + else: + return body + @is_unit_operation() def user_import(operation_logger, csvfile, update=False, delete=False): """ Import users from CSV Keyword argument: - csv -- CSV file with columns username, email, quota, groups and optionnally password + csv -- CSV file with columns username;firstname;lastname;password;mailbox_quota;mail;alias;forward;groups """ import csv # CSV are needed only in this function @@ -604,20 +662,35 @@ def user_import(operation_logger, csvfile, update=False, delete=False): 'firstname': r'^([^\W\d_]{2,30}[ ,.\'-]{0,3})+$', #FIXME Merge first and lastname and support more name (arabish, chinese...) 'lastname': r'^([^\W\d_]{2,30}[ ,.\'-]{0,3})+$', 'password': r'^|(.{3,})$', - 'mailbox_quota': r'^(\d+[bkMGT])|0$', 'mail': r'^([\w.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}))$', - 'alias': r'^|([\w.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}),?)+$', - 'forward': r'^|([\w\+.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}),?)+$', + 'mail-alias': r'^|([\w.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}),?)+$', + 'mail-forward': r'^|([\w\+.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}),?)+$', + 'mailbox-quota': r'^(\d+[bkMGT])|0$', 'groups': r'^|([a-z0-9_]+(,?[a-z0-9_]+)*)$' } + def to_list(str_list): return str_list.split(',') if str_list else [] - existing_users = user_list()['users'].keys() + + existing_users = user_list()['users'] + past_lines = [] reader = csv.DictReader(csvfile, delimiter=';', quotechar='"') for user in reader: - format_errors = [key + ':' + user[key] - for key, validator in validators.items() - if not re.match(validator, user[key])] + # Validation + try: + format_errors = [key + ':' + str(user[key]) + for key, validator in validators.items() + if user[key] is None or not re.match(validator, user[key])] + except KeyError, e: + logger.error(m18n.n('user_import_missing_column', + column=str(e))) + is_well_formatted = False + break + + if 'username' in user: + if user['username'] in past_lines: + format_errors.append('username: %s (duplicated)' % user['username']) + past_lines.append(user['username']) if format_errors: logger.error(m18n.n('user_import_bad_line', line=reader.line_num, @@ -625,9 +698,10 @@ def user_import(operation_logger, csvfile, update=False, delete=False): is_well_formatted = False continue + # Choose what to do with this line and prepare data user['groups'] = to_list(user['groups']) - user['alias'] = to_list(user['alias']) - user['forward'] = to_list(user['forward']) + user['mail-alias'] = to_list(user['mail-alias']) + user['mail-forward'] = to_list(user['mail-forward']) user['domain'] = user['mail'].split('@')[1] if user['username'] not in existing_users: # Generate password if not exists @@ -638,7 +712,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): else: if update: actions['updated'].append(user) - existing_users.remove(user['username']) + del existing_users[user['username']] if delete: for user in existing_users: @@ -650,7 +724,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): total = len(actions['created'] + actions['updated'] + actions['deleted']) if total == 0: - logger.info(m18n.n('nothing_to_do')) + logger.info(m18n.n('user_import_nothing_to_do')) return # Apply creation, update and deletion operation @@ -665,14 +739,13 @@ def user_import(operation_logger, csvfile, update=False, delete=False): result['errors'] += 1 logger.error(user + ': ' + str(exception)) - def update(user, created=False): + def update(user, info=False): remove_alias = None remove_forward = None - if not created: - info = user_info(user['username']) + if info: user['mail'] = None if info['mail'] == user['mail'] else user['mail'] - remove_alias = list(set(info['mail-aliases']) - set(user['alias'])) - remove_forward = list(set(info['mail-forward']) - set(user['forward'])) + remove_alias = list(set(info['mail-aliases']) - set(user['mail-alias'])) + remove_forward = list(set(info['mail-forward']) - set(user['mail-forward'])) user['alias'] = list(set(user['alias']) - set(info['mail-aliases'])) user['forward'] = list(set(user['forward']) - set(info['mail-forward'])) for group, infos in user_group_list()["groups"].items(): @@ -686,15 +759,16 @@ def user_import(operation_logger, csvfile, update=False, delete=False): user_update(user['username'], user['firstname'], user['lastname'], user['mail'], user['password'], - mailbox_quota=user['mailbox_quota'], - mail=user['mail'], add_mailalias=user['alias'], + mailbox_quota=user['mailbox-quota'], + mail=user['mail'], add_mailalias=user['mail-alias'], remove_mailalias=remove_alias, remove_mailforward=remove_forward, - add_mailforward=user['forward'], imported=True) + add_mailforward=user['mail-forward'], imported=True) for group in user['groups']: user_group_update(group, add=user['username'], sync_perm=False, imported=True) + users = user_list()['users'] operation_logger.start() # We do delete and update before to avoid mail uniqueness issues for user in actions['deleted']: @@ -706,7 +780,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): for user in actions['updated']: try: - update(user) + update(user, users[user['username']]) result['updated'] += 1 except YunohostError as e: on_failure(user['username'], e) @@ -716,8 +790,8 @@ def user_import(operation_logger, csvfile, update=False, delete=False): user_create(user['username'], user['firstname'], user['lastname'], user['domain'], user['password'], - user['mailbox_quota'], imported=True) - update(user, created=True) + user['mailbox-quota'], imported=True) + update(user) result['created'] += 1 except YunohostError as e: on_failure(user['username'], e) From fd06430e8f1a93f5b3236bbe0b48d9ca6e574b07 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 21 Dec 2020 02:29:17 +0100 Subject: [PATCH 06/40] [fix] Import user with update mode some unit test --- src/yunohost/tests/test_user-group.py | 64 +++++++++++++++++++++++++ src/yunohost/user.py | 69 ++++++++++++--------------- 2 files changed, 94 insertions(+), 39 deletions(-) diff --git a/src/yunohost/tests/test_user-group.py b/src/yunohost/tests/test_user-group.py index 251029796..e83425df9 100644 --- a/src/yunohost/tests/test_user-group.py +++ b/src/yunohost/tests/test_user-group.py @@ -1,5 +1,6 @@ import pytest +<<<<<<< HEAD from .conftest import message, raiseYunohostError from yunohost.user import ( @@ -8,6 +9,10 @@ from yunohost.user import ( user_create, user_delete, user_update, + user_import, + user_export, + CSV_FIELDNAMES, + FIRST_ALIASES, user_group_list, user_group_create, user_group_delete, @@ -110,6 +115,65 @@ def test_del_user(mocker): assert "alice" not in group_res["all_users"]["members"] +def test_import_user(mocker): + import csv + from io import BytesIO + fieldnames = [u'username', u'firstname', u'lastname', u'password', + u'mailbox-quota', u'mail', u'mail-alias', u'mail-forward', + u'groups'] + with BytesIO() as csv_io: + writer = csv.DictWriter(csv_io, fieldnames, delimiter=';', + quotechar='"') + writer.writeheader() + writer.writerow({ + 'username': "albert", + 'firstname': "Albert", + 'lastname': "Good", + 'password': "", + 'mailbox-quota': "1G", + 'mail': "albert@" + maindomain, + 'mail-alias': "albert2@" + maindomain, + 'mail-forward': "albert@example.com", + 'groups': "dev", + }) + writer.writerow({ + 'username': "alice", + 'firstname': "Alice", + 'lastname': "White", + 'password': "", + 'mailbox-quota': "1G", + 'mail': "alice@" + maindomain, + 'mail-alias': "alice1@" + maindomain + ",alice2@" + maindomain, + 'mail-forward': "", + 'groups': "apps", + }) + csv_io.seek(0) + with message(mocker, "user_import_success"): + user_import(csv_io, update=True, delete=True) + + group_res = user_group_list()['groups'] + user_res = user_list(CSV_FIELDNAMES)['users'] + assert "albert" in user_res + assert "alice" in user_res + assert "bob" not in user_res + assert len(user_res['alice']['mail-alias']) == 2 + assert "albert" in group_res['dev']['members'] + assert "alice" in group_res['apps']['members'] + + +def test_export_user(mocker): + result = user_export() + should_be = "username;firstname;lastname;password;" + should_be += "mailbox-quota;mail;mail-alias;mail-forward;groups" + should_be += "\r\nbob;Bob;Snow;;0;bob@" + maindomain + ";;;apps" + should_be += "\r\nalice;Alice;White;;0;alice@" + maindomain + ";" + should_be += ','.join([alias + maindomain for alias in FIRST_ALIASES]) + should_be += ";;dev" + should_be += "\r\njack;Jack;Black;;0;jack@" + maindomain + ";;;" + + assert result == should_be + + def test_create_group(mocker): with message(mocker, "group_created", group="adminsys"): diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 0fae9cf43..0bcce9cbc 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -43,6 +43,19 @@ from yunohost.log import is_unit_operation logger = getActionLogger("yunohost.user") +CSV_FIELDNAMES = [u'username', u'firstname', u'lastname', u'password', u'mailbox-quota', u'mail', u'mail-alias', u'mail-forward', u'groups'] +VALIDATORS = { + 'username': r'^[a-z0-9_]+$', + 'firstname': r'^([^\W\d_]{2,30}[ ,.\'-]{0,3})+$', #FIXME Merge first and lastname and support more name (arabish, chinese...) + 'lastname': r'^([^\W\d_]{2,30}[ ,.\'-]{0,3})+$', + 'password': r'^|(.{3,})$', + 'mail': r'^([\w.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}))$', + 'mail-alias': r'^|([\w.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}),?)+$', + 'mail-forward': r'^|([\w\+.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}),?)+$', + 'mailbox-quota': r'^(\d+[bkMGT])|0$', + 'groups': r'^|([a-z0-9_]+(,?[a-z0-9_]+)*)$' +} +FIRST_ALIASES = ['root@', 'admin@', 'webmaster@', 'postmaster@', 'abuse@'] def user_list(fields=None): @@ -87,7 +100,7 @@ def user_list(fields=None): for field in fields: if field in ldap_attrs: - attrs|=set([ldap_attrs[field]]) + attrs |= set([ldap_attrs[field]]) else: raise YunohostError('field_invalid', field) @@ -179,13 +192,7 @@ def user_create( raise YunohostValidationError("system_username_exists") main_domain = _get_maindomain() - aliases = [ - "root@" + main_domain, - "admin@" + main_domain, - "webmaster@" + main_domain, - "postmaster@" + main_domain, - "abuse@" + main_domain, - ] + aliases = [alias + main_domain for alias in FIRST_ALIASES] if mail in aliases: raise YunohostValidationError("mail_unavailable") @@ -416,13 +423,8 @@ def user_update( if mail: main_domain = _get_maindomain() - aliases = [ - "root@" + main_domain, - "admin@" + main_domain, - "webmaster@" + main_domain, - "postmaster@" + main_domain, - 'abuse@' + main_domain, - ] + aliases = [alias + main_domain for alias in FIRST_ALIASES] + if mail in user['mail']: user['mail'].remove(mail) else: @@ -606,23 +608,23 @@ def user_export(): Export users into CSV Keyword argument: - csv -- CSV file with columns username;firstname;lastname;password;mailbox_quota;mail;alias;forward;groups + csv -- CSV file with columns username;firstname;lastname;password;mailbox-quota;mail;mail-alias;mail-forward;groups """ - import csv # CSV are needed only in this function + import csv # CSV are needed only in this function from io import BytesIO - fieldnames = [u'username', u'firstname', u'lastname', u'password', u'mailbox-quota', u'mail', u'mail-alias', u'mail-forward', u'groups'] with BytesIO() as csv_io: - writer = csv.DictWriter(csv_io, fieldnames, delimiter=';', quotechar='"') + writer = csv.DictWriter(csv_io, CSV_FIELDNAMES, + delimiter=';', quotechar='"') writer.writeheader() - users = user_list(fieldnames)['users'] + users = user_list(CSV_FIELDNAMES)['users'] for username, user in users.items(): user['mail-alias'] = ','.join(user['mail-alias']) user['mail-forward'] = ','.join(user['mail-forward']) user['groups'] = ','.join(user['groups']) writer.writerow(user) - body = csv_io.getvalue() + body = csv_io.getvalue().rstrip() if msettings.get('interface') == 'api': # We return a raw bottle HTTPresponse (instead of serializable data like # list/dict, ...), which is gonna be picked and used directly by moulinette @@ -631,12 +633,12 @@ def user_export(): headers={ "Content-Disposition": "attachment; filename='users.csv'", "Content-Type": "text/csv", - } - ) + }) return response else: return body + @is_unit_operation() def user_import(operation_logger, csvfile, update=False, delete=False): """ @@ -657,17 +659,6 @@ def user_import(operation_logger, csvfile, update=False, delete=False): 'deleted': [] } is_well_formatted = True - validators = { - 'username': r'^[a-z0-9_]+$', - 'firstname': r'^([^\W\d_]{2,30}[ ,.\'-]{0,3})+$', #FIXME Merge first and lastname and support more name (arabish, chinese...) - 'lastname': r'^([^\W\d_]{2,30}[ ,.\'-]{0,3})+$', - 'password': r'^|(.{3,})$', - 'mail': r'^([\w.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}))$', - 'mail-alias': r'^|([\w.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}),?)+$', - 'mail-forward': r'^|([\w\+.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}),?)+$', - 'mailbox-quota': r'^(\d+[bkMGT])|0$', - 'groups': r'^|([a-z0-9_]+(,?[a-z0-9_]+)*)$' - } def to_list(str_list): return str_list.split(',') if str_list else [] @@ -679,7 +670,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): # Validation try: format_errors = [key + ':' + str(user[key]) - for key, validator in validators.items() + for key, validator in VALIDATORS.items() if user[key] is None or not re.match(validator, user[key])] except KeyError, e: logger.error(m18n.n('user_import_missing_column', @@ -744,10 +735,10 @@ def user_import(operation_logger, csvfile, update=False, delete=False): remove_forward = None if info: user['mail'] = None if info['mail'] == user['mail'] else user['mail'] - remove_alias = list(set(info['mail-aliases']) - set(user['mail-alias'])) + remove_alias = list(set(info['mail-alias']) - set(user['mail-alias'])) remove_forward = list(set(info['mail-forward']) - set(user['mail-forward'])) - user['alias'] = list(set(user['alias']) - set(info['mail-aliases'])) - user['forward'] = list(set(user['forward']) - set(info['mail-forward'])) + user['mail-alias'] = list(set(user['mail-alias']) - set(info['mail-alias'])) + user['mail-forward'] = list(set(user['mail-forward']) - set(info['mail-forward'])) for group, infos in user_group_list()["groups"].items(): if group == "all_users": continue @@ -768,7 +759,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): for group in user['groups']: user_group_update(group, add=user['username'], sync_perm=False, imported=True) - users = user_list()['users'] + users = user_list(CSV_FIELDNAMES)['users'] operation_logger.start() # We do delete and update before to avoid mail uniqueness issues for user in actions['deleted']: From 57dcf45a7c838d2fecb633a1f1f2f04929b305ce Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 21 Dec 2020 03:40:20 +0100 Subject: [PATCH 07/40] [fix] User import unit test --- src/yunohost/log.py | 3 +++ src/yunohost/tests/test_user-group.py | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index 9ea2c2024..a8a3281d2 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -32,6 +32,7 @@ import psutil from datetime import datetime, timedelta from logging import FileHandler, getLogger, Formatter +from io import IOBase from moulinette import m18n, msettings from moulinette.core import MoulinetteError @@ -374,6 +375,8 @@ def is_unit_operation( for field, value in context.items(): if isinstance(value, file): context[field] = value.name + elif isinstance(value, IOBase): + context[field] = 'IOBase' operation_logger = OperationLogger(op_key, related_to, args=context) try: diff --git a/src/yunohost/tests/test_user-group.py b/src/yunohost/tests/test_user-group.py index e83425df9..3252f0ef8 100644 --- a/src/yunohost/tests/test_user-group.py +++ b/src/yunohost/tests/test_user-group.py @@ -1,6 +1,5 @@ import pytest -<<<<<<< HEAD from .conftest import message, raiseYunohostError from yunohost.user import ( @@ -27,7 +26,7 @@ maindomain = "" def clean_user_groups(): for u in user_list()["users"]: - user_delete(u) + user_delete(u, purge=True) for g in user_group_list()["groups"]: if g not in ["all_users", "visitors"]: From c7c29285795c6823eaaa074e2f163af9136bb0a3 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 21 Dec 2020 04:27:03 +0100 Subject: [PATCH 08/40] [fix] Column list --- data/actionsmap/yunohost.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index a5fdf5872..4ef9ad008 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -218,7 +218,7 @@ user: api: POST /users/import arguments: csvfile: - help: "CSV file with columns username, email, quota, groups(separated by coma) and optionally password" + help: "CSV file with columns username, firstname, lastname, password, mail, mailbox-quota, mail-alias, mail-forward, groups (separated by coma)" type: open -u: full: --update From 59d7e2f247687143d3cc6fca462e5557e9e37621 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 21 Dec 2020 04:27:23 +0100 Subject: [PATCH 09/40] [fix] LDAP Size limits --- data/templates/slapd/slapd.ldif | 1 + 1 file changed, 1 insertion(+) diff --git a/data/templates/slapd/slapd.ldif b/data/templates/slapd/slapd.ldif index d3ed2e053..8692d2664 100644 --- a/data/templates/slapd/slapd.ldif +++ b/data/templates/slapd/slapd.ldif @@ -33,6 +33,7 @@ olcAuthzPolicy: none olcConcurrency: 0 olcConnMaxPending: 100 olcConnMaxPendingAuth: 1000 +olcSizeLimit: 10000 olcIdleTimeout: 0 olcIndexSubstrIfMaxLen: 4 olcIndexSubstrIfMinLen: 2 From 8e2f1c696b191b06d40a1646699d81f758aa1d6a Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 21 Dec 2020 04:27:50 +0100 Subject: [PATCH 10/40] [fix] Home not created --- locales/en.json | 2 +- src/yunohost/user.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/locales/en.json b/locales/en.json index 6a092d108..faa9e4556 100644 --- a/locales/en.json +++ b/locales/en.json @@ -627,7 +627,7 @@ "user_creation_failed": "Could not create user {user}: {error}", "user_deleted": "User deleted", "user_deletion_failed": "Could not delete user {user}: {error}", - "user_home_creation_failed": "Could not create 'home' folder for user", + "user_home_creation_failed": "Could not create '{home}' folder for user", "user_unknown": "Unknown user: {user:s}", "user_update_failed": "Could not update user {user}: {error}", "user_updated": "User info changed", diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 0bcce9cbc..0680af89d 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -254,6 +254,10 @@ def user_create( except subprocess.CalledProcessError: if not os.path.isdir("/home/{0}".format(username)): logger.warning(m18n.n("user_home_creation_failed"), exc_info=1) + home = '/home/{0}'.format(username) + if not os.path.isdir(home): + logger.warning(m18n.n('user_home_creation_failed', home=home), + exc_info=1) try: subprocess.check_call( @@ -726,6 +730,20 @@ def user_import(operation_logger, csvfile, update=False, delete=False): 'errors': 0 } + def progress(info=""): + progress.nb += 1 + width = 20 + bar = int(progress.nb * width / total) + bar = "[" + "#" * bar + "." * (width - bar) + "]" + if info: + bar += " > " + info + if progress.old == bar: + return + progress.old = bar + logger.info(bar) + progress.nb = 0 + progress.old = "" + def on_failure(user, exception): result['errors'] += 1 logger.error(user + ': ' + str(exception)) @@ -768,6 +786,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): result['deleted'] += 1 except YunohostError as e: on_failure(user, e) + progress("Deletion") for user in actions['updated']: try: @@ -775,6 +794,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): result['updated'] += 1 except YunohostError as e: on_failure(user['username'], e) + progress("Update") for user in actions['created']: try: @@ -786,6 +806,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): result['created'] += 1 except YunohostError as e: on_failure(user['username'], e) + progress("Creation") From a07314e66169b81efe270ee8018ee4d6f0629931 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 3 Jan 2021 19:44:46 +0100 Subject: [PATCH 11/40] [fix] Download CSV from webadmin - missing commit --- src/yunohost/user.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 0680af89d..88279997b 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -632,10 +632,10 @@ def user_export(): if msettings.get('interface') == 'api': # We return a raw bottle HTTPresponse (instead of serializable data like # list/dict, ...), which is gonna be picked and used directly by moulinette - from bottle import LocalResponse - response = LocalResponse(body=body, + from bottle import HTTPResponse + response = HTTPResponse(body=body, headers={ - "Content-Disposition": "attachment; filename='users.csv'", + "Content-Disposition": "attachment; filename=users.csv", "Content-Type": "text/csv", }) return response @@ -652,6 +652,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): csv -- CSV file with columns username;firstname;lastname;password;mailbox_quota;mail;alias;forward;groups """ + import csv # CSV are needed only in this function from moulinette.utils.text import random_ascii from yunohost.permission import permission_sync_to_user From a78e4c8eacca2d221b3b91693928437e0577ec05 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 3 Jan 2021 19:51:43 +0100 Subject: [PATCH 12/40] [fix] 1 letter firstname or lastname --- src/yunohost/user.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 88279997b..fe114da4b 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -46,8 +46,8 @@ logger = getActionLogger("yunohost.user") CSV_FIELDNAMES = [u'username', u'firstname', u'lastname', u'password', u'mailbox-quota', u'mail', u'mail-alias', u'mail-forward', u'groups'] VALIDATORS = { 'username': r'^[a-z0-9_]+$', - 'firstname': r'^([^\W\d_]{2,30}[ ,.\'-]{0,3})+$', #FIXME Merge first and lastname and support more name (arabish, chinese...) - 'lastname': r'^([^\W\d_]{2,30}[ ,.\'-]{0,3})+$', + 'firstname': r'^([^\W\d_]{1,30}[ ,.\'-]{0,3})+$', #FIXME Merge first and lastname and support more name (arabish, chinese...) + 'lastname': r'^([^\W\d_]{1,30}[ ,.\'-]{0,3})+$', 'password': r'^|(.{3,})$', 'mail': r'^([\w.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}))$', 'mail-alias': r'^|([\w.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}),?)+$', From 1d33f333cdff0ea7700b488a695ba8a3cd8b8b30 Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 8 May 2021 23:39:33 +0200 Subject: [PATCH 13/40] [fix] Python3 migration for export user feature --- src/yunohost/user.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index fe114da4b..5487ef18b 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -616,8 +616,8 @@ def user_export(): """ import csv # CSV are needed only in this function - from io import BytesIO - with BytesIO() as csv_io: + from io import StringIO + with StringIO() as csv_io: writer = csv.DictWriter(csv_io, CSV_FIELDNAMES, delimiter=';', quotechar='"') writer.writeheader() @@ -677,7 +677,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): format_errors = [key + ':' + str(user[key]) for key, validator in VALIDATORS.items() if user[key] is None or not re.match(validator, user[key])] - except KeyError, e: + except KeyError as e: logger.error(m18n.n('user_import_missing_column', column=str(e))) is_well_formatted = False From 91e7e5e1c80652a8977bba99d7a01075a1c86e2e Mon Sep 17 00:00:00 2001 From: ljf Date: Sat, 8 May 2021 23:58:36 +0200 Subject: [PATCH 14/40] [fix] Python3 migration: File args with log --- src/yunohost/log.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index a8a3281d2..9e6c8f165 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -373,10 +373,11 @@ def is_unit_operation( if field in context: context.pop(field, None) for field, value in context.items(): - if isinstance(value, file): - context[field] = value.name - elif isinstance(value, IOBase): - context[field] = 'IOBase' + if isinstance(value, IOBase): + try: + context[field] = value.name + except: + context[field] = 'IOBase' operation_logger = OperationLogger(op_key, related_to, args=context) try: From 6e880c8219846ce94a1a870c535334f16c5b1bcd Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 9 May 2021 01:02:51 +0200 Subject: [PATCH 15/40] [fix] Avoid password too small error during import operation --- src/yunohost/user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 5487ef18b..ee26533e8 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -413,7 +413,7 @@ def user_update( ] # change_password is None if user_update is not called to change the password - if change_password is not None: + if change_password is not None and change_password != '': # when in the cli interface if the option to change the password is called # without a specified value, change_password will be set to the const 0. # In this case we prompt for the new password. From ad73c29dad7d61db122436e75d394ee12bf70119 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 9 May 2021 01:08:43 +0200 Subject: [PATCH 16/40] [fix] Error in import user tests --- src/yunohost/tests/test_user-group.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/tests/test_user-group.py b/src/yunohost/tests/test_user-group.py index 3252f0ef8..ee5d07c40 100644 --- a/src/yunohost/tests/test_user-group.py +++ b/src/yunohost/tests/test_user-group.py @@ -116,7 +116,7 @@ def test_del_user(mocker): def test_import_user(mocker): import csv - from io import BytesIO + from io import StringIO fieldnames = [u'username', u'firstname', u'lastname', u'password', u'mailbox-quota', u'mail', u'mail-alias', u'mail-forward', u'groups'] From 8cf151668fda88b57e765e742e673bd6d1227efc Mon Sep 17 00:00:00 2001 From: ljf Date: Tue, 11 May 2021 12:08:00 +0200 Subject: [PATCH 17/40] [fix] CI test --- locales/de.json | 2 +- src/yunohost/tests/test_user-group.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/locales/de.json b/locales/de.json index 53297ed6d..73efca434 100644 --- a/locales/de.json +++ b/locales/de.json @@ -590,5 +590,5 @@ "diagnosis_sshd_config_insecure": "Die SSH-Konfiguration wurde scheinbar manuell abgeändert, und ist unsicher, weil sie keine 'AllowGroups'- oder 'AllowUsers' -Direktiven für die Begrenzung des Zugriffs durch autorisierte Benutzer enthält.", "backup_create_size_estimation": "Das Archiv wird etwa {size} Daten enthalten", "app_restore_script_failed": "Im Wiederherstellungsskript der Anwendung ist ein Fehler aufgetreten", - "app_restore_failed": "Konnte {apps:s} nicht wiederherstellen: {error:s}" + "app_restore_failed": "Konnte {app:s} nicht wiederherstellen: {error:s}" } diff --git a/src/yunohost/tests/test_user-group.py b/src/yunohost/tests/test_user-group.py index ee5d07c40..63d9a1930 100644 --- a/src/yunohost/tests/test_user-group.py +++ b/src/yunohost/tests/test_user-group.py @@ -120,7 +120,7 @@ def test_import_user(mocker): fieldnames = [u'username', u'firstname', u'lastname', u'password', u'mailbox-quota', u'mail', u'mail-alias', u'mail-forward', u'groups'] - with BytesIO() as csv_io: + with StringIO() as csv_io: writer = csv.DictWriter(csv_io, fieldnames, delimiter=';', quotechar='"') writer.writeheader() @@ -164,10 +164,10 @@ def test_export_user(mocker): result = user_export() should_be = "username;firstname;lastname;password;" should_be += "mailbox-quota;mail;mail-alias;mail-forward;groups" - should_be += "\r\nbob;Bob;Snow;;0;bob@" + maindomain + ";;;apps" should_be += "\r\nalice;Alice;White;;0;alice@" + maindomain + ";" should_be += ','.join([alias + maindomain for alias in FIRST_ALIASES]) should_be += ";;dev" + should_be += "\r\nbob;Bob;Snow;;0;bob@" + maindomain + ";;;apps" should_be += "\r\njack;Jack;Black;;0;jack@" + maindomain + ";;;" assert result == should_be From 4a22f6b390ec4277de478d50f49e9d7df8fa2773 Mon Sep 17 00:00:00 2001 From: ljf Date: Thu, 10 Jun 2021 19:28:49 +0200 Subject: [PATCH 18/40] [fix] yunohost user list --fields mail-alias --- src/yunohost/user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index ee26533e8..1037d417f 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -119,7 +119,7 @@ def user_list(fields=None): values = user[ldap_attrs[field]] entry[field] = display.get(field, display_default)(values, user) - users[entry['username']] = entry + users[user['uid'][0]] = entry return {"users": users} From b9e231241b53019a634bc0d43a80284732796611 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 13 Aug 2021 00:49:42 +0200 Subject: [PATCH 19/40] user imports: imported -> from_import, try to improve semantics --- src/yunohost/user.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 1037d417f..545d46a87 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -134,7 +134,7 @@ def user_create( password, mailbox_quota="0", mail=None, - imported=False + from_import=False ): from yunohost.domain import domain_list, _get_maindomain @@ -197,7 +197,7 @@ def user_create( if mail in aliases: raise YunohostValidationError("mail_unavailable") - if not imported: + if not from_import: operation_logger.start() # Get random UID/GID @@ -282,14 +282,14 @@ def user_create( hook_callback("post_user_create", args=[username, mail], env=env_dict) # TODO: Send a welcome mail to user - if not imported: + if not from_import: logger.success(m18n.n('user_created')) return {"fullname": fullname, "username": username, "mail": mail} @is_unit_operation([('username', 'user')]) -def user_delete(operation_logger, username, purge=False, imported=False): +def user_delete(operation_logger, username, purge=False, from_import=False): """ Delete user @@ -304,7 +304,7 @@ def user_delete(operation_logger, username, purge=False, imported=False): if username not in user_list()["users"]: raise YunohostValidationError("user_unknown", user=username) - if not imported: + if not from_import: operation_logger.start() user_group_update("all_users", remove=username, force=True, sync_perm=False) @@ -337,7 +337,7 @@ def user_delete(operation_logger, username, purge=False, imported=False): hook_callback('post_user_delete', args=[username, purge]) - if not imported: + if not from_import: logger.success(m18n.n('user_deleted')) @is_unit_operation([("username", "user")], exclude=["change_password"]) @@ -353,7 +353,7 @@ def user_update( add_mailalias=None, remove_mailalias=None, mailbox_quota=None, - imported=False + from_import=False ): """ Update user informations @@ -502,7 +502,7 @@ def user_update( new_attr_dict["mailuserquota"] = [mailbox_quota] env_dict["YNH_USER_MAILQUOTA"] = mailbox_quota - if not imported: + if not from_import: operation_logger.start() try: @@ -513,7 +513,7 @@ def user_update( # Trigger post_user_update hooks hook_callback("post_user_update", env=env_dict) - if not imported: + if not from_import: app_ssowatconf() logger.success(m18n.n('user_updated')) return user_info(username) @@ -764,7 +764,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): # If the user is in this group (and it's not the primary group), # remove the member from the group if user['username'] != group and user['username'] in infos["members"]: - user_group_update(group, remove=user['username'], sync_perm=False, imported=True) + user_group_update(group, remove=user['username'], sync_perm=False, from_import=True) user_update(user['username'], user['firstname'], user['lastname'], @@ -773,17 +773,17 @@ def user_import(operation_logger, csvfile, update=False, delete=False): mail=user['mail'], add_mailalias=user['mail-alias'], remove_mailalias=remove_alias, remove_mailforward=remove_forward, - add_mailforward=user['mail-forward'], imported=True) + add_mailforward=user['mail-forward'], from_import=True) for group in user['groups']: - user_group_update(group, add=user['username'], sync_perm=False, imported=True) + user_group_update(group, add=user['username'], sync_perm=False, from_import=True) users = user_list(CSV_FIELDNAMES)['users'] operation_logger.start() # We do delete and update before to avoid mail uniqueness issues for user in actions['deleted']: try: - user_delete(user, purge=True, imported=True) + user_delete(user, purge=True, from_import=True) result['deleted'] += 1 except YunohostError as e: on_failure(user, e) @@ -802,7 +802,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): user_create(user['username'], user['firstname'], user['lastname'], user['domain'], user['password'], - user['mailbox-quota'], imported=True) + user['mailbox-quota'], from_import=True) update(user) result['created'] += 1 except YunohostError as e: @@ -1006,7 +1006,7 @@ def user_group_update( remove=None, force=False, sync_perm=True, - imported=False + from_import=False ): """ Update user informations @@ -1076,7 +1076,7 @@ def user_group_update( ] if set(new_group) != set(current_group): - if not imported: + if not from_import: operation_logger.start() ldap = _get_ldap_interface() try: @@ -1090,7 +1090,7 @@ def user_group_update( if sync_perm: permission_sync_to_user() - if not imported: + if not from_import: if groupname != "all_users": logger.success(m18n.n("group_updated", group=groupname)) else: From 5c9fd158d9b31c83c4c9975e8f4609647f9fe17d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 13 Aug 2021 00:52:42 +0200 Subject: [PATCH 20/40] user imports: more attempts to improve semantics --- src/yunohost/user.py | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 545d46a87..dfa71708a 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -749,34 +749,34 @@ def user_import(operation_logger, csvfile, update=False, delete=False): result['errors'] += 1 logger.error(user + ': ' + str(exception)) - def update(user, info=False): + def update(new_infos, old_infos=False): remove_alias = None remove_forward = None if info: - user['mail'] = None if info['mail'] == user['mail'] else user['mail'] - remove_alias = list(set(info['mail-alias']) - set(user['mail-alias'])) - remove_forward = list(set(info['mail-forward']) - set(user['mail-forward'])) - user['mail-alias'] = list(set(user['mail-alias']) - set(info['mail-alias'])) - user['mail-forward'] = list(set(user['mail-forward']) - set(info['mail-forward'])) + new_infos['mail'] = None if old_infos['mail'] == new_infos['mail'] else new_infos['mail'] + remove_alias = list(set(old_infos['mail-alias']) - set(new_infos['mail-alias'])) + remove_forward = list(set(old_infos['mail-forward']) - set(new_infos['mail-forward'])) + new_infos['mail-alias'] = list(set(new_infos['mail-alias']) - set(old_infos['mail-alias'])) + new_infos['mail-forward'] = list(set(new_infos['mail-forward']) - set(old_infos['mail-forward'])) for group, infos in user_group_list()["groups"].items(): if group == "all_users": continue # If the user is in this group (and it's not the primary group), # remove the member from the group - if user['username'] != group and user['username'] in infos["members"]: - user_group_update(group, remove=user['username'], sync_perm=False, from_import=True) + if new_infos['username'] != group and new_infos['username'] in infos["members"]: + user_group_update(group, remove=new_infos['username'], sync_perm=False, from_import=True) - user_update(user['username'], - user['firstname'], user['lastname'], - user['mail'], user['password'], - mailbox_quota=user['mailbox-quota'], - mail=user['mail'], add_mailalias=user['mail-alias'], - remove_mailalias=remove_alias, - remove_mailforward=remove_forward, - add_mailforward=user['mail-forward'], from_import=True) + user_update(new_infos['username'], + new_infos['firstname'], new_infos['lastname'], + new_infos['mail'], new_infos['password'], + mailbox_quota=new_infos['mailbox-quota'], + mail=new_infos['mail'], add_mailalias=new_infos['mail-alias'], + remove_mailalias=remove_alias, + remove_mailforward=remove_forward, + add_mailforward=new_infos['mail-forward'], from_import=True) - for group in user['groups']: - user_group_update(group, add=user['username'], sync_perm=False, from_import=True) + for group in new_infos['groups']: + user_group_update(group, add=new_infos['username'], sync_perm=False, from_import=True) users = user_list(CSV_FIELDNAMES)['users'] operation_logger.start() @@ -809,8 +809,6 @@ def user_import(operation_logger, csvfile, update=False, delete=False): on_failure(user['username'], e) progress("Creation") - - permission_sync_to_user() app_ssowatconf() From b1102ba56e29f311f400648c9f0b6ff27394985a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 13 Aug 2021 01:10:59 +0200 Subject: [PATCH 21/40] user imports: misc formating, comments --- locales/en.json | 2 +- src/yunohost/user.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/locales/en.json b/locales/en.json index f8b26c0b1..5b93dbb9f 100644 --- a/locales/en.json +++ b/locales/en.json @@ -638,7 +638,7 @@ "user_import_partial_failed": "The users import operation partially failed", "user_import_failed": "The users import operation completely failed", "user_import_nothing_to_do": "No user needs to be imported", - "user_import_success": "Users have been imported", + "user_import_success": "Users successfully imported", "yunohost_already_installed": "YunoHost is already installed", "yunohost_configured": "YunoHost is now configured", "yunohost_installing": "Installing YunoHost...", diff --git a/src/yunohost/user.py b/src/yunohost/user.py index ca226f654..459b0decf 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -96,11 +96,11 @@ def user_list(fields=None): users = {} if not fields: - fields = ['username', 'fullname', 'mail', 'mailbox-quota', 'shell'] + fields = ['username', 'fullname', 'mail', 'mailbox-quota'] for field in fields: if field in ldap_attrs: - attrs |= set([ldap_attrs[field]]) + attrs.add(ldap_attrs[field]) else: raise YunohostError('field_invalid', field) @@ -252,7 +252,7 @@ def user_create( # Attempt to create user home folder subprocess.check_call(["mkhomedir_helper", username]) except subprocess.CalledProcessError: - home = '/home/{0}'.format(username) + home = f'/home/{username}' if not os.path.isdir(home): logger.warning(m18n.n('user_home_creation_failed', home=home), exc_info=1) @@ -427,8 +427,10 @@ def user_update( main_domain = _get_maindomain() aliases = [alias + main_domain for alias in FIRST_ALIASES] + # If the requested mail address is already as main address or as an alias by this user if mail in user['mail']: user['mail'].remove(mail) + # Othewise, check that this mail address is not already used by this user else: try: ldap.validate_uniqueness({'mail': mail}) @@ -445,6 +447,7 @@ def user_update( if not isinstance(add_mailalias, list): add_mailalias = [add_mailalias] for mail in add_mailalias: + # (c.f. similar stuff as before) if mail in user["mail"]: user["mail"].remove(mail) else: @@ -647,7 +650,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): Import users from CSV Keyword argument: - csv -- CSV file with columns username;firstname;lastname;password;mailbox_quota;mail;alias;forward;groups + csvfile -- CSV file with columns username;firstname;lastname;password;mailbox_quota;mail;alias;forward;groups """ @@ -655,6 +658,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): from moulinette.utils.text import random_ascii from yunohost.permission import permission_sync_to_user from yunohost.app import app_ssowatconf + # Pre-validate data and prepare what should be done actions = { 'created': [], From 781bc1cf7fdf3d3de8e1d7eddd9a9562d2149e39 Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Sat, 14 Aug 2021 16:38:21 +0200 Subject: [PATCH 22/40] Wording Co-authored-by: Alexandre Aubin --- data/actionsmap/yunohost.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 91e369dfe..64cef606d 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -222,7 +222,7 @@ user: type: open -u: full: --update - help: Update all existing users contained in the csv file (by default those users are ignored) + help: Update all existing users contained in the csv file (by default existing users are ignored) action: store_true -d: full: --delete From 2e745d2806dfbe416a03cd60197d3175106d3084 Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Sat, 14 Aug 2021 16:38:55 +0200 Subject: [PATCH 23/40] Wording Co-authored-by: Alexandre Aubin --- data/actionsmap/yunohost.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 64cef606d..817adeb92 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -226,7 +226,7 @@ user: action: store_true -d: full: --delete - help: Delete all existing users that are not contained in the csv file (by default those users are ignored) + help: Delete all existing users that are not contained in the csv file (by default existing users are kept) action: store_true subcategories: From 61bc676552a8b388beb5d49a4a0187c695b7482d Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Sat, 14 Aug 2021 16:41:58 +0200 Subject: [PATCH 24/40] [enh] Add a comment --- src/yunohost/log.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index 10ac097f5..f9f9334fb 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -372,6 +372,8 @@ def is_unit_operation( for field in exclude: if field in context: context.pop(field, None) + + # Manage file or stream for field, value in context.items(): if isinstance(value, IOBase): try: From 16f564622ee7fe2ff0ee8aa1b9916b1cc7d83e6b Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Sat, 14 Aug 2021 16:46:48 +0200 Subject: [PATCH 25/40] [enh] Remove uneeded comment --- src/yunohost/user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 459b0decf..11e82146a 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -46,7 +46,7 @@ logger = getActionLogger("yunohost.user") CSV_FIELDNAMES = [u'username', u'firstname', u'lastname', u'password', u'mailbox-quota', u'mail', u'mail-alias', u'mail-forward', u'groups'] VALIDATORS = { 'username': r'^[a-z0-9_]+$', - 'firstname': r'^([^\W\d_]{1,30}[ ,.\'-]{0,3})+$', #FIXME Merge first and lastname and support more name (arabish, chinese...) + 'firstname': r'^([^\W\d_]{1,30}[ ,.\'-]{0,3})+$', 'lastname': r'^([^\W\d_]{1,30}[ ,.\'-]{0,3})+$', 'password': r'^|(.{3,})$', 'mail': r'^([\w.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}))$', From efec34a3a654f5ec312c9dde03e9b13f7b05224d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 26 Aug 2021 20:31:20 +0200 Subject: [PATCH 26/40] tests: Improve code formatting --- src/yunohost/tests/test_user-group.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/yunohost/tests/test_user-group.py b/src/yunohost/tests/test_user-group.py index 63d9a1930..344a20fed 100644 --- a/src/yunohost/tests/test_user-group.py +++ b/src/yunohost/tests/test_user-group.py @@ -162,14 +162,13 @@ def test_import_user(mocker): def test_export_user(mocker): result = user_export() - should_be = "username;firstname;lastname;password;" - should_be += "mailbox-quota;mail;mail-alias;mail-forward;groups" - should_be += "\r\nalice;Alice;White;;0;alice@" + maindomain + ";" - should_be += ','.join([alias + maindomain for alias in FIRST_ALIASES]) - should_be += ";;dev" - should_be += "\r\nbob;Bob;Snow;;0;bob@" + maindomain + ";;;apps" - should_be += "\r\njack;Jack;Black;;0;jack@" + maindomain + ";;;" - + aliases = ','.join([alias + maindomain for alias in FIRST_ALIASES]) + should_be = ( + "username;firstname;lastname;password;mailbox-quota;mail;mail-alias;mail-forward;groups\r\n" + f"alice;Alice;White;;0;alice@{maindomain};{aliases};;dev\r\n" + f"bob;Bob;Snow;;0;bob@{maindomain};;;apps\r\n" + f"jack;Jack;Black;;0;jack@{maindomain};;;" + ) assert result == should_be From ad975a2dbb748b19f6abea17527452010ef30ba4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 26 Aug 2021 20:45:06 +0200 Subject: [PATCH 27/40] user import: clarify user deletion handling --- src/yunohost/user.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 11e82146a..076d930ca 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -670,6 +670,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): def to_list(str_list): return str_list.split(',') if str_list else [] + users_in_csv = [] existing_users = user_list()['users'] past_lines = [] reader = csv.DictReader(csvfile, delimiter=';', quotechar='"') @@ -701,20 +702,26 @@ def user_import(operation_logger, csvfile, update=False, delete=False): user['mail-alias'] = to_list(user['mail-alias']) user['mail-forward'] = to_list(user['mail-forward']) user['domain'] = user['mail'].split('@')[1] + + # User creation if user['username'] not in existing_users: # Generate password if not exists # This could be used when reset password will be merged if not user['password']: user['password'] = random_ascii(70) actions['created'].append(user) - else: - if update: - actions['updated'].append(user) - del existing_users[user['username']] + # User update + elif update: + actions['updated'].append(user) + + users_in_csv.add(user['username']) if delete: - for user in existing_users: - actions['deleted'].append(user) + actions['deleted'] = [user for user in existing_users if user not in users_in_csv] + + if delete and not users_in_csv: + logger.error("You used the delete option with an empty csv file ... You probably did not really mean to do that, did you !?") + is_well_formatted = False if not is_well_formatted: raise YunohostError('user_import_bad_file') From 128eb6a7d46f06c4bb2a0079134a20cf5d687b43 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 26 Aug 2021 21:06:16 +0200 Subject: [PATCH 28/40] user import: Clarify fields validation --- locales/en.json | 2 +- src/yunohost/tests/test_user-group.py | 4 +-- src/yunohost/user.py | 43 +++++++++++++-------------- 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/locales/en.json b/locales/en.json index 5b93dbb9f..06d2df0a4 100644 --- a/locales/en.json +++ b/locales/en.json @@ -634,7 +634,7 @@ "user_updated": "User info changed", "user_import_bad_line": "Incorrect line {line}: {details}", "user_import_bad_file": "Your CSV file is not correctly formatted it will be ignored to avoid potential data loss", - "user_import_missing_column": "The column {column} is missing", + "user_import_missing_columns": "The following columns are missing: {columns}", "user_import_partial_failed": "The users import operation partially failed", "user_import_failed": "The users import operation completely failed", "user_import_nothing_to_do": "No user needs to be imported", diff --git a/src/yunohost/tests/test_user-group.py b/src/yunohost/tests/test_user-group.py index 344a20fed..bbedfc27f 100644 --- a/src/yunohost/tests/test_user-group.py +++ b/src/yunohost/tests/test_user-group.py @@ -10,7 +10,7 @@ from yunohost.user import ( user_update, user_import, user_export, - CSV_FIELDNAMES, + FIELDS_FOR_IMPORT, FIRST_ALIASES, user_group_list, user_group_create, @@ -151,7 +151,7 @@ def test_import_user(mocker): user_import(csv_io, update=True, delete=True) group_res = user_group_list()['groups'] - user_res = user_list(CSV_FIELDNAMES)['users'] + user_res = user_list(list(FIELDS_FOR_IMPORT.keys()))['users'] assert "albert" in user_res assert "alice" in user_res assert "bob" not in user_res diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 076d930ca..b055d2cdb 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -43,8 +43,7 @@ from yunohost.log import is_unit_operation logger = getActionLogger("yunohost.user") -CSV_FIELDNAMES = [u'username', u'firstname', u'lastname', u'password', u'mailbox-quota', u'mail', u'mail-alias', u'mail-forward', u'groups'] -VALIDATORS = { +FIELDS_FOR_IMPORT = { 'username': r'^[a-z0-9_]+$', 'firstname': r'^([^\W\d_]{1,30}[ ,.\'-]{0,3})+$', 'lastname': r'^([^\W\d_]{1,30}[ ,.\'-]{0,3})+$', @@ -619,10 +618,10 @@ def user_export(): import csv # CSV are needed only in this function from io import StringIO with StringIO() as csv_io: - writer = csv.DictWriter(csv_io, CSV_FIELDNAMES, + writer = csv.DictWriter(csv_io, list(FIELDS_FOR_IMPORT.keys()), delimiter=';', quotechar='"') writer.writeheader() - users = user_list(CSV_FIELDNAMES)['users'] + users = user_list(list(FIELDS_FOR_IMPORT.keys()))['users'] for username, user in users.items(): user['mail-alias'] = ','.join(user['mail-alias']) user['mail-forward'] = ','.join(user['mail-forward']) @@ -672,24 +671,24 @@ def user_import(operation_logger, csvfile, update=False, delete=False): users_in_csv = [] existing_users = user_list()['users'] - past_lines = [] reader = csv.DictReader(csvfile, delimiter=';', quotechar='"') - for user in reader: - # Validation - try: - format_errors = [key + ':' + str(user[key]) - for key, validator in VALIDATORS.items() - if user[key] is None or not re.match(validator, user[key])] - except KeyError as e: - logger.error(m18n.n('user_import_missing_column', - column=str(e))) - is_well_formatted = False - break - if 'username' in user: - if user['username'] in past_lines: - format_errors.append('username: %s (duplicated)' % user['username']) - past_lines.append(user['username']) + missing_columns = [key for key in FIELDS_FOR_IMPORT.keys() if key not in reader.fieldnames] + if missing_columns: + raise YunohostValidationError("user_import_missing_columns", columns=', '.join(missing_columns)) + + for user in reader: + + # Validate column values against regexes + format_errors = [key + ':' + str(user[key]) + for key, validator in FIELDS_FOR_IMPORT.items() + if user[key] is None or not re.match(validator, user[key])] + + # Check for duplicated username lines + if user['username'] in users_in_csv: + format_errors.append(f'username: {user[username]} (duplicated)') + users_in_csv.append(user['username']) + if format_errors: logger.error(m18n.n('user_import_bad_line', line=reader.line_num, @@ -714,8 +713,6 @@ def user_import(operation_logger, csvfile, update=False, delete=False): elif update: actions['updated'].append(user) - users_in_csv.add(user['username']) - if delete: actions['deleted'] = [user for user in existing_users if user not in users_in_csv] @@ -787,7 +784,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): for group in new_infos['groups']: user_group_update(group, add=new_infos['username'], sync_perm=False, from_import=True) - users = user_list(CSV_FIELDNAMES)['users'] + users = user_list(list(FIELDS_FOR_IMPORT.keys()))['users'] operation_logger.start() # We do delete and update before to avoid mail uniqueness issues for user in actions['deleted']: From 8e35cd8103e8eb40c103ed92d80a2086b35a65b0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 27 Aug 2021 13:52:10 +0200 Subject: [PATCH 29/40] user import: strip spaces --- src/yunohost/user.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index b055d2cdb..ce2c2c34f 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -667,7 +667,9 @@ def user_import(operation_logger, csvfile, update=False, delete=False): is_well_formatted = True def to_list(str_list): - return str_list.split(',') if str_list else [] + L = str_list.split(',') if str_list else [] + L = [l.strip() for l in L] + return L users_in_csv = [] existing_users = user_list()['users'] From 24d87ea40eefbd5ae4fcda45be72314a30bb28d1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 27 Aug 2021 14:09:13 +0200 Subject: [PATCH 30/40] user import: validate that groups and domains exists --- src/yunohost/user.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index ce2c2c34f..1b0e47f80 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -671,9 +671,12 @@ def user_import(operation_logger, csvfile, update=False, delete=False): L = [l.strip() for l in L] return L - users_in_csv = [] existing_users = user_list()['users'] + existing_groups = user_group_list()["groups"] + existing_domains = domain_list()["domains"] + reader = csv.DictReader(csvfile, delimiter=';', quotechar='"') + users_in_csv = [] missing_columns = [key for key in FIELDS_FOR_IMPORT.keys() if key not in reader.fieldnames] if missing_columns: @@ -688,9 +691,29 @@ def user_import(operation_logger, csvfile, update=False, delete=False): # Check for duplicated username lines if user['username'] in users_in_csv: - format_errors.append(f'username: {user[username]} (duplicated)') + format_errors.append(f"username '{user[username]}' duplicated") users_in_csv.append(user['username']) + # Validate that groups exist + user['groups'] = to_list(user['groups']) + unknown_groups = [g for g in user['groups'] if g not in existing_groups] + if unknown_groups: + format_errors.append(f"username '{user[username]}': unknown groups %s" % ', '.join(unknown_groups)) + + # Validate that domains exist + user['mail-alias'] = to_list(user['mail-alias']) + user['mail-forward'] = to_list(user['mail-forward']) + user['domain'] = user['mail'].split('@')[1] + + unknown_domains = [] + if user['domain'] not in existing_domains: + unknown_domains.append(user['domain']) + + unknown_domains += [mail.split('@')[1:] for mail in user['mail-alias'] if mail.split('@')[1:] not in existing_domains] + + if unknown_domains: + format_errors.append(f"username '{user[username]}': unknown domains %s" % ', '.join(unknown_domains)) + if format_errors: logger.error(m18n.n('user_import_bad_line', line=reader.line_num, @@ -699,10 +722,6 @@ def user_import(operation_logger, csvfile, update=False, delete=False): continue # Choose what to do with this line and prepare data - user['groups'] = to_list(user['groups']) - user['mail-alias'] = to_list(user['mail-alias']) - user['mail-forward'] = to_list(user['mail-forward']) - user['domain'] = user['mail'].split('@')[1] # User creation if user['username'] not in existing_users: From a40084460b96254c4f2b8a3a1ff7031168b60afe Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 27 Aug 2021 14:10:20 +0200 Subject: [PATCH 31/40] csv -> CSV --- data/actionsmap/yunohost.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 6e9509bab..0c1895c47 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -222,11 +222,11 @@ user: type: open -u: full: --update - help: Update all existing users contained in the csv file (by default existing users are ignored) + help: Update all existing users contained in the CSV file (by default existing users are ignored) action: store_true -d: full: --delete - help: Delete all existing users that are not contained in the csv file (by default existing users are kept) + help: Delete all existing users that are not contained in the CSV file (by default existing users are kept) action: store_true subcategories: From 4de6bbdf84eb7965517395adacb7319506dc5ad3 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 27 Aug 2021 14:38:40 +0200 Subject: [PATCH 32/40] user import: Try to optimize group operations --- src/yunohost/user.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 1b0e47f80..bc684ec0c 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -54,8 +54,10 @@ FIELDS_FOR_IMPORT = { 'mailbox-quota': r'^(\d+[bkMGT])|0$', 'groups': r'^|([a-z0-9_]+(,?[a-z0-9_]+)*)$' } + FIRST_ALIASES = ['root@', 'admin@', 'webmaster@', 'postmaster@', 'abuse@'] + def user_list(fields=None): from yunohost.utils.ldap import _get_ldap_interface @@ -779,18 +781,26 @@ def user_import(operation_logger, csvfile, update=False, delete=False): def update(new_infos, old_infos=False): remove_alias = None remove_forward = None - if info: + remove_groups = [] + add_groups = new_infos["groups"] + if old_infos: new_infos['mail'] = None if old_infos['mail'] == new_infos['mail'] else new_infos['mail'] remove_alias = list(set(old_infos['mail-alias']) - set(new_infos['mail-alias'])) remove_forward = list(set(old_infos['mail-forward']) - set(new_infos['mail-forward'])) new_infos['mail-alias'] = list(set(new_infos['mail-alias']) - set(old_infos['mail-alias'])) new_infos['mail-forward'] = list(set(new_infos['mail-forward']) - set(old_infos['mail-forward'])) - for group, infos in user_group_list()["groups"].items(): - if group == "all_users": + + remove_groups = list(set(old_infos["groups"]) - set(new_infos["groups"])) + add_groups = list(set(new_infos["groups"]) - set(old_infos["groups"])) + + for group, infos in existing_groups: + # Loop only on groups in 'remove_groups' + # Ignore 'all_users' and primary group + if group in ["all_users", new_infos['username']] or group not in remove_groups: continue # If the user is in this group (and it's not the primary group), # remove the member from the group - if new_infos['username'] != group and new_infos['username'] in infos["members"]: + if new_infos['username'] in infos["members"]: user_group_update(group, remove=new_infos['username'], sync_perm=False, from_import=True) user_update(new_infos['username'], @@ -802,7 +812,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): remove_mailforward=remove_forward, add_mailforward=new_infos['mail-forward'], from_import=True) - for group in new_infos['groups']: + for group in add_groups: user_group_update(group, add=new_infos['username'], sync_perm=False, from_import=True) users = user_list(list(FIELDS_FOR_IMPORT.keys()))['users'] From 865d8e2c8c7838d8ccab813eb2d3b271e8168545 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 27 Aug 2021 14:48:30 +0200 Subject: [PATCH 33/40] user import: fix typo --- src/yunohost/user.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index bc684ec0c..07f3e546a 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -693,14 +693,14 @@ def user_import(operation_logger, csvfile, update=False, delete=False): # Check for duplicated username lines if user['username'] in users_in_csv: - format_errors.append(f"username '{user[username]}' duplicated") + format_errors.append(f"username '{user['username']}' duplicated") users_in_csv.append(user['username']) # Validate that groups exist user['groups'] = to_list(user['groups']) unknown_groups = [g for g in user['groups'] if g not in existing_groups] if unknown_groups: - format_errors.append(f"username '{user[username]}': unknown groups %s" % ', '.join(unknown_groups)) + format_errors.append(f"username '{user['username']}': unknown groups %s" % ', '.join(unknown_groups)) # Validate that domains exist user['mail-alias'] = to_list(user['mail-alias']) @@ -714,7 +714,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): unknown_domains += [mail.split('@')[1:] for mail in user['mail-alias'] if mail.split('@')[1:] not in existing_domains] if unknown_domains: - format_errors.append(f"username '{user[username]}': unknown domains %s" % ', '.join(unknown_domains)) + format_errors.append(f"username '{user['username']}': unknown domains %s" % ', '.join(unknown_domains)) if format_errors: logger.error(m18n.n('user_import_bad_line', From 4f6166f8bba942e420239bb3f8b32327ac3f915e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 27 Aug 2021 15:47:40 +0200 Subject: [PATCH 34/40] ldap config: Bump DbMaxSize to allow up to 100MB --- data/templates/slapd/config.ldif | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/templates/slapd/config.ldif b/data/templates/slapd/config.ldif index 4d72ab7cc..e1fe3b1b5 100644 --- a/data/templates/slapd/config.ldif +++ b/data/templates/slapd/config.ldif @@ -33,7 +33,7 @@ olcAuthzPolicy: none olcConcurrency: 0 olcConnMaxPending: 100 olcConnMaxPendingAuth: 1000 -olcSizeLimit: 10000 +olcSizeLimit: 50000 olcIdleTimeout: 0 olcIndexSubstrIfMaxLen: 4 olcIndexSubstrIfMinLen: 2 @@ -189,7 +189,7 @@ olcDbIndex: memberUid eq olcDbIndex: uniqueMember eq olcDbIndex: virtualdomain eq olcDbIndex: permission eq -olcDbMaxSize: 10485760 +olcDbMaxSize: 104857600 structuralObjectClass: olcMdbConfig # From 3ad48fd8bc323ad2bd42f4197f0b61cef54dac01 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 27 Aug 2021 16:34:34 +0200 Subject: [PATCH 35/40] Fixes after tests on the battlefield --- src/yunohost/user.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 07f3e546a..f6cbeb9bc 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -655,10 +655,11 @@ def user_import(operation_logger, csvfile, update=False, delete=False): """ - import csv # CSV are needed only in this function + import csv # CSV are needed only in this function from moulinette.utils.text import random_ascii from yunohost.permission import permission_sync_to_user from yunohost.app import app_ssowatconf + from yunohost.domain import domain_list # Pre-validate data and prepare what should be done actions = { @@ -711,7 +712,8 @@ def user_import(operation_logger, csvfile, update=False, delete=False): if user['domain'] not in existing_domains: unknown_domains.append(user['domain']) - unknown_domains += [mail.split('@')[1:] for mail in user['mail-alias'] if mail.split('@')[1:] not in existing_domains] + unknown_domains += [mail.split('@', 1)[1] for mail in user['mail-alias'] if mail.split('@', 1)[1] not in existing_domains] + unknown_domains = set(unknown_domains) if unknown_domains: format_errors.append(f"username '{user['username']}': unknown domains %s" % ', '.join(unknown_domains)) @@ -744,7 +746,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): is_well_formatted = False if not is_well_formatted: - raise YunohostError('user_import_bad_file') + raise YunohostValidationError('user_import_bad_file') total = len(actions['created'] + actions['updated'] + actions['deleted']) @@ -793,7 +795,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): remove_groups = list(set(old_infos["groups"]) - set(new_infos["groups"])) add_groups = list(set(new_infos["groups"]) - set(old_infos["groups"])) - for group, infos in existing_groups: + for group, infos in existing_groups.items(): # Loop only on groups in 'remove_groups' # Ignore 'all_users' and primary group if group in ["all_users", new_infos['username']] or group not in remove_groups: @@ -813,6 +815,8 @@ def user_import(operation_logger, csvfile, update=False, delete=False): add_mailforward=new_infos['mail-forward'], from_import=True) for group in add_groups: + if group in ["all_users", new_infos['username']]: + continue user_group_update(group, add=new_infos['username'], sync_perm=False, from_import=True) users = user_list(list(FIELDS_FOR_IMPORT.keys()))['users'] @@ -824,7 +828,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): result['deleted'] += 1 except YunohostError as e: on_failure(user, e) - progress("Deletion") + progress(f"Deleting {user}") for user in actions['updated']: try: @@ -832,7 +836,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): result['updated'] += 1 except YunohostError as e: on_failure(user['username'], e) - progress("Update") + progress(f"Updating {user['username']}") for user in actions['created']: try: @@ -844,7 +848,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): result['created'] += 1 except YunohostError as e: on_failure(user['username'], e) - progress("Creation") + progress(f"Creating {user['username']}") permission_sync_to_user() app_ssowatconf() From 0e2105311f1e376328a911dc0f961a5e99e87c28 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 27 Aug 2021 16:36:38 +0200 Subject: [PATCH 36/40] user import: fix tests --- src/yunohost/tests/test_user-group.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/yunohost/tests/test_user-group.py b/src/yunohost/tests/test_user-group.py index bbedfc27f..d3b3c81aa 100644 --- a/src/yunohost/tests/test_user-group.py +++ b/src/yunohost/tests/test_user-group.py @@ -164,10 +164,10 @@ def test_export_user(mocker): result = user_export() aliases = ','.join([alias + maindomain for alias in FIRST_ALIASES]) should_be = ( - "username;firstname;lastname;password;mailbox-quota;mail;mail-alias;mail-forward;groups\r\n" - f"alice;Alice;White;;0;alice@{maindomain};{aliases};;dev\r\n" - f"bob;Bob;Snow;;0;bob@{maindomain};;;apps\r\n" - f"jack;Jack;Black;;0;jack@{maindomain};;;" + "username;firstname;lastname;password;mail;mail-alias;mail-forward;mailbox-quota;groups\r\n" + f"alice;Alice;White;;alice@{maindomain};{aliases};;0;dev\r\n" + f"bob;Bob;Snow;;bob@{maindomain};;;0;apps\r\n" + f"jack;Jack;Black;;jack@{maindomain};;;0;" ) assert result == should_be From 5af70af47d0612378cccde87823fe85326825ccb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 27 Aug 2021 17:09:46 +0200 Subject: [PATCH 37/40] user import: Allow empty value for mailbox quota --- src/yunohost/user.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index f6cbeb9bc..0cdd0d3ae 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -51,7 +51,7 @@ FIELDS_FOR_IMPORT = { 'mail': r'^([\w.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}))$', 'mail-alias': r'^|([\w.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}),?)+$', 'mail-forward': r'^|([\w\+.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,}),?)+$', - 'mailbox-quota': r'^(\d+[bkMGT])|0$', + 'mailbox-quota': r'^(\d+[bkMGT])|0|$', 'groups': r'^|([a-z0-9_]+(,?[a-z0-9_]+)*)$' } @@ -688,7 +688,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): for user in reader: # Validate column values against regexes - format_errors = [key + ':' + str(user[key]) + format_errors = [f"{key}: '{user[key]}' doesn't match the expected format" for key, validator in FIELDS_FOR_IMPORT.items() if user[key] is None or not re.match(validator, user[key])] @@ -726,6 +726,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): continue # Choose what to do with this line and prepare data + user['mailbox-quota'] = user['mailbox-quota'] or "0" # User creation if user['username'] not in existing_users: From caef1e0577a1b0f3d0136d337193bc1c65280b4c Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Thu, 2 Sep 2021 14:08:18 +0200 Subject: [PATCH 38/40] Update src/yunohost/log.py --- src/yunohost/log.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index f9f9334fb..4994d608c 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -373,7 +373,11 @@ def is_unit_operation( if field in context: context.pop(field, None) - # Manage file or stream + # Context is made from args given to main function by argparse + # This context will be added in extra parameters in yml file, so this context should + # be serializable and short enough (it will be displayed in webadmin) + # Argparse can provide some File or Stream, so here we display the filename or + # the IOBase, if we have no name. for field, value in context.items(): if isinstance(value, IOBase): try: From f2487a2251fdc8c19dc5dbd14a8eb3707f97156f Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Thu, 2 Sep 2021 14:24:06 +0200 Subject: [PATCH 39/40] Avoid confusing things in user_list --- src/yunohost/user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/user.py b/src/yunohost/user.py index 0cdd0d3ae..65edf5821 100644 --- a/src/yunohost/user.py +++ b/src/yunohost/user.py @@ -64,7 +64,7 @@ def user_list(fields=None): ldap_attrs = { 'username': 'uid', - 'password': 'uid', + 'password': '', # We can't request password in ldap 'fullname': 'cn', 'firstname': 'givenName', 'lastname': 'sn', From e27f38ae69aeb9308d8166e366f764f8afd8378f Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Thu, 2 Sep 2021 14:33:06 +0200 Subject: [PATCH 40/40] Test group remove on csv import --- src/yunohost/tests/test_user-group.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yunohost/tests/test_user-group.py b/src/yunohost/tests/test_user-group.py index d3b3c81aa..ab7e72555 100644 --- a/src/yunohost/tests/test_user-group.py +++ b/src/yunohost/tests/test_user-group.py @@ -158,6 +158,7 @@ def test_import_user(mocker): assert len(user_res['alice']['mail-alias']) == 2 assert "albert" in group_res['dev']['members'] assert "alice" in group_res['apps']['members'] + assert "alice" not in group_res['dev']['members'] def test_export_user(mocker):