From b7c8ab541f42eeaff0f15e46f872d79e5fcc3b38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrea=20PIERR=C3=89?= Date: Fri, 5 Oct 2018 12:54:33 +0200 Subject: [PATCH 1/9] [#901] Human readable date --- data/actionsmap/yunohost.yml | 5 +++++ src/yunohost/app.py | 14 +++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index ec4a6bf08..fd1d98946 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -469,6 +469,11 @@ app: listlists: action_help: List registered application lists api: GET /appslists + arguments: + -H: + full: --human-readable + help: Return the lastUpdate with a human readable date + action: store_true ### app_removelist() removelist: diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 8c35d5e39..b2116d966 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -36,6 +36,7 @@ import glob import pwd import grp from collections import OrderedDict +import datetime from moulinette import msignals, m18n, msettings from moulinette.core import MoulinetteError @@ -66,10 +67,13 @@ re_app_instance_name = re.compile( ) -def app_listlists(): +def app_listlists(human_readable=False): """ List fetched lists + Keyword argument: + human_readable -- Show human readable dates + """ # Migrate appslist system if needed @@ -80,6 +84,14 @@ def app_listlists(): # Get the list appslist_list = _read_appslist_list() + # Human readable date + if human_readable: + for app in appslist_list: + now_for_humans = datetime.datetime.fromtimestamp( + appslist_list[app].get("lastUpdate")) + appslist_list[app]["lastUpdate"] = now_for_humans.strftime( + '%Y-%m-%d %H:%M:%S') + return appslist_list From 830341a687e2e0aeac3304f8508c20fc1b47f9c4 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 9 Dec 2018 19:27:44 +0100 Subject: [PATCH 2/9] [enh] Display date correctly --- data/actionsmap/yunohost.yml | 5 ----- debian/control | 2 +- src/yunohost/app.py | 14 ++++++-------- src/yunohost/log.py | 13 ++++++++++--- src/yunohost/monitor.py | 2 +- src/yunohost/service.py | 17 ++++++++--------- 6 files changed, 26 insertions(+), 27 deletions(-) diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index a6bc8ec3e..ba5f1d505 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -470,11 +470,6 @@ app: listlists: action_help: List registered application lists api: GET /appslists - arguments: - -H: - full: --human-readable - help: Return the lastUpdate with a human readable date - action: store_true ### app_removelist() removelist: diff --git a/debian/control b/debian/control index 9f72bf11a..f0423c4a3 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,7 @@ Package: yunohost Architecture: all Depends: ${python:Depends}, ${misc:Depends} , moulinette (>= 2.7.1), ssowat (>= 2.7.1) - , python-psutil, python-requests, python-dnspython, python-openssl + , python-psutil, python-requests, python-dnspython, python-openssl, python-tz , python-apt, python-miniupnpc, python-dbus, python-jinja2 , glances , dnsutils, bind9utils, unzip, git, curl, cron, wget, jq diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 66d83db9c..d5121a418 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -37,6 +37,7 @@ import pwd import grp from collections import OrderedDict import datetime +import pytz from moulinette import msignals, m18n, msettings from moulinette.core import MoulinetteError @@ -67,7 +68,7 @@ re_app_instance_name = re.compile( ) -def app_listlists(human_readable=False): +def app_listlists(): """ List fetched lists @@ -84,13 +85,10 @@ def app_listlists(human_readable=False): # Get the list appslist_list = _read_appslist_list() - # Human readable date - if human_readable: - for app in appslist_list: - now_for_humans = datetime.datetime.fromtimestamp( - appslist_list[app].get("lastUpdate")) - appslist_list[app]["lastUpdate"] = now_for_humans.strftime( - '%Y-%m-%d %H:%M:%S') + for app in appslist_list: + last_update = datetime.datetime.utcfromtimestamp( + appslist_list[app].get("lastUpdate")) + appslist_list[app]["lastUpdate"] = last_update.replace(tzinfo=pytz.utc) return appslist_list diff --git a/src/yunohost/log.py b/src/yunohost/log.py index c105b8279..c75300d3d 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -28,6 +28,7 @@ import os import yaml import errno import collections +import pytz from datetime import datetime from logging import FileHandler, getLogger, Formatter @@ -100,7 +101,7 @@ def log_list(category=[], limit=None): except ValueError: pass else: - entry["started_at"] = log_datetime + entry["started_at"] = log_datetime.replace(tzinfo=pytz.utc) result[category].append(entry) @@ -182,6 +183,10 @@ def log_display(path, number=50, share=False): metadata = yaml.safe_load(md_file) infos['metadata_path'] = md_path infos['metadata'] = metadata + if 'started_at' in infos['metadata']: + infos['metadata']['started_at'] = infos['metadata']['started_at'].replace(tzinfo=pytz.utc) + if 'ended_at' in infos['metadata']: + infos['metadata']['ended_at'] = infos['metadata']['ended_at'].replace(tzinfo=pytz.utc) if 'log_path' in metadata: log_path = metadata['log_path'] except yaml.YAMLError: @@ -316,7 +321,8 @@ class OperationLogger(object): """ if self.started_at is None: - self.started_at = datetime.now() + self.started_at = datetime.utcnow() + self.started_at = self.started_at.replace(tzinfo=pytz.utc) self.flush() self._register_log() @@ -408,7 +414,8 @@ class OperationLogger(object): return if error is not None and not isinstance(error, basestring): error = str(error) - self.ended_at = datetime.now() + self.ended_at = datetime.utcnow() + self.ended_at = self.ended_at.replace(tzinfo=pytz.utc) self._error = error self._success = error is None if self.logger is not None: diff --git a/src/yunohost/monitor.py b/src/yunohost/monitor.py index fc10a4fbc..cad467ed5 100644 --- a/src/yunohost/monitor.py +++ b/src/yunohost/monitor.py @@ -283,7 +283,7 @@ def monitor_system(units=None, human_readable=False): elif u == 'process': result[u] = json.loads(glances.getProcessCount()) elif u == 'uptime': - result[u] = (str(datetime.now() - datetime.fromtimestamp(psutil.boot_time())).split('.')[0]) + result[u] = (str(datetime.utcnow() - datetime.utcfromtimestamp(psutil.boot_time())).split('.')[0]) elif u == 'infos': result[u] = json.loads(glances.getSystem()) else: diff --git a/src/yunohost/service.py b/src/yunohost/service.py index 5b7680a80..73383cb44 100644 --- a/src/yunohost/service.py +++ b/src/yunohost/service.py @@ -31,6 +31,7 @@ import subprocess import errno import shutil import hashlib +import pytz from difflib import unified_diff from datetime import datetime @@ -248,10 +249,7 @@ def service_status(names=[]): 'status': "unknown", 'loaded': "unknown", 'active': "unknown", - 'active_at': { - "timestamp": "unknown", - "human": "unknown", - }, + 'active_at': "unknown", 'description': "Error: failed to get information for this service, it doesn't exists for systemd", 'service_file_path': "unknown", } @@ -273,13 +271,14 @@ def service_status(names=[]): 'status': str(status.get("SubState", "unknown")), 'loaded': "enabled" if str(status.get("LoadState", "unknown")) == "loaded" else str(status.get("LoadState", "unknown")), 'active': str(status.get("ActiveState", "unknown")), - 'active_at': { - "timestamp": str(status.get("ActiveEnterTimestamp", "unknown")), - "human": datetime.fromtimestamp(status["ActiveEnterTimestamp"] / 1000000).strftime("%F %X") if "ActiveEnterTimestamp" in status else "unknown", - }, 'description': description, 'service_file_path': str(status.get("FragmentPath", "unknown")), } + if "ActiveEnterTimestamp" in status: + result[name]['active_at'] = datetime.utcfromtimestamp(status["ActiveEnterTimestamp"] / 1000000) + result[name]['active_at'] = result[name]['active_at'].replace(tzinfo=pytz.utc) + else: + result[name]['active_at'] = "unknown" if len(names) == 1: return result[names[0]] @@ -293,7 +292,7 @@ def _get_service_information_from_systemd(service): d = dbus.SystemBus() - systemd = d.get_object('org.freedesktop.systemd1','/org/freedesktop/systemd1') + systemd = d.get_object('org.freedesktop.systemd1', '/org/freedesktop/systemd1') manager = dbus.Interface(systemd, 'org.freedesktop.systemd1.Manager') try: From c417c628948e1bbe928fd213e13fec0d2d554ce8 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 9 Dec 2018 19:35:38 +0100 Subject: [PATCH 3/9] [fix] Bad comment --- src/yunohost/app.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index d5121a418..e107d7ac9 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -72,9 +72,6 @@ def app_listlists(): """ List fetched lists - Keyword argument: - human_readable -- Show human readable dates - """ # Migrate appslist system if needed From 60843edc0f6bd6d59bb785f620c95958d71f0108 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 9 Dec 2018 21:08:36 +0100 Subject: [PATCH 4/9] [enh] Use aware UTC date in backup settings and app --- src/yunohost/app.py | 3 +-- src/yunohost/backup.py | 6 ++++-- src/yunohost/settings.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index e107d7ac9..0664a24b2 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -1762,8 +1762,7 @@ def _get_app_status(app_id, format_date=False): if not v: status[f] = '-' else: - status[f] = time.strftime(m18n.n('format_datetime_short'), - time.gmtime(v)) + status[f] = datetime.utcfromtimestamp(v).replace(tzinfo=pytz.utc) return status diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 6ef9a0554..492687eaf 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -33,6 +33,8 @@ import shutil import subprocess import csv import tempfile +import pytz +from datetime import datetime from glob import glob from collections import OrderedDict @@ -2266,8 +2268,8 @@ def backup_info(name, with_details=False, human_readable=False): result = { 'path': archive_file, - 'created_at': time.strftime(m18n.n('format_datetime_short'), - time.gmtime(info['created_at'])), + 'created_at': datetime.utcfromtimestamp(info['created_at']) + .replace(tzinfo=pytz.utc), 'description': info['description'], 'size': size, } diff --git a/src/yunohost/settings.py b/src/yunohost/settings.py index d2526316e..3227ad3e3 100644 --- a/src/yunohost/settings.py +++ b/src/yunohost/settings.py @@ -154,7 +154,7 @@ def settings_reset_all(): # addition but we'll see if this is a common need. # Another solution would be to use etckeeper and integrate those # modification inside of it and take advantage of its git history - old_settings_backup_path = SETTINGS_PATH_OTHER_LOCATION % datetime.now().strftime("%F_%X") + old_settings_backup_path = SETTINGS_PATH_OTHER_LOCATION % datetime.utcnow().strftime("%F_%X") _save_settings(settings, location=old_settings_backup_path) for value in settings.values(): From e43bcf253cef4137120e0fc8faa498ebc015a020 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 9 Dec 2018 21:24:59 +0100 Subject: [PATCH 5/9] [fix] Avoid -1 remaining_days --- src/yunohost/certificate.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/yunohost/certificate.py b/src/yunohost/certificate.py index 8da6ab52a..9975a349f 100644 --- a/src/yunohost/certificate.py +++ b/src/yunohost/certificate.py @@ -154,7 +154,7 @@ def _certificate_install_selfsigned(domain_list, force=False): args={'force': force}) # Paths of files and folder we'll need - date_tag = datetime.now().strftime("%Y%m%d.%H%M%S") + date_tag = datetime.utcnow().strftime("%Y%m%d.%H%M%S") new_cert_folder = "%s/%s-history/%s-selfsigned" % ( CERT_FOLDER, domain, date_tag) @@ -587,7 +587,7 @@ def _fetch_and_enable_new_certificate(domain, staging=False, no_checks=False): logger.debug("Saving the key and signed certificate...") # Create corresponding directory - date_tag = datetime.now().strftime("%Y%m%d.%H%M%S") + date_tag = datetime.utcnow().strftime("%Y%m%d.%H%M%S") if staging: folder_flag = "staging" @@ -674,7 +674,7 @@ def _get_status(domain): cert_subject = cert.get_subject().CN cert_issuer = cert.get_issuer().CN valid_up_to = datetime.strptime(cert.get_notAfter(), "%Y%m%d%H%M%SZ") - days_remaining = (valid_up_to - datetime.now()).days + days_remaining = (valid_up_to - datetime.utcnow()).days if cert_issuer == _name_self_CA(): CA_type = { @@ -816,7 +816,7 @@ def _backup_current_cert(domain): cert_folder_domain = os.path.join(CERT_FOLDER, domain) - date_tag = datetime.now().strftime("%Y%m%d.%H%M%S") + date_tag = datetime.utcnow().strftime("%Y%m%d.%H%M%S") backup_folder = "%s-backups/%s" % (cert_folder_domain, date_tag) shutil.copytree(cert_folder_domain, backup_folder) From db1b00e706a9c4f1ea16a95b6ca3c8184f831789 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 9 Dec 2018 23:48:56 +0100 Subject: [PATCH 6/9] [enh] Use UTC naive format and manage it in moulinette --- src/yunohost/app.py | 6 ++---- src/yunohost/backup.py | 6 ++---- src/yunohost/log.py | 8 +++----- src/yunohost/monitor.py | 2 +- src/yunohost/service.py | 4 +--- 5 files changed, 9 insertions(+), 17 deletions(-) diff --git a/src/yunohost/app.py b/src/yunohost/app.py index 0664a24b2..79af7faf2 100644 --- a/src/yunohost/app.py +++ b/src/yunohost/app.py @@ -37,7 +37,6 @@ import pwd import grp from collections import OrderedDict import datetime -import pytz from moulinette import msignals, m18n, msettings from moulinette.core import MoulinetteError @@ -83,9 +82,8 @@ def app_listlists(): appslist_list = _read_appslist_list() for app in appslist_list: - last_update = datetime.datetime.utcfromtimestamp( + appslist_list[app]["lastUpdate"] = datetime.datetime.utcfromtimestamp( appslist_list[app].get("lastUpdate")) - appslist_list[app]["lastUpdate"] = last_update.replace(tzinfo=pytz.utc) return appslist_list @@ -1762,7 +1760,7 @@ def _get_app_status(app_id, format_date=False): if not v: status[f] = '-' else: - status[f] = datetime.utcfromtimestamp(v).replace(tzinfo=pytz.utc) + status[f] = datetime.utcfromtimestamp(v) return status diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 492687eaf..528b2e24f 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -33,7 +33,6 @@ import shutil import subprocess import csv import tempfile -import pytz from datetime import datetime from glob import glob from collections import OrderedDict @@ -301,7 +300,7 @@ class BackupManager(): (string) A backup name created from current date 'YYMMDD-HHMMSS' """ # FIXME: case where this name already exist - return time.strftime('%Y%m%d-%H%M%S') + return time.strftime('%Y%m%d-%H%M%S', time.gmtime()) def _init_work_dir(self): """Initialize preparation directory @@ -2268,8 +2267,7 @@ def backup_info(name, with_details=False, human_readable=False): result = { 'path': archive_file, - 'created_at': datetime.utcfromtimestamp(info['created_at']) - .replace(tzinfo=pytz.utc), + 'created_at': datetime.utcfromtimestamp(info['created_at']), 'description': info['description'], 'size': size, } diff --git a/src/yunohost/log.py b/src/yunohost/log.py index c75300d3d..af19090c8 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -101,7 +101,7 @@ def log_list(category=[], limit=None): except ValueError: pass else: - entry["started_at"] = log_datetime.replace(tzinfo=pytz.utc) + entry["started_at"] = log_datetime result[category].append(entry) @@ -321,8 +321,7 @@ class OperationLogger(object): """ if self.started_at is None: - self.started_at = datetime.utcnow() - self.started_at = self.started_at.replace(tzinfo=pytz.utc) + self.started_at = datetime.now(tz=pytz.utc) self.flush() self._register_log() @@ -414,8 +413,7 @@ class OperationLogger(object): return if error is not None and not isinstance(error, basestring): error = str(error) - self.ended_at = datetime.utcnow() - self.ended_at = self.ended_at.replace(tzinfo=pytz.utc) + self.ended_at = datetime.now(tz=pytz.utc) self._error = error self._success = error is None if self.logger is not None: diff --git a/src/yunohost/monitor.py b/src/yunohost/monitor.py index cad467ed5..fc10a4fbc 100644 --- a/src/yunohost/monitor.py +++ b/src/yunohost/monitor.py @@ -283,7 +283,7 @@ def monitor_system(units=None, human_readable=False): elif u == 'process': result[u] = json.loads(glances.getProcessCount()) elif u == 'uptime': - result[u] = (str(datetime.utcnow() - datetime.utcfromtimestamp(psutil.boot_time())).split('.')[0]) + result[u] = (str(datetime.now() - datetime.fromtimestamp(psutil.boot_time())).split('.')[0]) elif u == 'infos': result[u] = json.loads(glances.getSystem()) else: diff --git a/src/yunohost/service.py b/src/yunohost/service.py index 73383cb44..fd75d0235 100644 --- a/src/yunohost/service.py +++ b/src/yunohost/service.py @@ -31,7 +31,6 @@ import subprocess import errno import shutil import hashlib -import pytz from difflib import unified_diff from datetime import datetime @@ -276,7 +275,6 @@ def service_status(names=[]): } if "ActiveEnterTimestamp" in status: result[name]['active_at'] = datetime.utcfromtimestamp(status["ActiveEnterTimestamp"] / 1000000) - result[name]['active_at'] = result[name]['active_at'].replace(tzinfo=pytz.utc) else: result[name]['active_at'] = "unknown" @@ -926,7 +924,7 @@ def _process_regen_conf(system_conf, new_conf=None, save=True): """ if save: backup_path = os.path.join(BACKUP_CONF_DIR, '{0}-{1}'.format( - system_conf.lstrip('/'), time.strftime("%Y%m%d.%H%M%S"))) + system_conf.lstrip('/'), datetime.utcnow().strftime("%Y%m%d.%H%M%S"))) backup_dir = os.path.dirname(backup_path) if not os.path.isdir(backup_dir): From cafd0dd884d9a10ea761efdef7bb1a1938082092 Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 9 Dec 2018 23:55:51 +0100 Subject: [PATCH 7/9] [enh] Remove pytz --- debian/control | 2 +- src/yunohost/log.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/debian/control b/debian/control index f0423c4a3..9f72bf11a 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,7 @@ Package: yunohost Architecture: all Depends: ${python:Depends}, ${misc:Depends} , moulinette (>= 2.7.1), ssowat (>= 2.7.1) - , python-psutil, python-requests, python-dnspython, python-openssl, python-tz + , python-psutil, python-requests, python-dnspython, python-openssl , python-apt, python-miniupnpc, python-dbus, python-jinja2 , glances , dnsutils, bind9utils, unzip, git, curl, cron, wget, jq diff --git a/src/yunohost/log.py b/src/yunohost/log.py index af19090c8..f11c587d1 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -28,7 +28,6 @@ import os import yaml import errno import collections -import pytz from datetime import datetime from logging import FileHandler, getLogger, Formatter @@ -184,9 +183,9 @@ def log_display(path, number=50, share=False): infos['metadata_path'] = md_path infos['metadata'] = metadata if 'started_at' in infos['metadata']: - infos['metadata']['started_at'] = infos['metadata']['started_at'].replace(tzinfo=pytz.utc) + infos['metadata']['started_at'] = infos['metadata']['started_at'] if 'ended_at' in infos['metadata']: - infos['metadata']['ended_at'] = infos['metadata']['ended_at'].replace(tzinfo=pytz.utc) + infos['metadata']['ended_at'] = infos['metadata']['ended_at'] if 'log_path' in metadata: log_path = metadata['log_path'] except yaml.YAMLError: @@ -321,7 +320,7 @@ class OperationLogger(object): """ if self.started_at is None: - self.started_at = datetime.now(tz=pytz.utc) + self.started_at = datetime.utcnow() self.flush() self._register_log() @@ -413,7 +412,7 @@ class OperationLogger(object): return if error is not None and not isinstance(error, basestring): error = str(error) - self.ended_at = datetime.now(tz=pytz.utc) + self.ended_at = datetime.utcnow() self._error = error self._success = error is None if self.logger is not None: From 7732136b7508a3056ae79a1745312f604098463a Mon Sep 17 00:00:00 2001 From: ljf Date: Sun, 9 Dec 2018 23:58:06 +0100 Subject: [PATCH 8/9] [enh] Remove unneeded code --- src/yunohost/log.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/yunohost/log.py b/src/yunohost/log.py index f11c587d1..80a7a134f 100644 --- a/src/yunohost/log.py +++ b/src/yunohost/log.py @@ -182,10 +182,6 @@ def log_display(path, number=50, share=False): metadata = yaml.safe_load(md_file) infos['metadata_path'] = md_path infos['metadata'] = metadata - if 'started_at' in infos['metadata']: - infos['metadata']['started_at'] = infos['metadata']['started_at'] - if 'ended_at' in infos['metadata']: - infos['metadata']['ended_at'] = infos['metadata']['ended_at'] if 'log_path' in metadata: log_path = metadata['log_path'] except yaml.YAMLError: From 76be566607405618ae535e7c8884125403044cf7 Mon Sep 17 00:00:00 2001 From: ljf Date: Mon, 10 Dec 2018 01:19:14 +0100 Subject: [PATCH 9/9] [fix] Bad time convesion in a debug message --- src/yunohost/backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yunohost/backup.py b/src/yunohost/backup.py index 528b2e24f..33fbccb7f 100644 --- a/src/yunohost/backup.py +++ b/src/yunohost/backup.py @@ -860,7 +860,7 @@ class RestoreManager(): raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive')) else: logger.debug("restoring from backup '%s' created on %s", self.name, - time.ctime(self.info['created_at'])) + datetime.utcfromtimestamp(self.info['created_at'])) def _postinstall_if_needed(self): """