diff --git a/action_map.yml b/action_map.yml index 0e5d7f1d..62860373 100644 --- a/action_map.yml +++ b/action_map.yml @@ -452,6 +452,7 @@ monitor: ### monitor_disk() disk: action_help: Monitor disk space and usage + api: GET /monitor/disk arguments: -f: full: --filesystem @@ -477,6 +478,7 @@ monitor: ### monitor_network() network: action_help: Monitor network interfaces + api: GET /monitor/network arguments: -u: full: --usage @@ -498,6 +500,7 @@ monitor: ### monitor_system() system: action_help: Monitor system informations and usage + api: GET /monitor/system arguments: -m: full: --memory @@ -534,6 +537,30 @@ monitor: help: Print sizes in human readable format action: store_true + ### monitor_updatestats() + updatestats: + action_help: Update monitored statistics + api: POST /monitor/updatestats + arguments: + period: + help: Time period to update + choices: + - day + - week + - month + + ### monitor_showstats() + showstats: + action_help: Show monitored statistics + api: GET /monitor/showstats + arguments: + period: + help: Time period to show + choices: + - day + - week + - month + ############################# # Service # diff --git a/yunohost_monitor.py b/yunohost_monitor.py index 43c6ac39..ae035a81 100644 --- a/yunohost_monitor.py +++ b/yunohost_monitor.py @@ -25,14 +25,19 @@ """ import re import json +import time +import pickle import psutil +import calendar import subprocess import xmlrpclib +import os.path from urllib import urlopen from datetime import datetime, timedelta from yunohost import YunoHostError glances_uri = 'http://127.0.0.1:61209' +stats_path = '/var/lib/yunohost/stats' def monitor_disk(units=None, mountpoint=None, human_readable=False): """ @@ -229,6 +234,108 @@ def monitor_system(units=None, human_readable=False): return result +def monitor_updatestats(period): + """ + Update monitored statistics + + Keyword argument: + period -- Time period to update (day, week, month) + + """ + if period not in ['day', 'week', 'month']: + raise YunoHostError(22, _("Invalid period")) + + stats = _retrieve_stats(period) + if not stats: + stats = { 'disk': {}, 'network': {}, 'system': {}, 'timestamp': [] } + + monitor = None + # Get monitored stats + if period == 'day': + monitor = _monitor_all('day') + else: + t = stats['timestamp'] + p = 'day' if period == 'week' else 'week' + if len(t) > 0: + monitor = _monitor_all(p, t[len(t) - 1]) + else: + monitor = _monitor_all(p, 0) + if not monitor: + raise YunoHostError(1, _("No monitored statistics to update")) + + stats['timestamp'].append(time.time()) + + # Append disk stats + for dname, units in monitor['disk'].items(): + disk = {} + # Retrieve current stats for disk name + if dname in stats['disk'].keys(): + disk = stats['disk'][dname] + + for unit, values in units.items(): + # Continue if unit doesn't contain stats + if not isinstance(values, dict): + continue + + # Retrieve current stats for unit and append new ones + curr = disk[unit] if unit in disk.keys() else {} + if unit == 'io': + disk[unit] = _append_to_stats(curr, values, 'time_since_update') + elif unit == 'filesystem': + disk[unit] = _append_to_stats(curr, values, ['fs_type', 'mnt_point']) + stats['disk'][dname] = disk + + # Append network stats + net_usage = {} + for iname, values in monitor['network']['usage'].items(): + # Continue if units doesn't contain stats + if not isinstance(values, dict): + continue + + # Retrieve current stats and append new ones + curr = {} + if 'usage' in stats['network'] and iname in stats['network']['usage']: + curr = stats['network']['usage'][iname] + net_usage[iname] = _append_to_stats(curr, values, 'time_since_update') + stats['network'] = { 'usage': net_usage, 'infos': monitor['network']['infos'] } + + # Append system stats + for unit, values in monitor['system'].items(): + # Continue if units doesn't contain stats + if not isinstance(values, dict): + continue + + # Set static infos unit + if unit == 'infos': + stats['system'][unit] = values + continue + + # Retrieve current stats and append new ones + curr = stats['system'][unit] if unit in stats['system'].keys() else {} + stats['system'][unit] = _append_to_stats(curr, values) + + _save_stats(stats, period) + + +def monitor_showstats(period, date=None): + """ + Show monitored statistics + + Keyword argument: + period -- Time period to show (day, week, month) + + """ + if period not in ['day', 'week', 'month']: + raise YunoHostError(22, _("Invalid period")) + + result = _retrieve_stats(period, date) + if result is False: + raise YunoHostError(167, _("Stats file not found")) + elif result is None: + raise YunoHostError(1, _("No available stats for the given period")) + return result + + def _get_glances_api(): """ Retrieve Glances API running on the local server @@ -292,3 +399,196 @@ def _binary_to_human(n, customary=False): value = float(n) / prefix[s] return '%.1f%s' % (value, s) return "%s" % n + + +def _retrieve_stats(period, date=None): + """ + Retrieve statistics from pickle file + + Keyword argument: + period -- Time period to retrieve (day, week, month) + date -- Date of stats to retrieve + + """ + pkl_file = None + + # Retrieve pickle file + if date is not None: + timestamp = calendar.timegm(date) + pkl_file = '%s/%d_%s.pkl' % (stats_path, timestamp, period) + else: + pkl_file = '%s/%s.pkl' % (stats_path, period) + if not os.path.isfile(pkl_file): + return False + + # Read file and process its content + with open(pkl_file, 'r') as f: + result = pickle.load(f) + if not isinstance(result, dict): + return None + return result + + +def _save_stats(stats, period, date=None): + """ + Save statistics to pickle file + + Keyword argument: + stats -- Stats dict to save + period -- Time period of stats (day, week, month) + date -- Date of stats + + """ + pkl_file = None + + # Set pickle file name + if date is not None: + timestamp = calendar.timegm(date) + pkl_file = '%s/%d_%s.pkl' % (stats_path, timestamp, period) + else: + pkl_file = '%s/%s.pkl' % (stats_path, period) + if not os.path.isdir(stats_path): + os.makedirs(stats_path) + + # Write file content + with open(pkl_file, 'w') as f: + pickle.dump(stats, f) + return True + + +def _monitor_all(period=None, since=None): + """ + Monitor all units (disk, network and system) for the given period + If since is None, real-time monitoring is returned. Otherwise, the + mean of stats since this timestamp is calculated and returned. + + Keyword argument: + period -- Time period to monitor (day, week, month) + since -- Timestamp of the stats beginning + + """ + result = { 'disk': {}, 'network': {}, 'system': {} } + + # Real-time stats + if period == 'day' and since is None: + result['disk'] = monitor_disk() + result['network'] = monitor_network() + result['system'] = monitor_system() + return result + + # Retrieve stats and calculate mean + stats = _retrieve_stats(period) + if not stats: + return None + stats = _filter_stats(stats, since) + if not stats: + return None + result = _calculate_stats_mean(stats) + + return result + + +def _filter_stats(stats, t_begin=None, t_end=None): + """ + Filter statistics by beginning and/or ending timestamp + + Keyword argument: + stats -- Dict stats to filter + t_begin -- Beginning timestamp + t_end -- Ending timestamp + + """ + if t_begin is None and t_end is None: + return stats + + i_begin = i_end = None + # Look for indexes of timestamp interval + for i, t in enumerate(stats['timestamp']): + if t_begin and i_begin is None and t >= t_begin: + i_begin = i + if t_end and i != 0 and i_end is None and t > t_end: + i_end = i - 1 + # Check indexes + if i_begin is None: + if t_begin and t_begin > stats['timestamp'][0]: + return None + i_begin = 0 + if i_end is None: + if t_end and t_end < stats['timestamp'][0]: + return None + i_end = len(stats['timestamp']) - 1 + if i_begin == 0 and i_end == (len(stats['timestamp']) - 1): + return stats + + # Filter function + def _filter(s, i, j): + for k, v in s.items(): + if isinstance(v, dict): + s[k] = _filter(v, i, j) + elif isinstance(v, list): + s[k] = v[i:j] if i != j else [v[i]] + return s + + stats = _filter(stats, i_begin, i_end) + return stats + + +def _calculate_stats_mean(stats): + """ + Calculate the weighted mean for each statistic + + Keyword argument: + stats -- Stats dict to process + + """ + timestamp = stats['timestamp'] + t_sum = sum(timestamp) + del stats['timestamp'] + + # Weighted mean function + def _mean(s, t, ts): + for k, v in s.items(): + if isinstance(v, dict): + s[k] = _mean(v, t, ts) + elif isinstance(v, list): + nums = [ float(x * t[i]) for i, x in enumerate(v) ] + s[k] = sum(nums) / float(ts) + return s + + stats = _mean(stats, timestamp, t_sum) + return stats + + +def _append_to_stats(stats, monitor, statics=[]): + """ + Append monitored statistics to current statistics + + Keyword argument: + stats -- Current stats dict + monitor -- Monitored statistics + statics -- List of stats static keys + + """ + if isinstance(statics, str): + statics = [statics] + + # Appending function + def _append(s, m, st): + for k, v in m.items(): + if k in st: + s[k] = v + elif isinstance(v, dict): + if k not in s: + s[k] = {} + s[k] = _append(s[k], v, st) + else: + if k not in s: + s[k] = [] + if isinstance(v, list): + s[k].extend(v) + else: + s[k].append(v) + return s + + stats = _append(stats, monitor, statics) + return stats