mirror of
https://github.com/YunoHost/moulinette.git
synced 2024-09-03 20:06:31 +02:00
Add update and show stats in monitoring functions
This commit is contained in:
Jerome Lebleu
parent
c553877d4c
commit
f93fb66e10
2 changed files with 327 additions and 0 deletions
|
|
@ -452,6 +452,7 @@ monitor:
|
|||
### monitor_disk()
|
||||
disk:
|
||||
action_help: Monitor disk space and usage
|
||||
api: GET /monitor/disk
|
||||
arguments:
|
||||
-f:
|
||||
full: --filesystem
|
||||
|
|
@ -477,6 +478,7 @@ monitor:
|
|||
### monitor_network()
|
||||
network:
|
||||
action_help: Monitor network interfaces
|
||||
api: GET /monitor/network
|
||||
arguments:
|
||||
-u:
|
||||
full: --usage
|
||||
|
|
@ -498,6 +500,7 @@ monitor:
|
|||
### monitor_system()
|
||||
system:
|
||||
action_help: Monitor system informations and usage
|
||||
api: GET /monitor/system
|
||||
arguments:
|
||||
-m:
|
||||
full: --memory
|
||||
|
|
@ -534,6 +537,30 @@ monitor:
|
|||
help: Print sizes in human readable format
|
||||
action: store_true
|
||||
|
||||
### monitor_updatestats()
|
||||
updatestats:
|
||||
action_help: Update monitored statistics
|
||||
api: POST /monitor/updatestats
|
||||
arguments:
|
||||
period:
|
||||
help: Time period to update
|
||||
choices:
|
||||
- day
|
||||
- week
|
||||
- month
|
||||
|
||||
### monitor_showstats()
|
||||
showstats:
|
||||
action_help: Show monitored statistics
|
||||
api: GET /monitor/showstats
|
||||
arguments:
|
||||
period:
|
||||
help: Time period to show
|
||||
choices:
|
||||
- day
|
||||
- week
|
||||
- month
|
||||
|
||||
|
||||
#############################
|
||||
# Service #
|
||||
|
|
|
|||
|
|
@ -25,14 +25,19 @@
|
|||
"""
|
||||
import re
|
||||
import json
|
||||
import time
|
||||
import pickle
|
||||
import psutil
|
||||
import calendar
|
||||
import subprocess
|
||||
import xmlrpclib
|
||||
import os.path
|
||||
from urllib import urlopen
|
||||
from datetime import datetime, timedelta
|
||||
from yunohost import YunoHostError
|
||||
|
||||
glances_uri = 'http://127.0.0.1:61209'
|
||||
stats_path = '/var/lib/yunohost/stats'
|
||||
|
||||
def monitor_disk(units=None, mountpoint=None, human_readable=False):
|
||||
"""
|
||||
|
|
@ -229,6 +234,108 @@ def monitor_system(units=None, human_readable=False):
|
|||
return result
|
||||
|
||||
|
||||
def monitor_updatestats(period):
|
||||
"""
|
||||
Update monitored statistics
|
||||
|
||||
Keyword argument:
|
||||
period -- Time period to update (day, week, month)
|
||||
|
||||
"""
|
||||
if period not in ['day', 'week', 'month']:
|
||||
raise YunoHostError(22, _("Invalid period"))
|
||||
|
||||
stats = _retrieve_stats(period)
|
||||
if not stats:
|
||||
stats = { 'disk': {}, 'network': {}, 'system': {}, 'timestamp': [] }
|
||||
|
||||
monitor = None
|
||||
# Get monitored stats
|
||||
if period == 'day':
|
||||
monitor = _monitor_all('day')
|
||||
else:
|
||||
t = stats['timestamp']
|
||||
p = 'day' if period == 'week' else 'week'
|
||||
if len(t) > 0:
|
||||
monitor = _monitor_all(p, t[len(t) - 1])
|
||||
else:
|
||||
monitor = _monitor_all(p, 0)
|
||||
if not monitor:
|
||||
raise YunoHostError(1, _("No monitored statistics to update"))
|
||||
|
||||
stats['timestamp'].append(time.time())
|
||||
|
||||
# Append disk stats
|
||||
for dname, units in monitor['disk'].items():
|
||||
disk = {}
|
||||
# Retrieve current stats for disk name
|
||||
if dname in stats['disk'].keys():
|
||||
disk = stats['disk'][dname]
|
||||
|
||||
for unit, values in units.items():
|
||||
# Continue if unit doesn't contain stats
|
||||
if not isinstance(values, dict):
|
||||
continue
|
||||
|
||||
# Retrieve current stats for unit and append new ones
|
||||
curr = disk[unit] if unit in disk.keys() else {}
|
||||
if unit == 'io':
|
||||
disk[unit] = _append_to_stats(curr, values, 'time_since_update')
|
||||
elif unit == 'filesystem':
|
||||
disk[unit] = _append_to_stats(curr, values, ['fs_type', 'mnt_point'])
|
||||
stats['disk'][dname] = disk
|
||||
|
||||
# Append network stats
|
||||
net_usage = {}
|
||||
for iname, values in monitor['network']['usage'].items():
|
||||
# Continue if units doesn't contain stats
|
||||
if not isinstance(values, dict):
|
||||
continue
|
||||
|
||||
# Retrieve current stats and append new ones
|
||||
curr = {}
|
||||
if 'usage' in stats['network'] and iname in stats['network']['usage']:
|
||||
curr = stats['network']['usage'][iname]
|
||||
net_usage[iname] = _append_to_stats(curr, values, 'time_since_update')
|
||||
stats['network'] = { 'usage': net_usage, 'infos': monitor['network']['infos'] }
|
||||
|
||||
# Append system stats
|
||||
for unit, values in monitor['system'].items():
|
||||
# Continue if units doesn't contain stats
|
||||
if not isinstance(values, dict):
|
||||
continue
|
||||
|
||||
# Set static infos unit
|
||||
if unit == 'infos':
|
||||
stats['system'][unit] = values
|
||||
continue
|
||||
|
||||
# Retrieve current stats and append new ones
|
||||
curr = stats['system'][unit] if unit in stats['system'].keys() else {}
|
||||
stats['system'][unit] = _append_to_stats(curr, values)
|
||||
|
||||
_save_stats(stats, period)
|
||||
|
||||
|
||||
def monitor_showstats(period, date=None):
|
||||
"""
|
||||
Show monitored statistics
|
||||
|
||||
Keyword argument:
|
||||
period -- Time period to show (day, week, month)
|
||||
|
||||
"""
|
||||
if period not in ['day', 'week', 'month']:
|
||||
raise YunoHostError(22, _("Invalid period"))
|
||||
|
||||
result = _retrieve_stats(period, date)
|
||||
if result is False:
|
||||
raise YunoHostError(167, _("Stats file not found"))
|
||||
elif result is None:
|
||||
raise YunoHostError(1, _("No available stats for the given period"))
|
||||
return result
|
||||
|
||||
|
||||
def _get_glances_api():
|
||||
"""
|
||||
Retrieve Glances API running on the local server
|
||||
|
|
@ -292,3 +399,196 @@ def _binary_to_human(n, customary=False):
|
|||
value = float(n) / prefix[s]
|
||||
return '%.1f%s' % (value, s)
|
||||
return "%s" % n
|
||||
|
||||
|
||||
def _retrieve_stats(period, date=None):
|
||||
"""
|
||||
Retrieve statistics from pickle file
|
||||
|
||||
Keyword argument:
|
||||
period -- Time period to retrieve (day, week, month)
|
||||
date -- Date of stats to retrieve
|
||||
|
||||
"""
|
||||
pkl_file = None
|
||||
|
||||
# Retrieve pickle file
|
||||
if date is not None:
|
||||
timestamp = calendar.timegm(date)
|
||||
pkl_file = '%s/%d_%s.pkl' % (stats_path, timestamp, period)
|
||||
else:
|
||||
pkl_file = '%s/%s.pkl' % (stats_path, period)
|
||||
if not os.path.isfile(pkl_file):
|
||||
return False
|
||||
|
||||
# Read file and process its content
|
||||
with open(pkl_file, 'r') as f:
|
||||
result = pickle.load(f)
|
||||
if not isinstance(result, dict):
|
||||
return None
|
||||
return result
|
||||
|
||||
|
||||
def _save_stats(stats, period, date=None):
|
||||
"""
|
||||
Save statistics to pickle file
|
||||
|
||||
Keyword argument:
|
||||
stats -- Stats dict to save
|
||||
period -- Time period of stats (day, week, month)
|
||||
date -- Date of stats
|
||||
|
||||
"""
|
||||
pkl_file = None
|
||||
|
||||
# Set pickle file name
|
||||
if date is not None:
|
||||
timestamp = calendar.timegm(date)
|
||||
pkl_file = '%s/%d_%s.pkl' % (stats_path, timestamp, period)
|
||||
else:
|
||||
pkl_file = '%s/%s.pkl' % (stats_path, period)
|
||||
if not os.path.isdir(stats_path):
|
||||
os.makedirs(stats_path)
|
||||
|
||||
# Write file content
|
||||
with open(pkl_file, 'w') as f:
|
||||
pickle.dump(stats, f)
|
||||
return True
|
||||
|
||||
|
||||
def _monitor_all(period=None, since=None):
|
||||
"""
|
||||
Monitor all units (disk, network and system) for the given period
|
||||
If since is None, real-time monitoring is returned. Otherwise, the
|
||||
mean of stats since this timestamp is calculated and returned.
|
||||
|
||||
Keyword argument:
|
||||
period -- Time period to monitor (day, week, month)
|
||||
since -- Timestamp of the stats beginning
|
||||
|
||||
"""
|
||||
result = { 'disk': {}, 'network': {}, 'system': {} }
|
||||
|
||||
# Real-time stats
|
||||
if period == 'day' and since is None:
|
||||
result['disk'] = monitor_disk()
|
||||
result['network'] = monitor_network()
|
||||
result['system'] = monitor_system()
|
||||
return result
|
||||
|
||||
# Retrieve stats and calculate mean
|
||||
stats = _retrieve_stats(period)
|
||||
if not stats:
|
||||
return None
|
||||
stats = _filter_stats(stats, since)
|
||||
if not stats:
|
||||
return None
|
||||
result = _calculate_stats_mean(stats)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _filter_stats(stats, t_begin=None, t_end=None):
|
||||
"""
|
||||
Filter statistics by beginning and/or ending timestamp
|
||||
|
||||
Keyword argument:
|
||||
stats -- Dict stats to filter
|
||||
t_begin -- Beginning timestamp
|
||||
t_end -- Ending timestamp
|
||||
|
||||
"""
|
||||
if t_begin is None and t_end is None:
|
||||
return stats
|
||||
|
||||
i_begin = i_end = None
|
||||
# Look for indexes of timestamp interval
|
||||
for i, t in enumerate(stats['timestamp']):
|
||||
if t_begin and i_begin is None and t >= t_begin:
|
||||
i_begin = i
|
||||
if t_end and i != 0 and i_end is None and t > t_end:
|
||||
i_end = i - 1
|
||||
# Check indexes
|
||||
if i_begin is None:
|
||||
if t_begin and t_begin > stats['timestamp'][0]:
|
||||
return None
|
||||
i_begin = 0
|
||||
if i_end is None:
|
||||
if t_end and t_end < stats['timestamp'][0]:
|
||||
return None
|
||||
i_end = len(stats['timestamp']) - 1
|
||||
if i_begin == 0 and i_end == (len(stats['timestamp']) - 1):
|
||||
return stats
|
||||
|
||||
# Filter function
|
||||
def _filter(s, i, j):
|
||||
for k, v in s.items():
|
||||
if isinstance(v, dict):
|
||||
s[k] = _filter(v, i, j)
|
||||
elif isinstance(v, list):
|
||||
s[k] = v[i:j] if i != j else [v[i]]
|
||||
return s
|
||||
|
||||
stats = _filter(stats, i_begin, i_end)
|
||||
return stats
|
||||
|
||||
|
||||
def _calculate_stats_mean(stats):
|
||||
"""
|
||||
Calculate the weighted mean for each statistic
|
||||
|
||||
Keyword argument:
|
||||
stats -- Stats dict to process
|
||||
|
||||
"""
|
||||
timestamp = stats['timestamp']
|
||||
t_sum = sum(timestamp)
|
||||
del stats['timestamp']
|
||||
|
||||
# Weighted mean function
|
||||
def _mean(s, t, ts):
|
||||
for k, v in s.items():
|
||||
if isinstance(v, dict):
|
||||
s[k] = _mean(v, t, ts)
|
||||
elif isinstance(v, list):
|
||||
nums = [ float(x * t[i]) for i, x in enumerate(v) ]
|
||||
s[k] = sum(nums) / float(ts)
|
||||
return s
|
||||
|
||||
stats = _mean(stats, timestamp, t_sum)
|
||||
return stats
|
||||
|
||||
|
||||
def _append_to_stats(stats, monitor, statics=[]):
|
||||
"""
|
||||
Append monitored statistics to current statistics
|
||||
|
||||
Keyword argument:
|
||||
stats -- Current stats dict
|
||||
monitor -- Monitored statistics
|
||||
statics -- List of stats static keys
|
||||
|
||||
"""
|
||||
if isinstance(statics, str):
|
||||
statics = [statics]
|
||||
|
||||
# Appending function
|
||||
def _append(s, m, st):
|
||||
for k, v in m.items():
|
||||
if k in st:
|
||||
s[k] = v
|
||||
elif isinstance(v, dict):
|
||||
if k not in s:
|
||||
s[k] = {}
|
||||
s[k] = _append(s[k], v, st)
|
||||
else:
|
||||
if k not in s:
|
||||
s[k] = []
|
||||
if isinstance(v, list):
|
||||
s[k].extend(v)
|
||||
else:
|
||||
s[k].append(v)
|
||||
return s
|
||||
|
||||
stats = _append(stats, monitor, statics)
|
||||
return stats
|
||||
|
|
|
|||
Loading…
Reference in a new issue