mirror of
https://github.com/YunoHost/yunohost.git
synced 2024-09-03 20:06:10 +02:00
[enh] Refactor applist management (#160)
* [mod] directly use python to retreive json list * [enh] app_fetchlist fetch all app_list by default * [fix] name variable doesn't exists here * [fix] re returns None when there is not matchs * [enh] app_fetchlist fetch all app_list by default * Some cleaning for better readability * Simpler variable name * Prepare a function that register lists to be fetched * Skeletong for applist system migration * Add implementation of migration system with tests * Refactorize app_fetchlist * Misc fixes + adding test for single app fetching * Fixing a few issues + test removelist * Adding fetchlist and cron install during postinstall * Adding debug messages * Adding particular exception for SSL connection error * Update actionmap help * We don't use urlretrieve * Clean tests, some description were bad * [mod] some cleaning * Moving to a .json file to store lists url + adjusting tests * Adding missing string in locale * Moving exception to logger.error when fetching fails * Adding name of applist in error messages * Fixing cron job stuff + adding proper tests * Using None instead of -1 for applist lastupdate * Handling exceptions when writing applist files * More exception handling... * [mod] pep8 * Updating test for migration of conflicting lists * More general error when return code is not 200 * [enh] Improve app_fetchlist help. * [fix] Use appslist instead of applist. * [fix] Consistent user string for translation.
This commit is contained in:
Laurent Peuch
Alexandre Aubin
parent
c954429cca
commit
a4c487a0aa
5 changed files with 693 additions and 77 deletions
|
|
@ -391,28 +391,28 @@ app:
|
|||
|
||||
### app_fetchlist()
|
||||
fetchlist:
|
||||
action_help: Fetch application list from app server
|
||||
action_help: Fetch application lists from app servers, or register a new one.
|
||||
api: PUT /appslists
|
||||
arguments:
|
||||
-u:
|
||||
full: --url
|
||||
help: URL of remote JSON list (default https://app.yunohost.org/official.json)
|
||||
-n:
|
||||
full: --name
|
||||
help: Name of the list (default yunohost)
|
||||
help: Name of the list to fetch (fetches all registered lists if empty)
|
||||
extra:
|
||||
pattern: &pattern_listname
|
||||
- !!str ^[a-z0-9_]+$
|
||||
- "pattern_listname"
|
||||
-u:
|
||||
full: --url
|
||||
help: URL of a new application list to register. To be specified with -n.
|
||||
|
||||
### app_listlists()
|
||||
listlists:
|
||||
action_help: List fetched lists
|
||||
action_help: List registered application lists
|
||||
api: GET /appslists
|
||||
|
||||
### app_removelist()
|
||||
removelist:
|
||||
action_help: Remove list from the repositories
|
||||
action_help: Remove and forget about a given application list
|
||||
api: DELETE /appslists
|
||||
arguments:
|
||||
name:
|
||||
|
|
|
|||
|
|
@ -28,11 +28,16 @@
|
|||
"app_unsupported_remote_type": "Unsupported remote type used for the app",
|
||||
"app_upgrade_failed": "Unable to upgrade {app:s}",
|
||||
"app_upgraded": "{app:s} has been upgraded",
|
||||
"appslist_fetched": "The app list has been fetched",
|
||||
"appslist_removed": "The app list has been removed",
|
||||
"appslist_retrieve_error": "Unable to retrieve the remote app list: {error}",
|
||||
"appslist_retrieve_bad_format": "Retrieved file is not a valid app list",
|
||||
"appslist_unknown": "Unknown app list",
|
||||
"appslist_fetched": "The application list {appslist:s} has been fetched",
|
||||
"appslist_removed": "The application list {appslist:s} has been removed",
|
||||
"appslist_unknown": "Application list {appslist:s} unknown.",
|
||||
"appslist_retrieve_error": "Unable to retrieve the remote application list {appslist:s}: {error:s}",
|
||||
"appslist_retrieve_bad_format": "Retrieved file for application list {appslist:s} is not a valid app list",
|
||||
"appslist_name_already_tracked": "There is already a registered application list with name {name:s}.",
|
||||
"appslist_url_already_tracked": "There is already a registered application list with url {url:s}.",
|
||||
"appslist_migrating": "Migrating application list {appslist:s} ...",
|
||||
"appslist_could_not_migrate": "Could not migrate app list {appslist:s} ! Unable to parse the url... The old cron job has been kept in {bkp_file:s}.",
|
||||
"appslist_corrupted_json": "Could not load the application lists. It looks like {filename:s} is corrupted.",
|
||||
"ask_current_admin_password": "Current administration password",
|
||||
"ask_email": "Email address",
|
||||
"ask_firstname": "First name",
|
||||
|
|
|
|||
|
|
@ -34,6 +34,9 @@ import urlparse
|
|||
import errno
|
||||
import subprocess
|
||||
import requests
|
||||
import glob
|
||||
import pwd
|
||||
import grp
|
||||
from collections import OrderedDict
|
||||
|
||||
from moulinette.core import MoulinetteError
|
||||
|
|
@ -49,6 +52,7 @@ APPS_PATH = '/usr/share/yunohost/apps'
|
|||
APPS_SETTING_PATH = '/etc/yunohost/apps/'
|
||||
INSTALL_TMP = '/var/cache/yunohost'
|
||||
APP_TMP_FOLDER = INSTALL_TMP + '/from_file'
|
||||
APPSLISTS_JSON = '/etc/yunohost/appslists.json'
|
||||
|
||||
re_github_repo = re.compile(
|
||||
r'^(http[s]?://|git@)github.com[/:]'
|
||||
|
|
@ -65,66 +69,122 @@ def app_listlists():
|
|||
"""
|
||||
List fetched lists
|
||||
|
||||
|
||||
"""
|
||||
list_list = []
|
||||
try:
|
||||
for filename in os.listdir(REPO_PATH):
|
||||
if '.json' in filename:
|
||||
list_list.append(filename[:len(filename)-5])
|
||||
except OSError:
|
||||
raise MoulinetteError(1, m18n.n('no_appslist_found'))
|
||||
|
||||
return { 'lists' : list_list }
|
||||
# Migrate appslist system if needed
|
||||
# XXX move to a migration when those are implemented
|
||||
if _using_legacy_appslist_system():
|
||||
_migrate_appslist_system()
|
||||
|
||||
# Get the list
|
||||
appslist_list = _read_appslist_list()
|
||||
|
||||
return appslist_list
|
||||
|
||||
|
||||
def app_fetchlist(url=None, name=None):
|
||||
"""
|
||||
Fetch application list from app server
|
||||
Fetch application list(s) from app server. By default, fetch all lists.
|
||||
|
||||
Keyword argument:
|
||||
name -- Name of the list (default yunohost)
|
||||
url -- URL of remote JSON list (default https://app.yunohost.org/official.json)
|
||||
|
||||
name -- Name of the list
|
||||
url -- URL of remote JSON list
|
||||
"""
|
||||
# Create app path if not exists
|
||||
# If needed, create folder where actual appslists are stored
|
||||
if not os.path.exists(REPO_PATH):
|
||||
os.makedirs(REPO_PATH)
|
||||
|
||||
if url is None:
|
||||
url = 'https://app.yunohost.org/official.json'
|
||||
name = 'yunohost'
|
||||
elif name is None:
|
||||
# Migrate appslist system if needed
|
||||
# XXX move that to a migration once they are finished
|
||||
if _using_legacy_appslist_system():
|
||||
_migrate_appslist_system()
|
||||
|
||||
# Read the list of appslist...
|
||||
appslists = _read_appslist_list()
|
||||
|
||||
# Determine the list of appslist to be fetched
|
||||
appslists_to_be_fetched = []
|
||||
|
||||
# If a url and and a name is given, try to register new list,
|
||||
# the fetch only this list
|
||||
if url is not None:
|
||||
if name:
|
||||
_register_new_appslist(url, name)
|
||||
# Refresh the appslists dict
|
||||
appslists = _read_appslist_list()
|
||||
appslists_to_be_fetched = [name]
|
||||
else:
|
||||
raise MoulinetteError(errno.EINVAL,
|
||||
m18n.n('custom_appslist_name_required'))
|
||||
|
||||
# If a name is given, look for an appslist with that name and fetch it
|
||||
elif name is not None:
|
||||
if name not in appslists.keys():
|
||||
raise MoulinetteError(errno.EINVAL,
|
||||
m18n.n('appslist_unknown', appslist=name))
|
||||
else:
|
||||
appslists_to_be_fetched = [name]
|
||||
|
||||
# Otherwise, fetch all lists
|
||||
else:
|
||||
appslists_to_be_fetched = appslists.keys()
|
||||
|
||||
# Fetch all appslists to be fetched
|
||||
for name in appslists_to_be_fetched:
|
||||
|
||||
url = appslists[name]["url"]
|
||||
|
||||
logger.debug("Attempting to fetch list %s at %s" % (name, url))
|
||||
|
||||
# Download file
|
||||
try:
|
||||
applist_request = requests.get(url, timeout=30)
|
||||
appslist_request = requests.get(url, timeout=30)
|
||||
except requests.exceptions.SSLError:
|
||||
logger.error(m18n.n('appslist_retrieve_error',
|
||||
appslist=name,
|
||||
error="SSL connection error"))
|
||||
continue
|
||||
except Exception as e:
|
||||
raise MoulinetteError(errno.EBADR, m18n.n('appslist_retrieve_error', error=str(e)))
|
||||
|
||||
if (applist_request.status_code != 200):
|
||||
raise MoulinetteError(errno.EBADR, m18n.n('appslist_retrieve_error', error="404, not found"))
|
||||
logger.error(m18n.n('appslist_retrieve_error',
|
||||
appslist=name,
|
||||
error=str(e)))
|
||||
continue
|
||||
if appslist_request.status_code != 200:
|
||||
logger.error(m18n.n('appslist_retrieve_error',
|
||||
appslist=name,
|
||||
error="Server returned code %s " %
|
||||
str(appslist_request.status_code)))
|
||||
continue
|
||||
|
||||
# Validate app list format
|
||||
# TODO / Possible improvement : better validation for app list (check that
|
||||
# json fields actually look like an app list and not any json file)
|
||||
applist = applist_request.text
|
||||
# TODO / Possible improvement : better validation for app list (check
|
||||
# that json fields actually look like an app list and not any json
|
||||
# file)
|
||||
appslist = appslist_request.text
|
||||
try:
|
||||
json.loads(applist)
|
||||
json.loads(appslist)
|
||||
except ValueError, e:
|
||||
raise MoulinetteError(errno.EBADR, m18n.n('appslist_retrieve_bad_format'))
|
||||
logger.error(m18n.n('appslist_retrieve_bad_format',
|
||||
appslist=name))
|
||||
continue
|
||||
|
||||
# Write app list to file
|
||||
list_file = '%s/%s.json' % (REPO_PATH, name)
|
||||
try:
|
||||
with open(list_file, "w") as f:
|
||||
f.write(applist)
|
||||
f.write(appslist)
|
||||
except Exception as e:
|
||||
raise MoulinetteError(errno.EIO,
|
||||
"Error while writing appslist %s: %s" %
|
||||
(name, str(e)))
|
||||
|
||||
# Setup a cron job to re-fetch the list at midnight
|
||||
open("/etc/cron.d/yunohost-applist-%s" % name, "w").write('00 00 * * * root yunohost app fetchlist -u %s -n %s > /dev/null 2>&1\n' % (url, name))
|
||||
now = int(time.time())
|
||||
appslists[name]["lastUpdate"] = now
|
||||
|
||||
logger.success(m18n.n('appslist_fetched'))
|
||||
logger.success(m18n.n('appslist_fetched', appslist=name))
|
||||
|
||||
# Write updated list of appslist
|
||||
_write_appslist_list(appslists)
|
||||
|
||||
|
||||
def app_removelist(name):
|
||||
|
|
@ -135,13 +195,22 @@ def app_removelist(name):
|
|||
name -- Name of the list to remove
|
||||
|
||||
"""
|
||||
try:
|
||||
os.remove('%s/%s.json' % (REPO_PATH, name))
|
||||
os.remove("/etc/cron.d/yunohost-applist-%s" % name)
|
||||
except OSError:
|
||||
raise MoulinetteError(errno.ENOENT, m18n.n('appslist_unknown'))
|
||||
appslists = _read_appslist_list()
|
||||
|
||||
logger.success(m18n.n('appslist_removed'))
|
||||
# Make sure we know this appslist
|
||||
if name not in appslists.keys():
|
||||
raise MoulinetteError(errno.ENOENT, m18n.n('appslist_unknown', appslist=name))
|
||||
|
||||
# Remove json
|
||||
json_path = '%s/%s.json' % (REPO_PATH, name)
|
||||
if os.path.exists(json_path):
|
||||
os.remove(json_path)
|
||||
|
||||
# Forget about this appslist
|
||||
del appslists[name]
|
||||
_write_appslist_list(appslists)
|
||||
|
||||
logger.success(m18n.n('appslist_removed', appslist=name))
|
||||
|
||||
|
||||
def app_list(filter=None, raw=False, installed=False, with_backup=False):
|
||||
|
|
@ -162,19 +231,18 @@ def app_list(filter=None, raw=False, installed=False, with_backup=False):
|
|||
app_dict = {}
|
||||
list_dict = {} if raw else []
|
||||
|
||||
try:
|
||||
applists = app_listlists()['lists']
|
||||
applists[0]
|
||||
except (IOError, IndexError):
|
||||
app_fetchlist()
|
||||
applists = app_listlists()['lists']
|
||||
appslists = _read_appslist_list()
|
||||
|
||||
# Construct a dictionnary of apps, based on known app lists
|
||||
for applist in applists:
|
||||
with open(os.path.join(REPO_PATH, applist + '.json')) as json_list:
|
||||
for app, info in json.load(json_list).items():
|
||||
for appslist in appslists.keys():
|
||||
|
||||
json_path = "%s/%s.json" % (REPO_PATH, appslist)
|
||||
if not os.path.exists(json_path):
|
||||
app_fetchlist(name=appslist)
|
||||
|
||||
with open(json_path) as json_list:
|
||||
for app, info in json.loads(str(json_list.read())).items():
|
||||
if app not in app_dict:
|
||||
info['repository'] = applist
|
||||
info['repository'] = appslist
|
||||
app_dict[app] = info
|
||||
|
||||
# Get app list from the app settings directory
|
||||
|
|
@ -1657,6 +1725,151 @@ def _parse_app_instance_name(app_instance_name):
|
|||
app_instance_nb = int(match.groupdict().get('appinstancenb')) if match.groupdict().get('appinstancenb') is not None else 1
|
||||
return (appid, app_instance_nb)
|
||||
|
||||
|
||||
def _using_legacy_appslist_system():
|
||||
"""
|
||||
Return True if we're using the old fetchlist scheme.
|
||||
This is determined by the presence of some cron job yunohost-appslist-foo
|
||||
"""
|
||||
|
||||
return glob.glob("/etc/cron.d/yunohost-appslist-*") != []
|
||||
|
||||
|
||||
def _migrate_appslist_system():
|
||||
"""
|
||||
Migrate from the legacy fetchlist system to the new one
|
||||
"""
|
||||
legacy_crons = glob.glob("/etc/cron.d/yunohost-appslist-*")
|
||||
|
||||
for cron_path in legacy_crons:
|
||||
appslist_name = os.path.basename(cron_path).replace("yunohost-appslist-", "")
|
||||
logger.info(m18n.n('appslist_migrating', appslist=appslist_name))
|
||||
|
||||
# Parse appslist url in cron
|
||||
cron_file_content = open(cron_path).read().strip()
|
||||
appslist_url_parse = re.search("-u (https?://[^ ]+)", cron_file_content)
|
||||
|
||||
# Abort if we did not find an url
|
||||
if not appslist_url_parse or not appslist_url_parse.groups():
|
||||
# Bkp the old cron job somewhere else
|
||||
bkp_file = "/etc/yunohost/%s.oldlist.bkp" % appslist_name
|
||||
os.rename(cron_path, bkp_file)
|
||||
# Notice the user
|
||||
logger.warning(m18n.n('appslist_could_not_migrate',
|
||||
appslist=appslist_name,
|
||||
bkp_file=bkp_file))
|
||||
# Otherwise, register the list and remove the legacy cron
|
||||
else:
|
||||
appslist_url = appslist_url_parse.groups()[0]
|
||||
try:
|
||||
_register_new_appslist(appslist_url, appslist_name)
|
||||
# Might get an exception if two legacy cron jobs conflict
|
||||
# in terms of url...
|
||||
except Exception as e:
|
||||
logger.error(str(e))
|
||||
# Bkp the old cron job somewhere else
|
||||
bkp_file = "/etc/yunohost/%s.oldlist.bkp" % appslist_name
|
||||
os.rename(cron_path, bkp_file)
|
||||
# Notice the user
|
||||
logger.warning(m18n.n('appslist_could_not_migrate',
|
||||
appslist=appslist_name,
|
||||
bkp_file=bkp_file))
|
||||
else:
|
||||
os.remove(cron_path)
|
||||
|
||||
|
||||
def _install_appslist_fetch_cron():
|
||||
|
||||
cron_job_file = "/etc/cron.daily/yunohost-fetch-appslists"
|
||||
|
||||
logger.debug("Installing appslist fetch cron job")
|
||||
|
||||
with open(cron_job_file, "w") as f:
|
||||
f.write('#!/bin/bash\n\nyunohost app fetchlist > /dev/null 2>&1\n')
|
||||
|
||||
_set_permissions(cron_job_file, "root", "root", 0755)
|
||||
|
||||
|
||||
# FIXME - Duplicate from certificate.py, should be moved into a common helper
|
||||
# thing...
|
||||
def _set_permissions(path, user, group, permissions):
|
||||
uid = pwd.getpwnam(user).pw_uid
|
||||
gid = grp.getgrnam(group).gr_gid
|
||||
|
||||
os.chown(path, uid, gid)
|
||||
os.chmod(path, permissions)
|
||||
|
||||
|
||||
def _read_appslist_list():
|
||||
"""
|
||||
Read the json corresponding to the list of appslists
|
||||
"""
|
||||
|
||||
# If file does not exists yet, return empty dict
|
||||
if not os.path.exists(APPSLISTS_JSON):
|
||||
return {}
|
||||
|
||||
# Read file content
|
||||
with open(APPSLISTS_JSON, "r") as f:
|
||||
appslists_json = f.read()
|
||||
|
||||
# Parse json, throw exception if what we got from file is not a valid json
|
||||
try:
|
||||
appslists = json.loads(appslists_json)
|
||||
except ValueError:
|
||||
raise MoulinetteError(errno.EBADR,
|
||||
m18n.n('appslist_corrupted_json', filename=APPSLISTS_JSON))
|
||||
|
||||
return appslists
|
||||
|
||||
|
||||
def _write_appslist_list(appslist_lists):
|
||||
"""
|
||||
Update the json containing list of appslists
|
||||
"""
|
||||
|
||||
# Write appslist list
|
||||
try:
|
||||
with open(APPSLISTS_JSON, "w") as f:
|
||||
json.dump(appslist_lists, f)
|
||||
except Exception as e:
|
||||
raise MoulinetteError(errno.EIO,
|
||||
"Error while writing list of appslist %s: %s" %
|
||||
(APPSLISTS_JSON, str(e)))
|
||||
|
||||
|
||||
def _register_new_appslist(url, name):
|
||||
"""
|
||||
Add a new appslist to be fetched regularly.
|
||||
Raise an exception if url or name conflicts with an existing list.
|
||||
"""
|
||||
|
||||
appslist_list = _read_appslist_list()
|
||||
|
||||
# Check if name conflicts with an existing list
|
||||
if name in appslist_list:
|
||||
raise MoulinetteError(errno.EEXIST,
|
||||
m18n.n('appslist_name_already_tracked', name=name))
|
||||
|
||||
# Check if url conflicts with an existing list
|
||||
known_appslist_urls = [appslist["url"] for _, appslist in appslist_list.items()]
|
||||
|
||||
if url in known_appslist_urls:
|
||||
raise MoulinetteError(errno.EEXIST,
|
||||
m18n.n('appslist_url_already_tracked', url=url))
|
||||
|
||||
logger.debug("Registering new appslist %s at %s" % (name, url))
|
||||
|
||||
appslist_list[name] = {
|
||||
"url": url,
|
||||
"lastUpdate": None
|
||||
}
|
||||
|
||||
_write_appslist_list(appslist_list)
|
||||
|
||||
_install_appslist_fetch_cron()
|
||||
|
||||
|
||||
def is_true(arg):
|
||||
"""
|
||||
Convert a string into a boolean
|
||||
|
|
|
|||
389
src/yunohost/tests/test_appslist.py
Normal file
389
src/yunohost/tests/test_appslist.py
Normal file
|
|
@ -0,0 +1,389 @@
|
|||
import os
|
||||
import pytest
|
||||
import requests
|
||||
import requests_mock
|
||||
import glob
|
||||
import time
|
||||
|
||||
from moulinette.core import MoulinetteError
|
||||
|
||||
from yunohost.app import app_fetchlist, app_removelist, app_listlists, _using_legacy_appslist_system, _migrate_appslist_system, _register_new_appslist
|
||||
|
||||
URL_OFFICIAL_APP_LIST = "https://app.yunohost.org/official.json"
|
||||
REPO_PATH = '/var/cache/yunohost/repo'
|
||||
APPSLISTS_JSON = '/etc/yunohost/appslists.json'
|
||||
|
||||
|
||||
def setup_function(function):
|
||||
|
||||
# Clear all appslist
|
||||
files = glob.glob(REPO_PATH+"/*")
|
||||
for f in files:
|
||||
os.remove(f)
|
||||
|
||||
# Clear appslist crons
|
||||
files = glob.glob("/etc/cron.d/yunohost-appslist-*")
|
||||
for f in files:
|
||||
os.remove(f)
|
||||
|
||||
if os.path.exists("/etc/cron.daily/yunohost-fetch-appslists"):
|
||||
os.remove("/etc/cron.daily/yunohost-fetch-appslists")
|
||||
|
||||
if os.path.exists(APPSLISTS_JSON):
|
||||
os.remove(APPSLISTS_JSON)
|
||||
|
||||
|
||||
def teardown_function(function):
|
||||
pass
|
||||
|
||||
|
||||
def cron_job_is_there():
|
||||
r = os.system("run-parts -v --test /etc/cron.daily/ | grep yunohost-fetch-appslists")
|
||||
return r == 0
|
||||
|
||||
|
||||
###############################################################################
|
||||
# Test listing of appslists and registering of appslists #
|
||||
###############################################################################
|
||||
|
||||
|
||||
def test_appslist_list_empty():
|
||||
"""
|
||||
Calling app_listlists() with no registered list should return empty dict
|
||||
"""
|
||||
|
||||
assert app_listlists() == {}
|
||||
|
||||
|
||||
def test_appslist_list_register():
|
||||
"""
|
||||
Register a new list
|
||||
"""
|
||||
|
||||
# Assume we're starting with an empty app list
|
||||
assert app_listlists() == {}
|
||||
|
||||
# Register a new dummy list
|
||||
_register_new_appslist("https://lol.com/appslist.json", "dummy")
|
||||
|
||||
appslist_dict = app_listlists()
|
||||
assert "dummy" in appslist_dict.keys()
|
||||
assert appslist_dict["dummy"]["url"] == "https://lol.com/appslist.json"
|
||||
|
||||
assert cron_job_is_there()
|
||||
|
||||
|
||||
def test_appslist_list_register_conflict_name():
|
||||
"""
|
||||
Attempt to register a new list with conflicting name
|
||||
"""
|
||||
|
||||
_register_new_appslist("https://lol.com/appslist.json", "dummy")
|
||||
with pytest.raises(MoulinetteError):
|
||||
_register_new_appslist("https://lol.com/appslist2.json", "dummy")
|
||||
|
||||
appslist_dict = app_listlists()
|
||||
|
||||
assert "dummy" in appslist_dict.keys()
|
||||
assert "dummy2" not in appslist_dict.keys()
|
||||
|
||||
|
||||
def test_appslist_list_register_conflict_url():
|
||||
"""
|
||||
Attempt to register a new list with conflicting url
|
||||
"""
|
||||
|
||||
_register_new_appslist("https://lol.com/appslist.json", "dummy")
|
||||
with pytest.raises(MoulinetteError):
|
||||
_register_new_appslist("https://lol.com/appslist.json", "plopette")
|
||||
|
||||
appslist_dict = app_listlists()
|
||||
|
||||
assert "dummy" in appslist_dict.keys()
|
||||
assert "plopette" not in appslist_dict.keys()
|
||||
|
||||
|
||||
###############################################################################
|
||||
# Test fetching of appslists #
|
||||
###############################################################################
|
||||
|
||||
|
||||
def test_appslist_fetch():
|
||||
"""
|
||||
Do a fetchlist and test the .json got updated.
|
||||
"""
|
||||
assert app_listlists() == {}
|
||||
|
||||
_register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
|
||||
|
||||
with requests_mock.Mocker() as m:
|
||||
|
||||
# Mock the server response with a valid (well, empty, yep) json
|
||||
m.register_uri("GET", URL_OFFICIAL_APP_LIST, text='{ }')
|
||||
|
||||
official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"]
|
||||
app_fetchlist()
|
||||
new_official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"]
|
||||
|
||||
assert new_official_lastUpdate > official_lastUpdate
|
||||
|
||||
|
||||
def test_appslist_fetch_single_appslist():
|
||||
"""
|
||||
Register several lists but only fetch one. Check only one got updated.
|
||||
"""
|
||||
|
||||
assert app_listlists() == {}
|
||||
_register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
|
||||
_register_new_appslist("https://lol.com/appslist.json", "dummy")
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
with requests_mock.Mocker() as m:
|
||||
|
||||
# Mock the server response with a valid (well, empty, yep) json
|
||||
m.register_uri("GET", URL_OFFICIAL_APP_LIST, text='{ }')
|
||||
|
||||
official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"]
|
||||
dummy_lastUpdate = app_listlists()["dummy"]["lastUpdate"]
|
||||
app_fetchlist(name="yunohost")
|
||||
new_official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"]
|
||||
new_dummy_lastUpdate = app_listlists()["dummy"]["lastUpdate"]
|
||||
|
||||
assert new_official_lastUpdate > official_lastUpdate
|
||||
assert new_dummy_lastUpdate == dummy_lastUpdate
|
||||
|
||||
|
||||
def test_appslist_fetch_unknownlist():
|
||||
"""
|
||||
Attempt to fetch an unknown list
|
||||
"""
|
||||
|
||||
assert app_listlists() == {}
|
||||
|
||||
with pytest.raises(MoulinetteError):
|
||||
app_fetchlist(name="swag")
|
||||
|
||||
|
||||
def test_appslist_fetch_url_but_no_name():
|
||||
"""
|
||||
Do a fetchlist with url given, but no name given
|
||||
"""
|
||||
|
||||
with pytest.raises(MoulinetteError):
|
||||
app_fetchlist(url=URL_OFFICIAL_APP_LIST)
|
||||
|
||||
|
||||
def test_appslist_fetch_badurl():
|
||||
"""
|
||||
Do a fetchlist with a bad url
|
||||
"""
|
||||
|
||||
app_fetchlist(url="https://not.a.valid.url/plop.json", name="plop")
|
||||
|
||||
|
||||
def test_appslist_fetch_badfile():
|
||||
"""
|
||||
Do a fetchlist and mock a response with a bad json
|
||||
"""
|
||||
assert app_listlists() == {}
|
||||
|
||||
_register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
|
||||
|
||||
with requests_mock.Mocker() as m:
|
||||
|
||||
m.register_uri("GET", URL_OFFICIAL_APP_LIST, text='{ not json lol }')
|
||||
|
||||
app_fetchlist()
|
||||
|
||||
|
||||
def test_appslist_fetch_404():
|
||||
"""
|
||||
Do a fetchlist and mock a 404 response
|
||||
"""
|
||||
assert app_listlists() == {}
|
||||
|
||||
_register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
|
||||
|
||||
with requests_mock.Mocker() as m:
|
||||
|
||||
m.register_uri("GET", URL_OFFICIAL_APP_LIST, status_code=404)
|
||||
|
||||
app_fetchlist()
|
||||
|
||||
|
||||
def test_appslist_fetch_sslerror():
|
||||
"""
|
||||
Do a fetchlist and mock an SSL error
|
||||
"""
|
||||
assert app_listlists() == {}
|
||||
|
||||
_register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
|
||||
|
||||
with requests_mock.Mocker() as m:
|
||||
|
||||
m.register_uri("GET", URL_OFFICIAL_APP_LIST,
|
||||
exc=requests.exceptions.SSLError)
|
||||
|
||||
app_fetchlist()
|
||||
|
||||
|
||||
def test_appslist_fetch_timeout():
|
||||
"""
|
||||
Do a fetchlist and mock a timeout
|
||||
"""
|
||||
assert app_listlists() == {}
|
||||
|
||||
_register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
|
||||
|
||||
with requests_mock.Mocker() as m:
|
||||
|
||||
m.register_uri("GET", URL_OFFICIAL_APP_LIST,
|
||||
exc=requests.exceptions.ConnectTimeout)
|
||||
|
||||
app_fetchlist()
|
||||
|
||||
|
||||
###############################################################################
|
||||
# Test remove of appslist #
|
||||
###############################################################################
|
||||
|
||||
|
||||
def test_appslist_remove():
|
||||
"""
|
||||
Register a new appslist, then remove it
|
||||
"""
|
||||
|
||||
# Assume we're starting with an empty app list
|
||||
assert app_listlists() == {}
|
||||
|
||||
# Register a new dummy list
|
||||
_register_new_appslist("https://lol.com/appslist.json", "dummy")
|
||||
app_removelist("dummy")
|
||||
|
||||
# Should end up with no list registered
|
||||
assert app_listlists() == {}
|
||||
|
||||
|
||||
def test_appslist_remove_unknown():
|
||||
"""
|
||||
Attempt to remove an unknown list
|
||||
"""
|
||||
|
||||
with pytest.raises(MoulinetteError):
|
||||
app_removelist("dummy")
|
||||
|
||||
|
||||
###############################################################################
|
||||
# Test migration from legacy appslist system #
|
||||
###############################################################################
|
||||
|
||||
|
||||
def add_legacy_cron(name, url):
|
||||
with open("/etc/cron.d/yunohost-appslist-%s" % name, "w") as f:
|
||||
f.write('00 00 * * * root yunohost app fetchlist -u %s -n %s > /dev/null 2>&1\n' % (url, name))
|
||||
|
||||
|
||||
def test_appslist_check_using_legacy_system_testFalse():
|
||||
"""
|
||||
If no legacy cron job is there, the check should return False
|
||||
"""
|
||||
assert glob.glob("/etc/cron.d/yunohost-appslist-*") == []
|
||||
assert _using_legacy_appslist_system() is False
|
||||
|
||||
|
||||
def test_appslist_check_using_legacy_system_testTrue():
|
||||
"""
|
||||
If there's a legacy cron job, the check should return True
|
||||
"""
|
||||
assert glob.glob("/etc/cron.d/yunohost-appslist-*") == []
|
||||
add_legacy_cron("yunohost", "https://app.yunohost.org/official.json")
|
||||
assert _using_legacy_appslist_system() is True
|
||||
|
||||
|
||||
def test_appslist_system_migration():
|
||||
"""
|
||||
Test that legacy cron jobs get migrated correctly when calling app_listlists
|
||||
"""
|
||||
|
||||
# Start with no legacy cron, no appslist registered
|
||||
assert glob.glob("/etc/cron.d/yunohost-appslist-*") == []
|
||||
assert app_listlists() == {}
|
||||
assert not os.path.exists("/etc/cron.daily/yunohost-fetch-appslists")
|
||||
|
||||
# Add a few legacy crons
|
||||
add_legacy_cron("yunohost", "https://app.yunohost.org/official.json")
|
||||
add_legacy_cron("dummy", "https://swiggitty.swaggy.lol/yolo.json")
|
||||
|
||||
# Migrate
|
||||
assert _using_legacy_appslist_system() is True
|
||||
_migrate_appslist_system()
|
||||
assert _using_legacy_appslist_system() is False
|
||||
|
||||
# No legacy cron job should remain
|
||||
assert glob.glob("/etc/cron.d/yunohost-appslist-*") == []
|
||||
|
||||
# Check they are in app_listlists anyway
|
||||
appslist_dict = app_listlists()
|
||||
assert "yunohost" in appslist_dict.keys()
|
||||
assert appslist_dict["yunohost"]["url"] == "https://app.yunohost.org/official.json"
|
||||
assert "dummy" in appslist_dict.keys()
|
||||
assert appslist_dict["dummy"]["url"] == "https://swiggitty.swaggy.lol/yolo.json"
|
||||
|
||||
assert cron_job_is_there()
|
||||
|
||||
|
||||
def test_appslist_system_migration_badcron():
|
||||
"""
|
||||
Test the migration on a bad legacy cron (no url found inside cron job)
|
||||
"""
|
||||
|
||||
# Start with no legacy cron, no appslist registered
|
||||
assert glob.glob("/etc/cron.d/yunohost-appslist-*") == []
|
||||
assert app_listlists() == {}
|
||||
assert not os.path.exists("/etc/cron.daily/yunohost-fetch-appslists")
|
||||
|
||||
# Add a "bad" legacy cron
|
||||
add_legacy_cron("wtflist", "ftp://the.fuck.is.this")
|
||||
|
||||
# Migrate
|
||||
assert _using_legacy_appslist_system() is True
|
||||
_migrate_appslist_system()
|
||||
assert _using_legacy_appslist_system() is False
|
||||
|
||||
# No legacy cron should remain, but it should be backuped in /etc/yunohost
|
||||
assert glob.glob("/etc/cron.d/yunohost-appslist-*") == []
|
||||
assert os.path.exists("/etc/yunohost/wtflist.oldlist.bkp")
|
||||
|
||||
# Appslist should still be empty
|
||||
assert app_listlists() == {}
|
||||
|
||||
|
||||
def test_appslist_system_migration_conflict():
|
||||
"""
|
||||
Test migration of conflicting cron job (in terms of url)
|
||||
"""
|
||||
|
||||
# Start with no legacy cron, no appslist registered
|
||||
assert glob.glob("/etc/cron.d/yunohost-appslist-*") == []
|
||||
assert app_listlists() == {}
|
||||
assert not os.path.exists("/etc/cron.daily/yunohost-fetch-appslists")
|
||||
|
||||
# Add a few legacy crons
|
||||
add_legacy_cron("yunohost", "https://app.yunohost.org/official.json")
|
||||
add_legacy_cron("dummy", "https://app.yunohost.org/official.json")
|
||||
|
||||
# Migrate
|
||||
assert _using_legacy_appslist_system() is True
|
||||
_migrate_appslist_system()
|
||||
assert _using_legacy_appslist_system() is False
|
||||
|
||||
# No legacy cron job should remain
|
||||
assert glob.glob("/etc/cron.d/yunohost-appslist-*") == []
|
||||
|
||||
# Only one among "dummy" and "yunohost" should be listed
|
||||
appslist_dict = app_listlists()
|
||||
assert (len(appslist_dict.keys()) == 1)
|
||||
assert ("dummy" in appslist_dict.keys()) or ("yunohost" in appslist_dict.keys())
|
||||
|
||||
assert cron_job_is_there()
|
||||
|
|
@ -38,7 +38,7 @@ import apt.progress
|
|||
|
||||
from moulinette.core import MoulinetteError, init_authenticator
|
||||
from moulinette.utils.log import getActionLogger
|
||||
from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, app_list
|
||||
from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, app_list, _install_appslist_fetch_cron
|
||||
from yunohost.domain import domain_add, domain_list, get_public_ip, _get_maindomain, _set_maindomain
|
||||
from yunohost.dyndns import dyndns_subscribe
|
||||
from yunohost.firewall import firewall_upnp
|
||||
|
|
@ -327,6 +327,15 @@ def tools_postinstall(domain, password, ignore_dyndns=False):
|
|||
# Enable UPnP silently and reload firewall
|
||||
firewall_upnp('enable', no_refresh=True)
|
||||
|
||||
# Setup the default official app list with cron job
|
||||
try:
|
||||
app_fetchlist(name="yunohost",
|
||||
url="https://app.yunohost.org/official.json")
|
||||
except Exception as e:
|
||||
logger.warning(str(e))
|
||||
|
||||
_install_appslist_fetch_cron()
|
||||
|
||||
os.system('touch /etc/yunohost/installed')
|
||||
|
||||
# Enable and start YunoHost firewall at boot time
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue