mirror of
https://github.com/YunoHost/yunohost.git
synced 2024-09-03 20:06:10 +02:00
Buuuuurn the appslist system
This commit is contained in:
Alexandre Aubin
parent
f0440fbd6b
commit
a6607eaf30
3 changed files with 0 additions and 357 deletions
|
|
@ -568,38 +568,6 @@ app:
|
|||
category_help: Manage apps
|
||||
actions:
|
||||
|
||||
### app_fetchlist()
|
||||
fetchlist:
|
||||
action_help: Fetch application lists from app servers, or register a new one.
|
||||
api: PUT /appslists
|
||||
arguments:
|
||||
-n:
|
||||
full: --name
|
||||
help: Name of the list to fetch (fetches all registered lists if empty)
|
||||
extra:
|
||||
pattern: &pattern_listname
|
||||
- !!str ^[a-z0-9_]+$
|
||||
- "pattern_listname"
|
||||
-u:
|
||||
full: --url
|
||||
help: URL of a new application list to register. To be specified with -n.
|
||||
|
||||
### app_listlists()
|
||||
listlists:
|
||||
action_help: List registered application lists
|
||||
api: GET /appslists
|
||||
|
||||
### app_removelist()
|
||||
removelist:
|
||||
action_help: Remove and forget about a given application list
|
||||
api: DELETE /appslists
|
||||
arguments:
|
||||
name:
|
||||
help: Name of the list to remove
|
||||
extra:
|
||||
ask: ask_list_to_remove
|
||||
pattern: *pattern_listname
|
||||
|
||||
### app_list()
|
||||
list:
|
||||
action_help: List apps
|
||||
|
|
|
|||
|
|
@ -51,21 +51,10 @@
|
|||
"app_upgraded": "{app:s} has been upgraded",
|
||||
"apps_permission_not_found": "No permission found for the installed apps",
|
||||
"apps_permission_restoration_failed": "Permission '{permission:s}' for app {app:s} restoration has failed",
|
||||
"appslist_corrupted_json": "Could not load the application lists. It looks like {filename:s} is corrupted.",
|
||||
"appslist_could_not_migrate": "Could not migrate app list {appslist:s}! Unable to parse the url… The old cron job has been kept in {bkp_file:s}.",
|
||||
"appslist_fetched": "The application list {appslist:s} has been fetched",
|
||||
"appslist_migrating": "Migrating application list {appslist:s}…",
|
||||
"appslist_name_already_tracked": "There is already a registered application list with name {name:s}.",
|
||||
"appslist_removed": "The application list {appslist:s} has been removed",
|
||||
"appslist_retrieve_bad_format": "Retrieved file for application list {appslist:s} is not valid",
|
||||
"appslist_retrieve_error": "Unable to retrieve the remote application list {appslist:s}: {error:s}",
|
||||
"appslist_unknown": "Application list {appslist:s} unknown.",
|
||||
"appslist_url_already_tracked": "There is already a registered application list with url {url:s}.",
|
||||
"ask_current_admin_password": "Current administration password",
|
||||
"ask_email": "Email address",
|
||||
"ask_firstname": "First name",
|
||||
"ask_lastname": "Last name",
|
||||
"ask_list_to_remove": "List to remove",
|
||||
"ask_main_domain": "Main domain",
|
||||
"ask_new_admin_password": "New administration password",
|
||||
"ask_new_domain": "New domain",
|
||||
|
|
@ -152,7 +141,6 @@
|
|||
"confirm_app_install_danger": "WARNING! This application is still experimental (if not explicitly not working) and it is likely to break your system! You should probably NOT install it unless you know what you are doing. Are you willing to take that risk? [{answers:s}] ",
|
||||
"confirm_app_install_thirdparty": "WARNING! Installing 3rd party applications may compromise the integrity and security of your system. You should probably NOT install it unless you know what you are doing. Are you willing to take that risk? [{answers:s}] ",
|
||||
"custom_app_url_required": "You must provide a URL to upgrade your custom app {app:s}",
|
||||
"custom_appslist_name_required": "You must provide a name for your custom app list",
|
||||
"diagnosis_debian_version_error": "Can't retrieve the Debian version: {error}",
|
||||
"diagnosis_kernel_version_error": "Can't retrieve kernel version: {error}",
|
||||
"diagnosis_monitor_disk_error": "Can't monitor disks: {error}",
|
||||
|
|
@ -263,8 +251,6 @@
|
|||
"log_app_addaccess": "Add access to '{}'",
|
||||
"log_app_removeaccess": "Remove access to '{}'",
|
||||
"log_app_clearaccess": "Remove all access to '{}'",
|
||||
"log_app_fetchlist": "Add an application list",
|
||||
"log_app_removelist": "Remove an application list",
|
||||
"log_app_change_url": "Change the url of '{}' application",
|
||||
"log_app_install": "Install '{}' application",
|
||||
"log_app_remove": "Remove '{}' application",
|
||||
|
|
@ -401,7 +387,6 @@
|
|||
"network_check_smtp_ko": "Outbound mail (SMTP port 25) seems to be blocked by your network",
|
||||
"network_check_smtp_ok": "Outbound mail (SMTP port 25) is not blocked",
|
||||
"new_domain_required": "You must provide the new main domain",
|
||||
"no_appslist_found": "No app list found",
|
||||
"no_internet_connection": "Server is not connected to the Internet",
|
||||
"no_ipv6_connectivity": "IPv6 connectivity is not available",
|
||||
"no_restore_script": "No restore script found for the app '{app:s}'",
|
||||
|
|
@ -538,7 +523,6 @@
|
|||
"system_upgraded": "The system has been upgraded",
|
||||
"system_username_exists": "Username already exists in the system users",
|
||||
"this_action_broke_dpkg": "This action broke dpkg/apt (the system package managers)... You can try to solve this issue by connecting through SSH and running `sudo dpkg --configure -a`.",
|
||||
"tools_update_failed_to_app_fetchlist": "Failed to update YunoHost's applists because: {error}",
|
||||
"tools_upgrade_at_least_one": "Please specify --apps OR --system",
|
||||
"tools_upgrade_cant_both": "Cannot upgrade both system and apps at the same time",
|
||||
"tools_upgrade_cant_hold_critical_packages": "Unable to hold critical packages ...",
|
||||
|
|
|
|||
|
|
@ -33,8 +33,6 @@ import re
|
|||
import urlparse
|
||||
import subprocess
|
||||
import glob
|
||||
import pwd
|
||||
import grp
|
||||
import urllib
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime
|
||||
|
|
@ -68,166 +66,6 @@ re_app_instance_name = re.compile(
|
|||
)
|
||||
|
||||
|
||||
def app_listlists():
|
||||
"""
|
||||
List fetched lists
|
||||
|
||||
"""
|
||||
|
||||
# Migrate appslist system if needed
|
||||
# XXX move to a migration when those are implemented
|
||||
if _using_legacy_appslist_system():
|
||||
_migrate_appslist_system()
|
||||
|
||||
# Get the list
|
||||
appslist_list = _read_appslist_list()
|
||||
|
||||
# Convert 'lastUpdate' timestamp to datetime
|
||||
for name, infos in appslist_list.items():
|
||||
if infos["lastUpdate"] is None:
|
||||
infos["lastUpdate"] = 0
|
||||
infos["lastUpdate"] = datetime.utcfromtimestamp(infos["lastUpdate"])
|
||||
|
||||
return appslist_list
|
||||
|
||||
|
||||
def app_fetchlist(url=None, name=None):
|
||||
"""
|
||||
Fetch application list(s) from app server. By default, fetch all lists.
|
||||
|
||||
Keyword argument:
|
||||
name -- Name of the list
|
||||
url -- URL of remote JSON list
|
||||
"""
|
||||
if url and not url.endswith(".json"):
|
||||
raise YunohostError("This is not a valid application list url. It should end with .json.")
|
||||
|
||||
# If needed, create folder where actual appslists are stored
|
||||
if not os.path.exists(REPO_PATH):
|
||||
os.makedirs(REPO_PATH)
|
||||
|
||||
# Migrate appslist system if needed
|
||||
# XXX move that to a migration once they are finished
|
||||
if _using_legacy_appslist_system():
|
||||
_migrate_appslist_system()
|
||||
|
||||
# Read the list of appslist...
|
||||
appslists = _read_appslist_list()
|
||||
|
||||
# Determine the list of appslist to be fetched
|
||||
appslists_to_be_fetched = []
|
||||
|
||||
# If a url and and a name is given, try to register new list,
|
||||
# the fetch only this list
|
||||
if url is not None:
|
||||
if name:
|
||||
operation_logger = OperationLogger('app_fetchlist')
|
||||
operation_logger.start()
|
||||
_register_new_appslist(url, name)
|
||||
# Refresh the appslists dict
|
||||
appslists = _read_appslist_list()
|
||||
appslists_to_be_fetched = [name]
|
||||
operation_logger.success()
|
||||
else:
|
||||
raise YunohostError('custom_appslist_name_required')
|
||||
|
||||
# If a name is given, look for an appslist with that name and fetch it
|
||||
elif name is not None:
|
||||
if name not in appslists.keys():
|
||||
raise YunohostError('appslist_unknown', appslist=name)
|
||||
else:
|
||||
appslists_to_be_fetched = [name]
|
||||
|
||||
# Otherwise, fetch all lists
|
||||
else:
|
||||
appslists_to_be_fetched = appslists.keys()
|
||||
|
||||
import requests # lazy loading this module for performance reasons
|
||||
# Fetch all appslists to be fetched
|
||||
for name in appslists_to_be_fetched:
|
||||
|
||||
url = appslists[name]["url"]
|
||||
|
||||
logger.debug("Attempting to fetch list %s at %s" % (name, url))
|
||||
|
||||
# Download file
|
||||
try:
|
||||
appslist_request = requests.get(url, timeout=30)
|
||||
except requests.exceptions.SSLError:
|
||||
logger.error(m18n.n('appslist_retrieve_error',
|
||||
appslist=name,
|
||||
error="SSL connection error"))
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.error(m18n.n('appslist_retrieve_error',
|
||||
appslist=name,
|
||||
error=str(e)))
|
||||
continue
|
||||
if appslist_request.status_code != 200:
|
||||
logger.error(m18n.n('appslist_retrieve_error',
|
||||
appslist=name,
|
||||
error="Server returned code %s " %
|
||||
str(appslist_request.status_code)))
|
||||
continue
|
||||
|
||||
# Validate app list format
|
||||
# TODO / Possible improvement : better validation for app list (check
|
||||
# that json fields actually look like an app list and not any json
|
||||
# file)
|
||||
appslist = appslist_request.text
|
||||
try:
|
||||
json.loads(appslist)
|
||||
except ValueError as e:
|
||||
logger.error(m18n.n('appslist_retrieve_bad_format',
|
||||
appslist=name))
|
||||
continue
|
||||
|
||||
# Write app list to file
|
||||
list_file = '%s/%s.json' % (REPO_PATH, name)
|
||||
try:
|
||||
with open(list_file, "w") as f:
|
||||
f.write(appslist)
|
||||
except Exception as e:
|
||||
raise YunohostError("Error while writing appslist %s: %s" % (name, str(e)), raw_msg=True)
|
||||
|
||||
now = int(time.time())
|
||||
appslists[name]["lastUpdate"] = now
|
||||
|
||||
logger.success(m18n.n('appslist_fetched', appslist=name))
|
||||
|
||||
# Write updated list of appslist
|
||||
_write_appslist_list(appslists)
|
||||
|
||||
|
||||
@is_unit_operation()
|
||||
def app_removelist(operation_logger, name):
|
||||
"""
|
||||
Remove list from the repositories
|
||||
|
||||
Keyword argument:
|
||||
name -- Name of the list to remove
|
||||
|
||||
"""
|
||||
appslists = _read_appslist_list()
|
||||
|
||||
# Make sure we know this appslist
|
||||
if name not in appslists.keys():
|
||||
raise YunohostError('appslist_unknown', appslist=name)
|
||||
|
||||
operation_logger.start()
|
||||
|
||||
# Remove json
|
||||
json_path = '%s/%s.json' % (REPO_PATH, name)
|
||||
if os.path.exists(json_path):
|
||||
os.remove(json_path)
|
||||
|
||||
# Forget about this appslist
|
||||
del appslists[name]
|
||||
_write_appslist_list(appslists)
|
||||
|
||||
logger.success(m18n.n('appslist_removed', appslist=name))
|
||||
|
||||
|
||||
def app_list(filter=None, raw=False, installed=False, with_backup=False):
|
||||
"""
|
||||
List apps
|
||||
|
|
@ -2704,153 +2542,6 @@ def _parse_app_instance_name(app_instance_name):
|
|||
return (appid, app_instance_nb)
|
||||
|
||||
|
||||
def _using_legacy_appslist_system():
|
||||
"""
|
||||
Return True if we're using the old fetchlist scheme.
|
||||
This is determined by the presence of some cron job yunohost-applist-foo
|
||||
"""
|
||||
|
||||
return glob.glob("/etc/cron.d/yunohost-applist-*") != []
|
||||
|
||||
|
||||
def _migrate_appslist_system():
|
||||
"""
|
||||
Migrate from the legacy fetchlist system to the new one
|
||||
"""
|
||||
legacy_crons = glob.glob("/etc/cron.d/yunohost-applist-*")
|
||||
|
||||
for cron_path in legacy_crons:
|
||||
appslist_name = os.path.basename(cron_path).replace("yunohost-applist-", "")
|
||||
logger.debug(m18n.n('appslist_migrating', appslist=appslist_name))
|
||||
|
||||
# Parse appslist url in cron
|
||||
cron_file_content = open(cron_path).read().strip()
|
||||
appslist_url_parse = re.search("-u (https?://[^ ]+)", cron_file_content)
|
||||
|
||||
# Abort if we did not find an url
|
||||
if not appslist_url_parse or not appslist_url_parse.groups():
|
||||
# Bkp the old cron job somewhere else
|
||||
bkp_file = "/etc/yunohost/%s.oldlist.bkp" % appslist_name
|
||||
os.rename(cron_path, bkp_file)
|
||||
# Notice the user
|
||||
logger.warning(m18n.n('appslist_could_not_migrate',
|
||||
appslist=appslist_name,
|
||||
bkp_file=bkp_file))
|
||||
# Otherwise, register the list and remove the legacy cron
|
||||
else:
|
||||
appslist_url = appslist_url_parse.groups()[0]
|
||||
try:
|
||||
_register_new_appslist(appslist_url, appslist_name)
|
||||
# Might get an exception if two legacy cron jobs conflict
|
||||
# in terms of url...
|
||||
except Exception as e:
|
||||
logger.error(str(e))
|
||||
# Bkp the old cron job somewhere else
|
||||
bkp_file = "/etc/yunohost/%s.oldlist.bkp" % appslist_name
|
||||
os.rename(cron_path, bkp_file)
|
||||
# Notice the user
|
||||
logger.warning(m18n.n('appslist_could_not_migrate',
|
||||
appslist=appslist_name,
|
||||
bkp_file=bkp_file))
|
||||
else:
|
||||
os.remove(cron_path)
|
||||
|
||||
|
||||
def _install_appslist_fetch_cron():
|
||||
|
||||
cron_job_file = "/etc/cron.daily/yunohost-fetch-appslists"
|
||||
|
||||
logger.debug("Installing appslist fetch cron job")
|
||||
|
||||
cron_job = []
|
||||
cron_job.append("#!/bin/bash")
|
||||
# We add a random delay between 0 and 60 min to avoid every instance fetching
|
||||
# the appslist at the same time every night
|
||||
cron_job.append("(sleep $((RANDOM%3600));")
|
||||
cron_job.append("yunohost app fetchlist > /dev/null 2>&1) &")
|
||||
|
||||
with open(cron_job_file, "w") as f:
|
||||
f.write('\n'.join(cron_job))
|
||||
|
||||
_set_permissions(cron_job_file, "root", "root", 0o755)
|
||||
|
||||
|
||||
# FIXME - Duplicate from certificate.py, should be moved into a common helper
|
||||
# thing...
|
||||
def _set_permissions(path, user, group, permissions):
|
||||
uid = pwd.getpwnam(user).pw_uid
|
||||
gid = grp.getgrnam(group).gr_gid
|
||||
|
||||
os.chown(path, uid, gid)
|
||||
os.chmod(path, permissions)
|
||||
|
||||
|
||||
def _read_appslist_list():
|
||||
"""
|
||||
Read the json corresponding to the list of appslists
|
||||
"""
|
||||
|
||||
# If file does not exists yet, return empty dict
|
||||
if not os.path.exists(APPSLISTS_JSON):
|
||||
return {}
|
||||
|
||||
# Read file content
|
||||
with open(APPSLISTS_JSON, "r") as f:
|
||||
appslists_json = f.read()
|
||||
|
||||
# Parse json, throw exception if what we got from file is not a valid json
|
||||
try:
|
||||
appslists = json.loads(appslists_json)
|
||||
except ValueError:
|
||||
raise YunohostError('appslist_corrupted_json', filename=APPSLISTS_JSON)
|
||||
|
||||
return appslists
|
||||
|
||||
|
||||
def _write_appslist_list(appslist_lists):
|
||||
"""
|
||||
Update the json containing list of appslists
|
||||
"""
|
||||
|
||||
# Write appslist list
|
||||
try:
|
||||
with open(APPSLISTS_JSON, "w") as f:
|
||||
json.dump(appslist_lists, f)
|
||||
except Exception as e:
|
||||
raise YunohostError("Error while writing list of appslist %s: %s" %
|
||||
(APPSLISTS_JSON, str(e)), raw_msg=True)
|
||||
|
||||
|
||||
def _register_new_appslist(url, name):
|
||||
"""
|
||||
Add a new appslist to be fetched regularly.
|
||||
Raise an exception if url or name conflicts with an existing list.
|
||||
"""
|
||||
|
||||
appslist_list = _read_appslist_list()
|
||||
|
||||
# Check if name conflicts with an existing list
|
||||
if name in appslist_list:
|
||||
raise YunohostError('appslist_name_already_tracked', name=name)
|
||||
|
||||
# Check if url conflicts with an existing list
|
||||
known_appslist_urls = [appslist["url"] for _, appslist in appslist_list.items()]
|
||||
|
||||
if url in known_appslist_urls:
|
||||
raise YunohostError('appslist_url_already_tracked', url=url)
|
||||
|
||||
logger.debug("Registering new appslist %s at %s" % (name, url))
|
||||
|
||||
appslist_list[name] = {
|
||||
"url": url,
|
||||
"lastUpdate": None
|
||||
}
|
||||
|
||||
_write_appslist_list(appslist_list)
|
||||
|
||||
_install_appslist_fetch_cron()
|
||||
|
||||
|
||||
def is_true(arg):
|
||||
"""
|
||||
Convert a string into a boolean
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue