mirror of
https://github.com/YunoHost/yunohost.git
synced 2024-09-03 20:06:10 +02:00
2327 lines
83 KiB
Python
2327 lines
83 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
""" License
|
|
|
|
Copyright (C) 2013 YunoHost
|
|
|
|
This program is free software; you can redistribute it and/or modify
|
|
it under the terms of the GNU Affero General Public License as published
|
|
by the Free Software Foundation, either version 3 of the License, or
|
|
(at your option) any later version.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU Affero General Public License for more details.
|
|
|
|
You should have received a copy of the GNU Affero General Public License
|
|
along with this program; if not, see http://www.gnu.org/licenses
|
|
|
|
"""
|
|
|
|
""" yunohost_backup.py
|
|
|
|
Manage backups
|
|
"""
|
|
import os
|
|
import re
|
|
import json
|
|
import time
|
|
import tarfile
|
|
import shutil
|
|
import subprocess
|
|
import csv
|
|
import tempfile
|
|
from datetime import datetime
|
|
from glob import glob
|
|
from collections import OrderedDict
|
|
|
|
from moulinette import msignals, m18n
|
|
from yunohost.utils.error import YunohostError
|
|
from moulinette.utils import filesystem
|
|
from moulinette.utils.log import getActionLogger
|
|
from moulinette.utils.filesystem import read_file, mkdir
|
|
|
|
from yunohost.app import (
|
|
app_info, _is_installed, _parse_app_instance_name, _patch_php5
|
|
)
|
|
from yunohost.hook import (
|
|
hook_list, hook_info, hook_callback, hook_exec, CUSTOM_HOOK_FOLDER
|
|
)
|
|
from yunohost.monitor import binary_to_human
|
|
from yunohost.tools import tools_postinstall
|
|
from yunohost.service import service_regen_conf
|
|
from yunohost.log import OperationLogger
|
|
from functools import reduce
|
|
|
|
BACKUP_PATH = '/home/yunohost.backup'
|
|
ARCHIVES_PATH = '%s/archives' % BACKUP_PATH
|
|
APP_MARGIN_SPACE_SIZE = 100 # In MB
|
|
CONF_MARGIN_SPACE_SIZE = 10 # IN MB
|
|
POSTINSTALL_ESTIMATE_SPACE_SIZE = 5 # In MB
|
|
MB_ALLOWED_TO_ORGANIZE = 10
|
|
logger = getActionLogger('yunohost.backup')
|
|
|
|
|
|
class BackupRestoreTargetsManager(object):
|
|
|
|
"""
|
|
BackupRestoreTargetsManager manage the targets
|
|
in BackupManager and RestoreManager
|
|
"""
|
|
|
|
def __init__(self):
|
|
|
|
self.targets = {}
|
|
self.results = {
|
|
"system": {},
|
|
"apps": {}
|
|
}
|
|
|
|
def set_result(self, category, element, value):
|
|
"""
|
|
Change (or initialize) the current status/result of a given target.
|
|
|
|
Args:
|
|
category -- The category of the target
|
|
|
|
element -- The target for which to change the status/result
|
|
|
|
value -- The new status/result, among "Unknown", "Success",
|
|
"Warning", "Error" and "Skipped"
|
|
"""
|
|
|
|
levels = ["Unknown", "Success", "Warning", "Error", "Skipped"]
|
|
|
|
assert value in levels
|
|
|
|
if element not in self.results[category].keys():
|
|
self.results[category][element] = value
|
|
else:
|
|
currentValue = self.results[category][element]
|
|
if (levels.index(currentValue) > levels.index(value)):
|
|
return
|
|
else:
|
|
self.results[category][element] = value
|
|
|
|
def set_wanted(self, category,
|
|
wanted_targets, available_targets,
|
|
error_if_wanted_target_is_unavailable):
|
|
"""
|
|
Define and validate targets to be backuped or to be restored (list of
|
|
system parts, apps..). The wanted targets are compared and filtered
|
|
with respect to the available targets. If a wanted targets is not
|
|
available, a call to "error_if_wanted_target_is_unavailable" is made.
|
|
|
|
Args:
|
|
category -- The category (apps or system) for which to set the
|
|
targets ;
|
|
|
|
wanted_targets -- List of targets which are wanted by the user. Can be
|
|
"None" or [], corresponding to "No targets" or "All
|
|
targets" ;
|
|
|
|
available_targets -- List of targets which are really available ;
|
|
|
|
error_if_wanted_target_is_unavailable
|
|
-- Callback for targets which are not available.
|
|
"""
|
|
|
|
# If no targets wanted, set as empty list
|
|
if wanted_targets is None:
|
|
self.targets[category] = []
|
|
|
|
# If all targets wanted, use all available targets
|
|
elif wanted_targets == []:
|
|
self.targets[category] = available_targets
|
|
|
|
# If the user manually specified which targets to backup, we need to
|
|
# validate that each target is actually available
|
|
else:
|
|
self.targets[category] = [part for part in wanted_targets
|
|
if part in available_targets]
|
|
|
|
# Display an error for each target asked by the user but which is
|
|
# unknown
|
|
unavailable_targets = [part for part in wanted_targets
|
|
if part not in available_targets]
|
|
|
|
for target in unavailable_targets:
|
|
self.set_result(category, target, "Skipped")
|
|
error_if_wanted_target_is_unavailable(target)
|
|
|
|
# For target with no result yet (like 'Skipped'), set it as unknown
|
|
if self.targets[category] is not None:
|
|
for target in self.targets[category]:
|
|
self.set_result(category, target, "Unknown")
|
|
|
|
return self.list(category, exclude=["Skipped"])
|
|
|
|
def list(self, category, include=None, exclude=None):
|
|
"""
|
|
List targets in a given category.
|
|
|
|
The list is filtered with a whitelist (include) or blacklist (exclude)
|
|
with respect to the current 'result' of the target.
|
|
"""
|
|
|
|
assert (include and isinstance(include, list) and not exclude) \
|
|
or (exclude and isinstance(exclude, list) and not include)
|
|
|
|
if include:
|
|
return [target.encode("Utf-8") for target in self.targets[category]
|
|
if self.results[category][target] in include]
|
|
|
|
if exclude:
|
|
return [target.encode("Utf-8") for target in self.targets[category]
|
|
if self.results[category][target] not in exclude]
|
|
|
|
|
|
class BackupManager():
|
|
|
|
"""
|
|
This class collect files to backup in a list and apply one or several
|
|
backup method on it.
|
|
|
|
The list contains dict with source and dest properties. The goal of this csv
|
|
is to list all directories and files which need to be backup in this
|
|
archive. The `source` property is the path of the source (dir or file).
|
|
The `dest` property is the path where it could be placed in the archive.
|
|
|
|
The list is filled by app backup scripts and system/user backup hooks.
|
|
Files located in the work_dir are automatically added.
|
|
|
|
With this list, "backup methods" are able to apply their backup strategy on
|
|
data listed in it. It's possible to tar each path (tar methods), to mount
|
|
each dir into the work_dir, to copy each files (copy method) or to call a
|
|
custom method (via a custom script).
|
|
|
|
Note: some future backups methods (like borg) are not able to specify a
|
|
different place than the original path. That's why the ynh_restore_file
|
|
helpers use primarily the SOURCE_PATH as argument.
|
|
|
|
Public properties:
|
|
info (getter)
|
|
work_dir (getter) # FIXME currently it's not a getter
|
|
is_tmp_work_dir (getter)
|
|
paths_to_backup (getter) # FIXME not a getter and list is not protected
|
|
name (getter) # FIXME currently it's not a getter
|
|
size (getter) # FIXME currently it's not a getter
|
|
|
|
Public methods:
|
|
add(self, method)
|
|
set_system_targets(self, system_parts=[])
|
|
set_apps_targets(self, apps=[])
|
|
collect_files(self)
|
|
backup(self)
|
|
|
|
Usage:
|
|
backup_manager = BackupManager(name="mybackup", description="bkp things")
|
|
|
|
# Add backup method to apply
|
|
backup_manager.add(BackupMethod.create('copy','/mnt/local_fs'))
|
|
backup_manager.add(BackupMethod.create('tar','/mnt/remote_fs'))
|
|
|
|
# Define targets to be backuped
|
|
backup_manager.set_system_targets(["data"])
|
|
backup_manager.set_apps_targets(["wordpress"])
|
|
|
|
# Collect files to backup from targets
|
|
backup_manager.collect_files()
|
|
|
|
# Apply backup methods
|
|
backup_manager.backup()
|
|
"""
|
|
|
|
def __init__(self, name=None, description='', work_dir=None):
|
|
"""
|
|
BackupManager constructor
|
|
|
|
Args:
|
|
name -- (string) The name of this backup (without spaces). If
|
|
None, the name will be generated (default: None)
|
|
|
|
description -- (string) A description for this future backup archive
|
|
(default: '')
|
|
|
|
work_dir -- (None|string) A path where prepare the archive. If None,
|
|
temporary work_dir will be created (default: None)
|
|
"""
|
|
self.description = description or ''
|
|
self.created_at = int(time.time())
|
|
self.apps_return = {}
|
|
self.system_return = {}
|
|
self.methods = []
|
|
self.paths_to_backup = []
|
|
self.size_details = {
|
|
'system': {},
|
|
'apps': {}
|
|
}
|
|
self.targets = BackupRestoreTargetsManager()
|
|
|
|
# Define backup name if needed
|
|
if not name:
|
|
name = self._define_backup_name()
|
|
self.name = name
|
|
|
|
# Define working directory if needed and initialize it
|
|
self.work_dir = work_dir
|
|
if self.work_dir is None:
|
|
self.work_dir = os.path.join(BACKUP_PATH, 'tmp', name)
|
|
self._init_work_dir()
|
|
|
|
#
|
|
# Misc helpers #
|
|
#
|
|
|
|
@property
|
|
def info(self):
|
|
"""(Getter) Dict containing info about the archive being created"""
|
|
return {
|
|
'description': self.description,
|
|
'created_at': self.created_at,
|
|
'size': self.size,
|
|
'size_details': self.size_details,
|
|
'apps': self.apps_return,
|
|
'system': self.system_return
|
|
}
|
|
|
|
@property
|
|
def is_tmp_work_dir(self):
|
|
"""(Getter) Return true if the working directory is temporary and should
|
|
be clean at the end of the backup"""
|
|
return self.work_dir == os.path.join(BACKUP_PATH, 'tmp', self.name)
|
|
|
|
def __repr__(self):
|
|
return json.dumps(self.info)
|
|
|
|
def _define_backup_name(self):
|
|
"""Define backup name
|
|
|
|
Return:
|
|
(string) A backup name created from current date 'YYMMDD-HHMMSS'
|
|
"""
|
|
# FIXME: case where this name already exist
|
|
return time.strftime('%Y%m%d-%H%M%S', time.gmtime())
|
|
|
|
def _init_work_dir(self):
|
|
"""Initialize preparation directory
|
|
|
|
Ensure the working directory exists and is empty
|
|
|
|
exception:
|
|
backup_output_directory_not_empty -- (YunohostError) Raised if the
|
|
directory was given by the user and isn't empty
|
|
|
|
(TODO) backup_cant_clean_tmp_working_directory -- (YunohostError)
|
|
Raised if the working directory isn't empty, is temporary and can't
|
|
be automaticcaly cleaned
|
|
|
|
(TODO) backup_cant_create_working_directory -- (YunohostError) Raised
|
|
if iyunohost can't create the working directory
|
|
"""
|
|
|
|
# FIXME replace isdir by exists ? manage better the case where the path
|
|
# exists
|
|
if not os.path.isdir(self.work_dir):
|
|
filesystem.mkdir(self.work_dir, 0o750, parents=True, uid='admin')
|
|
elif self.is_tmp_work_dir:
|
|
logger.debug("temporary directory for backup '%s' already exists",
|
|
self.work_dir)
|
|
# FIXME May be we should clean the workdir here
|
|
raise YunohostError('backup_output_directory_not_empty')
|
|
|
|
#
|
|
# Backup target management #
|
|
#
|
|
|
|
def set_system_targets(self, system_parts=[]):
|
|
"""
|
|
Define and validate targetted apps to be backuped
|
|
|
|
Args:
|
|
system_parts -- (list) list of system parts which should be backuped.
|
|
If empty list, all system will be backuped. If None,
|
|
no system parts will be backuped.
|
|
"""
|
|
def unknown_error(part):
|
|
logger.error(m18n.n('backup_hook_unknown', hook=part))
|
|
|
|
self.targets.set_wanted("system",
|
|
system_parts, hook_list('backup')["hooks"],
|
|
unknown_error)
|
|
|
|
def set_apps_targets(self, apps=[]):
|
|
"""
|
|
Define and validate targetted apps to be backuped
|
|
|
|
Args:
|
|
apps -- (list) list of apps which should be backuped. If given an empty
|
|
list, all apps will be backuped. If given None, no apps will be
|
|
backuped.
|
|
"""
|
|
def unknown_error(app):
|
|
logger.error(m18n.n('unbackup_app', app=app))
|
|
|
|
target_list = self.targets.set_wanted("apps", apps,
|
|
os.listdir('/etc/yunohost/apps'),
|
|
unknown_error)
|
|
|
|
# Additionnaly, we need to check that each targetted app has a
|
|
# backup and restore scripts
|
|
|
|
for app in target_list:
|
|
app_script_folder = "/etc/yunohost/apps/%s/scripts" % app
|
|
backup_script_path = os.path.join(app_script_folder, "backup")
|
|
restore_script_path = os.path.join(app_script_folder, "restore")
|
|
|
|
if not os.path.isfile(backup_script_path):
|
|
logger.warning(m18n.n('backup_with_no_backup_script_for_app', app=app))
|
|
self.targets.set_result("apps", app, "Skipped")
|
|
|
|
elif not os.path.isfile(restore_script_path):
|
|
logger.warning(m18n.n('backup_with_no_restore_script_for_app', app=app))
|
|
self.targets.set_result("apps", app, "Warning")
|
|
|
|
#
|
|
# Management of files to backup / "The CSV" #
|
|
#
|
|
|
|
def _import_to_list_to_backup(self, tmp_csv):
|
|
"""
|
|
Commit collected path from system hooks or app scripts
|
|
|
|
Args:
|
|
tmp_csv -- (string) Path to a temporary csv file with source and
|
|
destinations column to add to the list of paths to backup
|
|
"""
|
|
_call_for_each_path(self, BackupManager._add_to_list_to_backup, tmp_csv)
|
|
|
|
def _add_to_list_to_backup(self, source, dest=None):
|
|
"""
|
|
Mark file or directory to backup
|
|
|
|
This method add source/dest couple to the "paths_to_backup" list.
|
|
|
|
Args:
|
|
source -- (string) Source path to backup
|
|
|
|
dest -- (string) Destination path in the archive. If it ends by a
|
|
slash the basename of the source path will be added. If None,
|
|
the source path will be used, so source files will be set up
|
|
at the same place and with same name than on the system.
|
|
(default: None)
|
|
|
|
Usage:
|
|
self._add_to_list_to_backup('/var/www/wordpress', 'sources')
|
|
# => "wordpress" dir will be move and rename as "sources"
|
|
|
|
self._add_to_list_to_backup('/var/www/wordpress', 'sources/')
|
|
# => "wordpress" dir will be put inside "sources/" and won't be renamed
|
|
|
|
"""
|
|
if dest is None:
|
|
dest = source
|
|
source = os.path.join(self.work_dir, source)
|
|
if dest.endswith("/"):
|
|
dest = os.path.join(dest, os.path.basename(source))
|
|
self.paths_to_backup.append({'source': source, 'dest': dest})
|
|
|
|
def _write_csv(self):
|
|
"""
|
|
Write the backup list into a CSV
|
|
|
|
The goal of this csv is to list all directories and files which need to
|
|
be backup in this archive. For the moment, this CSV contains 2 columns.
|
|
The first column `source` is the path of the source (dir or file). The
|
|
second `dest` is the path where it could be placed in the archive.
|
|
|
|
This CSV is filled by app backup scripts and system/user hooks.
|
|
Files in the work_dir are automatically added.
|
|
|
|
With this CSV, "backup methods" are able to apply their backup strategy
|
|
on data listed in it. It's possible to tar each path (tar methods), to
|
|
mount each dir into the work_dir, to copy each files (copy methods) or
|
|
a custom method (via a custom script).
|
|
|
|
Note: some future backups methods (like borg) are not able to specify a
|
|
different place than the original path. That's why the ynh_restore_file
|
|
helpers use primarily the SOURCE_PATH as argument.
|
|
|
|
Error:
|
|
backup_csv_creation_failed -- Raised if the CSV couldn't be created
|
|
backup_csv_addition_failed -- Raised if we can't write in the CSV
|
|
"""
|
|
self.csv_path = os.path.join(self.work_dir, 'backup.csv')
|
|
try:
|
|
self.csv_file = open(self.csv_path, 'a')
|
|
self.fieldnames = ['source', 'dest']
|
|
self.csv = csv.DictWriter(self.csv_file, fieldnames=self.fieldnames,
|
|
quoting=csv.QUOTE_ALL)
|
|
except (IOError, OSError, csv.Error):
|
|
logger.error(m18n.n('backup_csv_creation_failed'))
|
|
|
|
for row in self.paths_to_backup:
|
|
try:
|
|
self.csv.writerow(row)
|
|
except csv.Error:
|
|
logger.error(m18n.n('backup_csv_addition_failed'))
|
|
self.csv_file.close()
|
|
|
|
#
|
|
# File collection from system parts and apps #
|
|
#
|
|
|
|
def collect_files(self):
|
|
"""
|
|
Collect all files to backup, write its into a CSV and create a
|
|
info.json file
|
|
|
|
Files to backup are listed by system parts backup hooks and by backup
|
|
app scripts that have been defined with the set_targets() method.
|
|
|
|
Some files or directories inside the working directory are added by
|
|
default:
|
|
|
|
info.json -- info about the archive
|
|
backup.csv -- a list of paths to backup
|
|
apps/ -- some apps generate here temporary files to backup (like
|
|
database dump)
|
|
conf/ -- system configuration backup scripts could generate here
|
|
temporary files to backup
|
|
data/ -- system data backup scripts could generate here temporary
|
|
files to backup
|
|
hooks/ -- restore scripts associated to system backup scripts are
|
|
copied here
|
|
|
|
Exceptions:
|
|
"backup_nothings_done" -- (YunohostError) This exception is raised if
|
|
nothing has been listed.
|
|
"""
|
|
|
|
self._collect_system_files()
|
|
self._collect_apps_files()
|
|
|
|
# Check if something has been saved ('success' or 'warning')
|
|
successfull_apps = self.targets.list("apps", include=["Success", "Warning"])
|
|
successfull_system = self.targets.list("system", include=["Success", "Warning"])
|
|
|
|
if not successfull_apps and not successfull_system:
|
|
filesystem.rm(self.work_dir, True, True)
|
|
raise YunohostError('backup_nothings_done')
|
|
|
|
# Add unlisted files from backup tmp dir
|
|
self._add_to_list_to_backup('backup.csv')
|
|
self._add_to_list_to_backup('info.json')
|
|
if len(self.apps_return) > 0:
|
|
self._add_to_list_to_backup('apps')
|
|
if os.path.isdir(os.path.join(self.work_dir, 'conf')):
|
|
self._add_to_list_to_backup('conf')
|
|
if os.path.isdir(os.path.join(self.work_dir, 'data')):
|
|
self._add_to_list_to_backup('data')
|
|
|
|
# Write CSV file
|
|
self._write_csv()
|
|
|
|
# Calculate total size
|
|
self._compute_backup_size()
|
|
|
|
# Create backup info file
|
|
with open("%s/info.json" % self.work_dir, 'w') as f:
|
|
f.write(json.dumps(self.info))
|
|
|
|
def _get_env_var(self, app=None):
|
|
"""
|
|
Define environment variables for apps or system backup scripts.
|
|
|
|
Args:
|
|
app -- (string|None) The instance name of the app we want the variable
|
|
environment. If you want a variable environment for a system backup
|
|
script keep None. (default: None)
|
|
|
|
Return:
|
|
(Dictionnary) The environment variables to apply to the script
|
|
"""
|
|
env_var = {}
|
|
|
|
_, tmp_csv = tempfile.mkstemp(prefix='backupcsv_')
|
|
env_var['YNH_BACKUP_DIR'] = self.work_dir
|
|
env_var['YNH_BACKUP_CSV'] = tmp_csv
|
|
|
|
if app is not None:
|
|
app_id, app_instance_nb = _parse_app_instance_name(app)
|
|
env_var["YNH_APP_ID"] = app_id
|
|
env_var["YNH_APP_INSTANCE_NAME"] = app
|
|
env_var["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb)
|
|
tmp_app_dir = os.path.join('apps/', app)
|
|
tmp_app_bkp_dir = os.path.join(self.work_dir, tmp_app_dir, 'backup')
|
|
env_var["YNH_APP_BACKUP_DIR"] = tmp_app_bkp_dir
|
|
|
|
return env_var
|
|
|
|
def _collect_system_files(self):
|
|
"""
|
|
List file to backup for each selected system part
|
|
|
|
This corresponds to scripts in data/hooks/backup/ (system hooks) and
|
|
to those in /etc/yunohost/hooks.d/backup/ (user hooks)
|
|
|
|
Environment variables:
|
|
YNH_BACKUP_DIR -- The backup working directory (in
|
|
"/home/yunohost.backup/tmp/BACKUPNAME" or could be
|
|
defined by the user)
|
|
YNH_BACKUP_CSV -- A temporary CSV where the script whould list paths toi
|
|
backup
|
|
"""
|
|
|
|
system_targets = self.targets.list("system", exclude=["Skipped"])
|
|
|
|
# If nothing to backup, return immediately
|
|
if system_targets == []:
|
|
return
|
|
|
|
logger.debug(m18n.n('backup_running_hooks'))
|
|
|
|
# Prepare environnement
|
|
env_dict = self._get_env_var()
|
|
|
|
# Actual call to backup scripts/hooks
|
|
|
|
ret = hook_callback('backup',
|
|
system_targets,
|
|
args=[self.work_dir],
|
|
env=env_dict,
|
|
chdir=self.work_dir)
|
|
|
|
if ret["succeed"] != []:
|
|
self.system_return = ret["succeed"]
|
|
|
|
# Add files from targets (which they put in the CSV) to the list of
|
|
# files to backup
|
|
self._import_to_list_to_backup(env_dict["YNH_BACKUP_CSV"])
|
|
|
|
# Save restoration hooks for each part that suceeded (and which have
|
|
# a restore hook available)
|
|
|
|
restore_hooks_dir = os.path.join(self.work_dir, "hooks", "restore")
|
|
if not os.path.exists(restore_hooks_dir):
|
|
filesystem.mkdir(restore_hooks_dir, mode=0o750,
|
|
parents=True, uid='admin')
|
|
|
|
restore_hooks = hook_list("restore")["hooks"]
|
|
|
|
for part in ret['succeed'].keys():
|
|
if part in restore_hooks:
|
|
part_restore_hooks = hook_info("restore", part)["hooks"]
|
|
for hook in part_restore_hooks:
|
|
self._add_to_list_to_backup(hook["path"], "hooks/restore/")
|
|
self.targets.set_result("system", part, "Success")
|
|
else:
|
|
logger.warning(m18n.n('restore_hook_unavailable', hook=part))
|
|
self.targets.set_result("system", part, "Warning")
|
|
|
|
for part in ret['failed'].keys():
|
|
logger.error(m18n.n('backup_system_part_failed', part=part))
|
|
self.targets.set_result("system", part, "Error")
|
|
|
|
def _collect_apps_files(self):
|
|
""" Prepare backup for each selected apps """
|
|
|
|
apps_targets = self.targets.list("apps", exclude=["Skipped"])
|
|
|
|
for app_instance_name in apps_targets:
|
|
self._collect_app_files(app_instance_name)
|
|
|
|
def _collect_app_files(self, app):
|
|
"""
|
|
List files to backup for the app into the paths_to_backup dict.
|
|
|
|
If the app backup script fails, paths from this app already listed for
|
|
backup aren't added to the general list and will be ignored
|
|
|
|
Environment variables:
|
|
YNH_BACKUP_DIR -- The backup working directory (in
|
|
"/home/yunohost.backup/tmp/BACKUPNAME" or could be
|
|
defined by the user)
|
|
YNH_BACKUP_CSV -- A temporary CSV where the script whould list paths toi
|
|
backup
|
|
YNH_APP_BACKUP_DIR -- The directory where the script should put
|
|
temporary files to backup like database dump,
|
|
files in this directory don't need to be added to
|
|
the temporary CSV.
|
|
YNH_APP_ID -- The app id (eg wordpress)
|
|
YNH_APP_INSTANCE_NAME -- The app instance name (eg wordpress__3)
|
|
YNH_APP_INSTANCE_NUMBER -- The app instance number (eg 3)
|
|
|
|
|
|
Args:
|
|
app -- (string) an app instance name (already installed) to backup
|
|
|
|
Exceptions:
|
|
backup_app_failed -- Raised at the end if the app backup script
|
|
execution failed
|
|
"""
|
|
app_setting_path = os.path.join('/etc/yunohost/apps/', app)
|
|
|
|
# Prepare environment
|
|
env_dict = self._get_env_var(app)
|
|
tmp_app_bkp_dir = env_dict["YNH_APP_BACKUP_DIR"]
|
|
settings_dir = os.path.join(self.work_dir, 'apps', app, 'settings')
|
|
|
|
logger.info(m18n.n("app_start_backup", app=app))
|
|
try:
|
|
# Prepare backup directory for the app
|
|
filesystem.mkdir(tmp_app_bkp_dir, 0o750, True, uid='admin')
|
|
|
|
# Copy the app settings to be able to call _common.sh
|
|
shutil.copytree(app_setting_path, settings_dir)
|
|
|
|
# Copy app backup script in a temporary folder and execute it
|
|
_, tmp_script = tempfile.mkstemp(prefix='backup_')
|
|
app_script = os.path.join(app_setting_path, 'scripts/backup')
|
|
subprocess.call(['install', '-Dm555', app_script, tmp_script])
|
|
|
|
hook_exec(tmp_script, args=[tmp_app_bkp_dir, app],
|
|
raise_on_error=True, chdir=tmp_app_bkp_dir, env=env_dict)
|
|
|
|
self._import_to_list_to_backup(env_dict["YNH_BACKUP_CSV"])
|
|
except:
|
|
abs_tmp_app_dir = os.path.join(self.work_dir, 'apps/', app)
|
|
shutil.rmtree(abs_tmp_app_dir, ignore_errors=True)
|
|
logger.exception(m18n.n('backup_app_failed', app=app))
|
|
self.targets.set_result("apps", app, "Error")
|
|
else:
|
|
# Add app info
|
|
i = app_info(app)
|
|
self.apps_return[app] = {
|
|
'version': i['version'],
|
|
'name': i['name'],
|
|
'description': i['description'],
|
|
}
|
|
self.targets.set_result("apps", app, "Success")
|
|
|
|
# Remove tmp files in all situations
|
|
finally:
|
|
filesystem.rm(tmp_script, force=True)
|
|
filesystem.rm(env_dict["YNH_BACKUP_CSV"], force=True)
|
|
|
|
#
|
|
# Actual backup archive creation / method management #
|
|
#
|
|
|
|
def add(self, method):
|
|
"""
|
|
Add a backup method that will be applied after the files collection step
|
|
|
|
Args:
|
|
method -- (BackupMethod) A backup method. Currently, you can use those:
|
|
TarBackupMethod
|
|
CopyBackupMethod
|
|
CustomBackupMethod
|
|
"""
|
|
self.methods.append(method)
|
|
|
|
def backup(self):
|
|
"""Apply backup methods"""
|
|
|
|
for method in self.methods:
|
|
logger.debug(m18n.n('backup_applying_method_' + method.method_name))
|
|
method.mount_and_backup(self)
|
|
logger.debug(m18n.n('backup_method_' + method.method_name + '_finished'))
|
|
|
|
def _compute_backup_size(self):
|
|
"""
|
|
Compute backup global size and details size for each apps and system
|
|
parts
|
|
|
|
Update self.size and self.size_details
|
|
|
|
Note: currently, these sizes are the size in this archive, not really
|
|
the size of needed to restore the archive. To know the size needed to
|
|
restore we should consider apt/npm/pip dependencies space and database
|
|
dump restore operations.
|
|
|
|
Return:
|
|
(int) The global size of the archive in bytes
|
|
"""
|
|
# FIXME Database dump will be loaded, so dump should use almost the
|
|
# double of their space
|
|
# FIXME Some archive will set up dependencies, those are not in this
|
|
# size info
|
|
self.size = 0
|
|
for system_key in self.system_return:
|
|
self.size_details['system'][system_key] = 0
|
|
for app_key in self.apps_return:
|
|
self.size_details['apps'][app_key] = 0
|
|
|
|
for row in self.paths_to_backup:
|
|
if row['dest'] != "info.json":
|
|
size = disk_usage(row['source'])
|
|
|
|
# Add size to apps details
|
|
splitted_dest = row['dest'].split('/')
|
|
category = splitted_dest[0]
|
|
if category == 'apps':
|
|
for app_key in self.apps_return:
|
|
if row['dest'].startswith('apps/' + app_key):
|
|
self.size_details['apps'][app_key] += size
|
|
break
|
|
# OR Add size to the correct system element
|
|
elif category == 'data' or category == 'conf':
|
|
for system_key in self.system_return:
|
|
if row['dest'].startswith(system_key.replace('_', '/')):
|
|
self.size_details['system'][system_key] += size
|
|
break
|
|
|
|
self.size += size
|
|
|
|
return self.size
|
|
|
|
|
|
class RestoreManager():
|
|
|
|
"""
|
|
RestoreManager allow to restore a past backup archive
|
|
|
|
Currently it's a tar.gz file, but it could be another kind of archive
|
|
|
|
Public properties:
|
|
info (getter)i # FIXME
|
|
work_dir (getter) # FIXME currently it's not a getter
|
|
name (getter) # FIXME currently it's not a getter
|
|
success (getter)
|
|
result (getter) # FIXME
|
|
|
|
Public methods:
|
|
set_targets(self, system_parts=[], apps=[])
|
|
restore(self)
|
|
|
|
Usage:
|
|
restore_manager = RestoreManager(name)
|
|
|
|
restore_manager.set_targets(None, ['wordpress__3'])
|
|
|
|
restore_manager.restore()
|
|
|
|
if restore_manager.success:
|
|
logger.success(m18n.n('restore_complete'))
|
|
|
|
return restore_manager.result
|
|
"""
|
|
|
|
def __init__(self, name, repo=None, method='tar'):
|
|
"""
|
|
RestoreManager constructor
|
|
|
|
Args:
|
|
name -- (string) Archive name
|
|
repo -- (string|None) Repository where is this archive, it could be a
|
|
path (default: /home/yunohost.backup/archives)
|
|
method -- (string) Method name to use to mount the archive
|
|
"""
|
|
# Retrieve and open the archive
|
|
# FIXME this way to get the info is not compatible with copy or custom
|
|
# backup methods
|
|
self.info = backup_info(name, with_details=True)
|
|
self.archive_path = self.info['path']
|
|
self.name = name
|
|
self.method = BackupMethod.create(method)
|
|
self.targets = BackupRestoreTargetsManager()
|
|
|
|
#
|
|
# Misc helpers #
|
|
#
|
|
|
|
@property
|
|
def success(self):
|
|
|
|
successful_apps = self.targets.list("apps", include=["Success", "Warning"])
|
|
successful_system = self.targets.list("system", include=["Success", "Warning"])
|
|
|
|
return len(successful_apps) != 0 \
|
|
or len(successful_system) != 0
|
|
|
|
def _read_info_files(self):
|
|
"""
|
|
Read the info file from inside an archive
|
|
|
|
Exceptions:
|
|
backup_invalid_archive -- Raised if we can't read the info
|
|
"""
|
|
# Retrieve backup info
|
|
info_file = os.path.join(self.work_dir, "info.json")
|
|
try:
|
|
with open(info_file, 'r') as f:
|
|
self.info = json.load(f)
|
|
|
|
# Historically, "system" was "hooks"
|
|
if "system" not in self.info.keys():
|
|
self.info["system"] = self.info["hooks"]
|
|
except IOError:
|
|
logger.debug("unable to load '%s'", info_file, exc_info=1)
|
|
raise YunohostError('backup_invalid_archive')
|
|
else:
|
|
logger.debug("restoring from backup '%s' created on %s", self.name,
|
|
datetime.utcfromtimestamp(self.info['created_at']))
|
|
|
|
def _postinstall_if_needed(self):
|
|
"""
|
|
Post install yunohost if needed
|
|
|
|
Exceptions:
|
|
backup_invalid_archive -- Raised if the current_host isn't in the
|
|
archive
|
|
"""
|
|
# Check if YunoHost is installed
|
|
if not os.path.isfile('/etc/yunohost/installed'):
|
|
# Retrieve the domain from the backup
|
|
try:
|
|
with open("%s/conf/ynh/current_host" % self.work_dir, 'r') as f:
|
|
domain = f.readline().rstrip()
|
|
except IOError:
|
|
logger.debug("unable to retrieve current_host from the backup",
|
|
exc_info=1)
|
|
# FIXME include the current_host by default ?
|
|
raise YunohostError('backup_invalid_archive')
|
|
|
|
logger.debug("executing the post-install...")
|
|
tools_postinstall(domain, 'Yunohost', True)
|
|
|
|
def clean(self):
|
|
"""
|
|
End a restore operations by cleaning the working directory and
|
|
regenerate ssowat conf (if some apps were restored)
|
|
"""
|
|
|
|
successfull_apps = self.targets.list("apps", include=["Success", "Warning"])
|
|
|
|
if successfull_apps != []:
|
|
# Quickfix: the old app_ssowatconf(auth) instruction failed due to
|
|
# ldap restore hooks
|
|
os.system('sudo yunohost app ssowatconf')
|
|
|
|
if os.path.ismount(self.work_dir):
|
|
ret = subprocess.call(["umount", self.work_dir])
|
|
if ret != 0:
|
|
logger.warning(m18n.n('restore_cleaning_failed'))
|
|
filesystem.rm(self.work_dir, True, True)
|
|
|
|
#
|
|
# Restore target manangement #
|
|
#
|
|
|
|
def set_system_targets(self, system_parts=[]):
|
|
"""
|
|
Define system parts that will be restored
|
|
|
|
Args:
|
|
system_parts -- (list) list of system parts which should be restored.
|
|
If an empty list if given, restore all system part in
|
|
the archive. If None is given, no system will be restored.
|
|
"""
|
|
|
|
def unknown_error(part):
|
|
logger.error(m18n.n("backup_archive_system_part_not_available",
|
|
part=part))
|
|
|
|
target_list = self.targets.set_wanted("system",
|
|
system_parts,
|
|
self.info['system'].keys(),
|
|
unknown_error)
|
|
|
|
# Now we need to check that the restore hook is actually available for
|
|
# all targets we want to restore
|
|
|
|
# These are the hooks on the current installation
|
|
available_restore_system_hooks = hook_list("restore")["hooks"]
|
|
|
|
for system_part in target_list:
|
|
# By default, we'll use the restore hooks on the current install
|
|
# if available
|
|
|
|
# FIXME: so if the restore hook exist we use the new one and not
|
|
# the one from backup. So hook should not break compatibility..
|
|
|
|
if system_part in available_restore_system_hooks:
|
|
continue
|
|
|
|
# Otherwise, attempt to find it (or them?) in the archive
|
|
hook_paths = '{:s}/hooks/restore/*-{:s}'.format(self.work_dir, system_part)
|
|
hook_paths = glob(hook_paths)
|
|
|
|
# If we didn't find it, we ain't gonna be able to restore it
|
|
if len(hook_paths) == 0:
|
|
logger.exception(m18n.n('restore_hook_unavailable', part=system_part))
|
|
self.targets.set_result("system", system_part, "Skipped")
|
|
continue
|
|
|
|
# Otherwise, add it from the archive to the system
|
|
# FIXME: Refactor hook_add and use it instead
|
|
custom_restore_hook_folder = os.path.join(CUSTOM_HOOK_FOLDER, 'restore')
|
|
filesystem.mkdir(custom_restore_hook_folder, 755, True)
|
|
for hook_path in hook_paths:
|
|
logger.debug("Adding restoration script '%s' to the system "
|
|
"from the backup archive '%s'", hook_path,
|
|
self.archive_path)
|
|
shutil.copy(hook_path, custom_restore_hook_folder)
|
|
|
|
def set_apps_targets(self, apps=[]):
|
|
"""
|
|
Define and validate targetted apps to be restored
|
|
|
|
Args:
|
|
apps -- (list) list of apps which should be restored. If [] is given,
|
|
all apps in the archive will be restored. If None is given,
|
|
no apps will be restored.
|
|
"""
|
|
|
|
def unknown_error(app):
|
|
logger.error(m18n.n('backup_archive_app_not_found',
|
|
app=app))
|
|
|
|
self.targets.set_wanted("apps",
|
|
apps,
|
|
self.info['apps'].keys(),
|
|
unknown_error)
|
|
|
|
#
|
|
# Archive mounting #
|
|
#
|
|
|
|
def mount(self):
|
|
"""
|
|
Mount the archive. We avoid copy to be able to restore on system without
|
|
too many space.
|
|
|
|
Use the mount method from the BackupMethod instance and read info about
|
|
this archive
|
|
|
|
Exceptions:
|
|
restore_removing_tmp_dir_failed -- Raised if it's not possible to remove
|
|
the working directory
|
|
"""
|
|
|
|
self.work_dir = os.path.join(BACKUP_PATH, "tmp", self.name)
|
|
|
|
if os.path.ismount(self.work_dir):
|
|
logger.debug("An already mounting point '%s' already exists",
|
|
self.work_dir)
|
|
ret = subprocess.call(['umount', self.work_dir])
|
|
if ret == 0:
|
|
subprocess.call(['rmdir', self.work_dir])
|
|
logger.debug("Unmount dir: {}".format(self.work_dir))
|
|
else:
|
|
raise YunohostError('restore_removing_tmp_dir_failed')
|
|
elif os.path.isdir(self.work_dir):
|
|
logger.debug("temporary restore directory '%s' already exists",
|
|
self.work_dir)
|
|
ret = subprocess.call(['rm', '-Rf', self.work_dir])
|
|
if ret == 0:
|
|
logger.debug("Delete dir: {}".format(self.work_dir))
|
|
else:
|
|
raise YunohostError('restore_removing_tmp_dir_failed')
|
|
|
|
filesystem.mkdir(self.work_dir, parents=True)
|
|
|
|
self.method.mount(self)
|
|
|
|
self._read_info_files()
|
|
|
|
#
|
|
# Space computation / checks #
|
|
#
|
|
|
|
def _compute_needed_space(self):
|
|
"""
|
|
Compute needed space to be able to restore
|
|
|
|
Return:
|
|
size -- (int) needed space to backup in bytes
|
|
margin -- (int) margin to be sure the backup don't fail by missing space
|
|
in bytes
|
|
"""
|
|
system = self.targets.list("system", exclude=["Skipped"])
|
|
apps = self.targets.list("apps", exclude=["Skipped"])
|
|
restore_all_system = (system == self.info['system'].keys())
|
|
restore_all_apps = (apps == self.info['apps'].keys())
|
|
|
|
# If complete restore operations (or legacy archive)
|
|
margin = CONF_MARGIN_SPACE_SIZE * 1024 * 1024
|
|
if (restore_all_system and restore_all_apps) or 'size_details' not in self.info:
|
|
size = self.info['size']
|
|
if 'size_details' not in self.info or \
|
|
self.info['size_details']['apps'] != {}:
|
|
margin = APP_MARGIN_SPACE_SIZE * 1024 * 1024
|
|
# Partial restore don't need all backup size
|
|
else:
|
|
size = 0
|
|
if system is not None:
|
|
for system_element in system:
|
|
size += self.info['size_details']['system'][system_element]
|
|
|
|
# TODO how to know the dependencies size ?
|
|
if apps is not None:
|
|
for app in apps:
|
|
size += self.info['size_details']['apps'][app]
|
|
margin = APP_MARGIN_SPACE_SIZE * 1024 * 1024
|
|
|
|
if not os.path.isfile('/etc/yunohost/installed'):
|
|
size += POSTINSTALL_ESTIMATE_SPACE_SIZE * 1024 * 1024
|
|
return (size, margin)
|
|
|
|
def assert_enough_free_space(self):
|
|
"""
|
|
Check available disk space
|
|
|
|
Exceptions:
|
|
restore_may_be_not_enough_disk_space -- Raised if there isn't enough
|
|
space to cover the security margin space
|
|
restore_not_enough_disk_space -- Raised if there isn't enough space
|
|
"""
|
|
|
|
free_space = free_space_in_directory(BACKUP_PATH)
|
|
|
|
(needed_space, margin) = self._compute_needed_space()
|
|
if free_space >= needed_space + margin:
|
|
return True
|
|
elif free_space > needed_space:
|
|
# TODO Add --force options to avoid the error raising
|
|
raise YunohostError('restore_may_be_not_enough_disk_space', free_space=free_space, needed_space=needed_space, margin=margin)
|
|
else:
|
|
raise YunohostError('restore_not_enough_disk_space', free_space=free_space, needed_space=needed_space, margin=margin)
|
|
|
|
#
|
|
# "Actual restore" (reverse step of the backup collect part) #
|
|
#
|
|
|
|
def restore(self):
|
|
"""
|
|
Restore the archive
|
|
|
|
Restore system parts and apps after mounting the archive, checking free
|
|
space and postinstall if needed
|
|
"""
|
|
|
|
try:
|
|
self._postinstall_if_needed()
|
|
|
|
# Apply dirty patch to redirect php5 file on php7
|
|
self._patch_backup_csv_file()
|
|
|
|
self._restore_system()
|
|
self._restore_apps()
|
|
finally:
|
|
self.clean()
|
|
|
|
def _patch_backup_csv_file(self):
|
|
"""
|
|
Apply dirty patch to redirect php5 file on php7
|
|
"""
|
|
|
|
backup_csv = os.path.join(self.work_dir, 'backup.csv')
|
|
|
|
if not os.path.isfile(backup_csv):
|
|
return
|
|
|
|
try:
|
|
contains_php5 = False
|
|
with open(backup_csv) as csvfile:
|
|
reader = csv.DictReader(csvfile, fieldnames=['source', 'dest'])
|
|
newlines = []
|
|
for row in reader:
|
|
if 'php5' in row['source']:
|
|
contains_php5 = True
|
|
row['source'] = row['source'].replace('/etc/php5', '/etc/php/7.0') \
|
|
.replace('/var/run/php5-fpm', '/var/run/php/php7.0-fpm') \
|
|
.replace('php5', 'php7')
|
|
|
|
newlines.append(row)
|
|
except (IOError, OSError, csv.Error) as e:
|
|
raise YunohostError('error_reading_file', file=backup_csv, error=str(e))
|
|
|
|
if not contains_php5:
|
|
return
|
|
|
|
try:
|
|
with open(backup_csv, 'w') as csvfile:
|
|
writer = csv.DictWriter(csvfile,
|
|
fieldnames=['source', 'dest'],
|
|
quoting=csv.QUOTE_ALL)
|
|
for row in newlines:
|
|
writer.writerow(row)
|
|
except (IOError, OSError, csv.Error) as e:
|
|
logger.warning(m18n.n('backup_php5_to_php7_migration_may_fail',
|
|
error=str(e)))
|
|
|
|
def _restore_system(self):
|
|
""" Restore user and system parts """
|
|
|
|
system_targets = self.targets.list("system", exclude=["Skipped"])
|
|
|
|
# If nothing to restore, return immediately
|
|
if system_targets == []:
|
|
return
|
|
|
|
# Start register change on system
|
|
operation_logger = OperationLogger('backup_restore_system')
|
|
operation_logger.start()
|
|
|
|
logger.debug(m18n.n('restore_running_hooks'))
|
|
|
|
env_dict = self._get_env_var()
|
|
operation_logger.extra['env'] = env_dict
|
|
operation_logger.flush()
|
|
ret = hook_callback('restore',
|
|
system_targets,
|
|
args=[self.work_dir],
|
|
env=env_dict,
|
|
chdir=self.work_dir)
|
|
|
|
for part in ret['succeed'].keys():
|
|
self.targets.set_result("system", part, "Success")
|
|
|
|
error_part = []
|
|
for part in ret['failed'].keys():
|
|
logger.error(m18n.n('restore_system_part_failed', part=part))
|
|
self.targets.set_result("system", part, "Error")
|
|
error_part.append(part)
|
|
|
|
if ret['failed']:
|
|
operation_logger.error(m18n.n('restore_system_part_failed', part=', '.join(error_part)))
|
|
else:
|
|
operation_logger.success()
|
|
|
|
service_regen_conf()
|
|
|
|
def _restore_apps(self):
|
|
"""Restore all apps targeted"""
|
|
|
|
apps_targets = self.targets.list("apps", exclude=["Skipped"])
|
|
|
|
for app in apps_targets:
|
|
self._restore_app(app)
|
|
|
|
def _restore_app(self, app_instance_name):
|
|
"""
|
|
Restore an app
|
|
|
|
Environment variables:
|
|
YNH_BACKUP_DIR -- The backup working directory (in
|
|
"/home/yunohost.backup/tmp/BACKUPNAME" or could be
|
|
defined by the user)
|
|
YNH_BACKUP_CSV -- A temporary CSV where the script whould list paths to
|
|
backup
|
|
YNH_APP_BACKUP_DIR -- The directory where the script should put
|
|
temporary files to backup like database dump,
|
|
files in this directory don't need to be added to
|
|
the temporary CSV.
|
|
YNH_APP_ID -- The app id (eg wordpress)
|
|
YNH_APP_INSTANCE_NAME -- The app instance name (eg wordpress__3)
|
|
YNH_APP_INSTANCE_NUMBER -- The app instance number (eg 3)
|
|
|
|
Args:
|
|
app_instance_name -- (string) The app name to restore (no app with this
|
|
name should be already install)
|
|
|
|
Exceptions:
|
|
restore_already_installed_app -- Raised if an app with this app instance
|
|
name already exists
|
|
restore_app_failed -- Raised if the restore bash script failed
|
|
"""
|
|
def copytree(src, dst, symlinks=False, ignore=None):
|
|
for item in os.listdir(src):
|
|
s = os.path.join(src, item)
|
|
d = os.path.join(dst, item)
|
|
if os.path.isdir(s):
|
|
shutil.copytree(s, d, symlinks, ignore)
|
|
else:
|
|
shutil.copy2(s, d)
|
|
|
|
# Start register change on system
|
|
related_to = [('app', app_instance_name)]
|
|
operation_logger = OperationLogger('backup_restore_app', related_to)
|
|
operation_logger.start()
|
|
|
|
logger.info(m18n.n("app_start_restore", app=app_instance_name))
|
|
|
|
# Check if the app is not already installed
|
|
if _is_installed(app_instance_name):
|
|
logger.error(m18n.n('restore_already_installed_app',
|
|
app=app_instance_name))
|
|
self.targets.set_result("apps", app_instance_name, "Error")
|
|
return
|
|
|
|
app_dir_in_archive = os.path.join(self.work_dir, 'apps', app_instance_name)
|
|
app_backup_in_archive = os.path.join(app_dir_in_archive, 'backup')
|
|
app_settings_in_archive = os.path.join(app_dir_in_archive, 'settings')
|
|
app_scripts_in_archive = os.path.join(app_settings_in_archive, 'scripts')
|
|
|
|
# Apply dirty patch to make php5 apps compatible with php7
|
|
_patch_php5(app_settings_in_archive)
|
|
|
|
# Delete _common.sh file in backup
|
|
common_file = os.path.join(app_backup_in_archive, '_common.sh')
|
|
filesystem.rm(common_file, force=True)
|
|
|
|
# Check if the app has a restore script
|
|
app_restore_script_in_archive = os.path.join(app_scripts_in_archive,
|
|
'restore')
|
|
if not os.path.isfile(app_restore_script_in_archive):
|
|
logger.warning(m18n.n('unrestore_app', app=app_instance_name))
|
|
self.targets.set_result("apps", app_instance_name, "Warning")
|
|
return
|
|
|
|
logger.debug(m18n.n('restore_running_app_script', app=app_instance_name))
|
|
try:
|
|
# Restore app settings
|
|
app_settings_new_path = os.path.join('/etc/yunohost/apps/',
|
|
app_instance_name)
|
|
app_scripts_new_path = os.path.join(app_settings_new_path, 'scripts')
|
|
shutil.copytree(app_settings_in_archive, app_settings_new_path)
|
|
filesystem.chmod(app_settings_new_path, 0o400, 0o400, True)
|
|
filesystem.chown(app_scripts_new_path, 'admin', None, True)
|
|
|
|
# Copy the app scripts to a writable temporary folder
|
|
# FIXME : use 'install -Dm555' or something similar to what's done
|
|
# in the backup method ?
|
|
tmp_folder_for_app_restore = tempfile.mkdtemp(prefix='restore')
|
|
copytree(app_scripts_in_archive, tmp_folder_for_app_restore)
|
|
filesystem.chmod(tmp_folder_for_app_restore, 0o550, 0o550, True)
|
|
filesystem.chown(tmp_folder_for_app_restore, 'admin', None, True)
|
|
restore_script = os.path.join(tmp_folder_for_app_restore, 'restore')
|
|
|
|
# Prepare env. var. to pass to script
|
|
env_dict = self._get_env_var(app_instance_name)
|
|
|
|
operation_logger.extra['env'] = env_dict
|
|
operation_logger.flush()
|
|
|
|
# Execute app restore script
|
|
hook_exec(restore_script,
|
|
args=[app_backup_in_archive, app_instance_name],
|
|
chdir=app_backup_in_archive,
|
|
raise_on_error=True,
|
|
env=env_dict)
|
|
except:
|
|
msg = m18n.n('restore_app_failed', app=app_instance_name)
|
|
logger.exception(msg)
|
|
operation_logger.error(msg)
|
|
|
|
self.targets.set_result("apps", app_instance_name, "Error")
|
|
|
|
remove_script = os.path.join(app_scripts_in_archive, 'remove')
|
|
|
|
# Setup environment for remove script
|
|
app_id, app_instance_nb = _parse_app_instance_name(app_instance_name)
|
|
env_dict_remove = {}
|
|
env_dict_remove["YNH_APP_ID"] = app_id
|
|
env_dict_remove["YNH_APP_INSTANCE_NAME"] = app_instance_name
|
|
env_dict_remove["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb)
|
|
|
|
operation_logger = OperationLogger('remove_on_failed_restore',
|
|
[('app', app_instance_name)],
|
|
env=env_dict_remove)
|
|
operation_logger.start()
|
|
|
|
# Execute remove script
|
|
# TODO: call app_remove instead
|
|
if hook_exec(remove_script, args=[app_instance_name],
|
|
env=env_dict_remove) != 0:
|
|
msg = m18n.n('app_not_properly_removed', app=app_instance_name)
|
|
logger.warning(msg)
|
|
operation_logger.error(msg)
|
|
else:
|
|
operation_logger.success()
|
|
|
|
# Cleaning app directory
|
|
shutil.rmtree(app_settings_new_path, ignore_errors=True)
|
|
|
|
# TODO Cleaning app hooks
|
|
else:
|
|
self.targets.set_result("apps", app_instance_name, "Success")
|
|
operation_logger.success()
|
|
finally:
|
|
# Cleaning temporary scripts directory
|
|
shutil.rmtree(tmp_folder_for_app_restore, ignore_errors=True)
|
|
|
|
def _get_env_var(self, app=None):
|
|
""" Define environment variable for hooks call """
|
|
env_var = {}
|
|
env_var['YNH_BACKUP_DIR'] = self.work_dir
|
|
env_var['YNH_BACKUP_CSV'] = os.path.join(self.work_dir, "backup.csv")
|
|
|
|
if app is not None:
|
|
app_dir_in_archive = os.path.join(self.work_dir, 'apps', app)
|
|
app_backup_in_archive = os.path.join(app_dir_in_archive, 'backup')
|
|
|
|
# Parse app instance name and id
|
|
app_id, app_instance_nb = _parse_app_instance_name(app)
|
|
|
|
env_var["YNH_APP_ID"] = app_id
|
|
env_var["YNH_APP_INSTANCE_NAME"] = app
|
|
env_var["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb)
|
|
env_var["YNH_APP_BACKUP_DIR"] = app_backup_in_archive
|
|
|
|
return env_var
|
|
|
|
#
|
|
# Backup methods #
|
|
#
|
|
|
|
|
|
class BackupMethod(object):
|
|
|
|
"""
|
|
BackupMethod is an abstract class that represents a way to backup and
|
|
restore a list of files.
|
|
|
|
Daughters of this class can be used by a BackupManager or RestoreManager
|
|
instance. Some methods are meant to be used by BackupManager and others
|
|
by RestoreManager.
|
|
|
|
BackupMethod has a factory method "create" to initialize instances.
|
|
|
|
Currently, there are 3 BackupMethods implemented:
|
|
|
|
CopyBackupMethod
|
|
----------------
|
|
This method corresponds to a raw (uncompressed) copy of files to a location,
|
|
and (could?) reverse the copy when restoring.
|
|
|
|
TarBackupMethod
|
|
---------------
|
|
This method compresses all files to backup in a .tar.gz archive. When
|
|
restoring, it untars the required parts.
|
|
|
|
CustomBackupMethod
|
|
------------------
|
|
This one use a custom bash scrip/hook "backup_method" to do the
|
|
backup/restore operations. A user can add his own hook inside
|
|
/etc/yunohost/hooks.d/backup_method/
|
|
|
|
Public properties:
|
|
method_name
|
|
|
|
Public methods:
|
|
mount_and_backup(self, backup_manager)
|
|
mount(self, restore_manager)
|
|
create(cls, method, **kwargs)
|
|
|
|
Usage:
|
|
method = BackupMethod.create("tar")
|
|
method.mount_and_backup(backup_manager)
|
|
#or
|
|
method = BackupMethod.create("copy")
|
|
method.mount(restore_manager)
|
|
"""
|
|
|
|
def __init__(self, repo=None):
|
|
"""
|
|
BackupMethod constructors
|
|
|
|
Note it is an abstract class. You should use the "create" class method
|
|
to create instance.
|
|
|
|
Args:
|
|
repo -- (string|None) A string that represent the repo where put or
|
|
get the backup. It could be a path, and in future a
|
|
BackupRepository object. If None, the default repo is used :
|
|
/home/yunohost.backup/archives/
|
|
"""
|
|
self.repo = ARCHIVES_PATH if repo is None else repo
|
|
|
|
@property
|
|
def method_name(self):
|
|
"""Return the string name of a BackupMethod (eg "tar" or "copy")"""
|
|
raise YunohostError('backup_abstract_method')
|
|
|
|
@property
|
|
def name(self):
|
|
"""Return the backup name"""
|
|
return self.manager.name
|
|
|
|
@property
|
|
def work_dir(self):
|
|
"""
|
|
Return the working directory
|
|
|
|
For a BackupManager, it is the directory where we prepare the files to
|
|
backup
|
|
|
|
For a RestoreManager, it is the directory where we mount the archive
|
|
before restoring
|
|
"""
|
|
return self.manager.work_dir
|
|
|
|
def need_mount(self):
|
|
"""
|
|
Return True if this backup method need to organize path to backup by
|
|
binding its in the working directory before to backup its.
|
|
|
|
Indeed, some methods like tar or copy method don't need to organize
|
|
files before to add it inside the archive, but others like borgbackup
|
|
are not able to organize directly the files. In this case we have the
|
|
choice to organize in the working directory before to put in the archive
|
|
or to organize after mounting the archive before the restoring
|
|
operation.
|
|
|
|
The default behaviour is to return False. To change it override the
|
|
method.
|
|
|
|
Note it's not a property because some overrided methods could do long
|
|
treatment to get this info
|
|
"""
|
|
return False
|
|
|
|
def mount_and_backup(self, backup_manager):
|
|
"""
|
|
Run the backup on files listed by the BackupManager instance
|
|
|
|
This method shouldn't be overrided, prefer overriding self.backup() and
|
|
self.clean()
|
|
|
|
Args:
|
|
backup_manager -- (BackupManager) A backup manager instance that has
|
|
already done the files collection step.
|
|
"""
|
|
self.manager = backup_manager
|
|
if self.need_mount():
|
|
self._organize_files()
|
|
|
|
try:
|
|
self.backup()
|
|
finally:
|
|
self.clean()
|
|
|
|
def mount(self, restore_manager):
|
|
"""
|
|
Mount the archive from RestoreManager instance in the working directory
|
|
|
|
This method should be extended.
|
|
|
|
Args:
|
|
restore_manager -- (RestoreManager) A restore manager instance
|
|
contains an archive to restore.
|
|
"""
|
|
self.manager = restore_manager
|
|
|
|
def clean(self):
|
|
"""
|
|
Umount sub directories of working dirextories and delete it if temporary
|
|
|
|
Exceptions:
|
|
backup_cleaning_failed -- Raise if we were not able to unmount sub
|
|
directories of the working directories
|
|
"""
|
|
if self.need_mount():
|
|
if self._recursive_umount(self.work_dir) > 0:
|
|
raise YunohostError('backup_cleaning_failed')
|
|
|
|
if self.manager.is_tmp_work_dir:
|
|
filesystem.rm(self.work_dir, True, True)
|
|
|
|
def _recursive_umount(self, directory):
|
|
"""
|
|
Recursively umount sub directories of a directory
|
|
|
|
Args:
|
|
directory -- a directory path
|
|
"""
|
|
mount_lines = subprocess.check_output("mount").split("\n")
|
|
|
|
points_to_umount = [line.split(" ")[2]
|
|
for line in mount_lines
|
|
if len(line) >= 3 and line.split(" ")[2].startswith(directory)]
|
|
ret = 0
|
|
for point in reversed(points_to_umount):
|
|
ret = subprocess.call(["umount", point])
|
|
if ret != 0:
|
|
ret = 1
|
|
logger.warning(m18n.n('backup_cleaning_failed', point))
|
|
continue
|
|
|
|
return ret
|
|
|
|
def _check_is_enough_free_space(self):
|
|
"""
|
|
Check free space in repository or output directory before to backup
|
|
|
|
Exceptions:
|
|
not_enough_disk_space -- Raise if there isn't enough space.
|
|
"""
|
|
# TODO How to do with distant repo or with deduplicated backup ?
|
|
backup_size = self.manager.size
|
|
|
|
free_space = free_space_in_directory(self.repo)
|
|
|
|
if free_space < backup_size:
|
|
logger.debug('Not enough space at %s (free: %s / needed: %d)',
|
|
self.repo, free_space, backup_size)
|
|
raise YunohostError('not_enough_disk_space', path=self.repo)
|
|
|
|
def _organize_files(self):
|
|
"""
|
|
Mount all csv src in their related path
|
|
|
|
The goal is to organize the files app by app and hook by hook, before
|
|
custom backup method or before the restore operation (in the case of an
|
|
unorganize archive).
|
|
|
|
The usage of binding could be strange for a user because the du -sb
|
|
command will return that the working directory is big.
|
|
|
|
Exceptions:
|
|
backup_unable_to_organize_files
|
|
"""
|
|
paths_needed_to_be_copied = []
|
|
for path in self.manager.paths_to_backup:
|
|
src = path['source']
|
|
|
|
if self.manager is RestoreManager:
|
|
# TODO Support to run this before a restore (and not only before
|
|
# backup). To do that RestoreManager.unorganized_work_dir should
|
|
# be implemented
|
|
src = os.path.join(self.unorganized_work_dir, src)
|
|
|
|
dest = os.path.join(self.work_dir, path['dest'])
|
|
if dest == src:
|
|
continue
|
|
dest_dir = os.path.dirname(dest)
|
|
|
|
# Be sure the parent dir of destination exists
|
|
if not os.path.isdir(dest_dir):
|
|
filesystem.mkdir(dest_dir, parents=True)
|
|
|
|
# For directory, attempt to mount bind
|
|
if os.path.isdir(src):
|
|
filesystem.mkdir(dest, parents=True, force=True)
|
|
|
|
try:
|
|
subprocess.check_call(["mount", "--rbind", src, dest])
|
|
subprocess.check_call(["mount", "-o", "remount,ro,bind", dest])
|
|
except Exception as e:
|
|
logger.warning(m18n.n("backup_couldnt_bind", src=src, dest=dest))
|
|
# To check if dest is mounted, use /proc/mounts that
|
|
# escape spaces as \040
|
|
raw_mounts = read_file("/proc/mounts").strip().split('\n')
|
|
mounts = [m.split()[1] for m in raw_mounts]
|
|
mounts = [m.replace("\\040", " ") for m in mounts]
|
|
if dest in mounts:
|
|
subprocess.check_call(["umount", "-R", dest])
|
|
else:
|
|
# Success, go to next file to organize
|
|
continue
|
|
|
|
# For files, create a hardlink
|
|
elif os.path.isfile(src) or os.path.islink(src):
|
|
# Can create a hard link only if files are on the same fs
|
|
# (i.e. we can't if it's on a different fs)
|
|
if os.stat(src).st_dev == os.stat(dest_dir).st_dev:
|
|
# Don't hardlink /etc/cron.d files to avoid cron bug
|
|
# 'NUMBER OF HARD LINKS > 1' see #1043
|
|
cron_path = os.path.abspath('/etc/cron') + '.'
|
|
if not os.path.abspath(src).startswith(cron_path):
|
|
os.link(src, dest)
|
|
# Success, go to next file to organize
|
|
continue
|
|
|
|
# If mountbind or hardlink couldnt be created,
|
|
# prepare a list of files that need to be copied
|
|
paths_needed_to_be_copied.append(path)
|
|
|
|
if len(paths_needed_to_be_copied) == 0:
|
|
return
|
|
# Manage the case where we are not able to use mount bind abilities
|
|
# It could be just for some small files on different filesystems or due
|
|
# to mounting error
|
|
|
|
# Compute size to copy
|
|
size = sum(disk_usage(path['source']) for path in paths_needed_to_be_copied)
|
|
size /= (1024 * 1024) # Convert bytes to megabytes
|
|
|
|
# Ask confirmation for copying
|
|
if size > MB_ALLOWED_TO_ORGANIZE:
|
|
try:
|
|
i = msignals.prompt(m18n.n('backup_ask_for_copying_if_needed',
|
|
answers='y/N', size=str(size)))
|
|
except NotImplemented:
|
|
raise YunohostError('backup_unable_to_organize_files')
|
|
else:
|
|
if i != 'y' and i != 'Y':
|
|
raise YunohostError('backup_unable_to_organize_files')
|
|
|
|
# Copy unbinded path
|
|
logger.debug(m18n.n('backup_copying_to_organize_the_archive',
|
|
size=str(size)))
|
|
for path in paths_needed_to_be_copied:
|
|
dest = os.path.join(self.work_dir, path['dest'])
|
|
if os.path.isdir(path['source']):
|
|
shutil.copytree(path['source'], dest, symlinks=True)
|
|
else:
|
|
shutil.copy(path['source'], dest)
|
|
|
|
@classmethod
|
|
def create(cls, method, *args):
|
|
"""
|
|
Factory method to create instance of BackupMethod
|
|
|
|
Args:
|
|
method -- (string) The method name of an existing BackupMethod. If the
|
|
name is unknown the CustomBackupMethod will be tried
|
|
|
|
... -- Specific args for the method, could be the repo target by the
|
|
method
|
|
|
|
Return a BackupMethod instance
|
|
"""
|
|
if not isinstance(method, basestring):
|
|
methods = []
|
|
for m in method:
|
|
methods.append(BackupMethod.create(m, *args))
|
|
return methods
|
|
|
|
bm_class = {
|
|
'copy': CopyBackupMethod,
|
|
'tar': TarBackupMethod,
|
|
'borg': BorgBackupMethod
|
|
}
|
|
if method in ["copy", "tar", "borg"]:
|
|
return bm_class[method](*args)
|
|
else:
|
|
return CustomBackupMethod(method=method, *args)
|
|
|
|
|
|
class CopyBackupMethod(BackupMethod):
|
|
|
|
"""
|
|
This class just do an uncompress copy of each file in a location, and
|
|
could be the inverse for restoring
|
|
"""
|
|
|
|
def __init__(self, repo=None):
|
|
super(CopyBackupMethod, self).__init__(repo)
|
|
|
|
@property
|
|
def method_name(self):
|
|
return 'copy'
|
|
|
|
def backup(self):
|
|
""" Copy prepared files into a the repo """
|
|
# Check free space in output
|
|
self._check_is_enough_free_space()
|
|
|
|
for path in self.manager.paths_to_backup:
|
|
source = path['source']
|
|
dest = os.path.join(self.repo, path['dest'])
|
|
if source == dest:
|
|
logger.debug("Files already copyed")
|
|
return
|
|
|
|
dest_parent = os.path.dirname(dest)
|
|
if not os.path.exists(dest_parent):
|
|
filesystem.mkdir(dest_parent, 0o750, True, uid='admin')
|
|
|
|
if os.path.isdir(source):
|
|
shutil.copytree(source, dest)
|
|
else:
|
|
shutil.copy(source, dest)
|
|
|
|
def mount(self):
|
|
"""
|
|
Mount the uncompress backup in readonly mode to the working directory
|
|
|
|
Exceptions:
|
|
backup_no_uncompress_archive_dir -- Raised if the repo doesn't exists
|
|
backup_cant_mount_uncompress_archive -- Raised if the binding failed
|
|
"""
|
|
# FIXME: This code is untested because there is no way to run it from
|
|
# the ynh cli
|
|
super(CopyBackupMethod, self).mount()
|
|
|
|
if not os.path.isdir(self.repo):
|
|
raise YunohostError('backup_no_uncompress_archive_dir')
|
|
|
|
filesystem.mkdir(self.work_dir, parent=True)
|
|
ret = subprocess.call(["mount", "-r", "--rbind", self.repo,
|
|
self.work_dir])
|
|
if ret == 0:
|
|
return
|
|
else:
|
|
logger.warning(m18n.n("bind_mouting_disable"))
|
|
subprocess.call(["mountpoint", "-q", self.work_dir,
|
|
"&&", "umount", "-R", self.work_dir])
|
|
raise YunohostError('backup_cant_mount_uncompress_archive')
|
|
|
|
|
|
class TarBackupMethod(BackupMethod):
|
|
|
|
"""
|
|
This class compress all files to backup in archive.
|
|
"""
|
|
|
|
def __init__(self, repo=None):
|
|
super(TarBackupMethod, self).__init__(repo)
|
|
|
|
@property
|
|
def method_name(self):
|
|
return 'tar'
|
|
|
|
@property
|
|
def _archive_file(self):
|
|
"""Return the compress archive path"""
|
|
return os.path.join(self.repo, self.name + '.tar.gz')
|
|
|
|
def backup(self):
|
|
"""
|
|
Compress prepared files
|
|
|
|
It adds the info.json in /home/yunohost.backup/archives and if the
|
|
compress archive isn't located here, add a symlink to the archive to.
|
|
|
|
Exceptions:
|
|
backup_archive_open_failed -- Raised if we can't open the archive
|
|
backup_creation_failed -- Raised if we can't write in the
|
|
compress archive
|
|
"""
|
|
|
|
if not os.path.exists(self.repo):
|
|
filesystem.mkdir(self.repo, 0o750, parents=True, uid='admin')
|
|
|
|
# Check free space in output
|
|
self._check_is_enough_free_space()
|
|
|
|
# Open archive file for writing
|
|
try:
|
|
tar = tarfile.open(self._archive_file, "w:gz")
|
|
except:
|
|
logger.debug("unable to open '%s' for writing",
|
|
self._archive_file, exc_info=1)
|
|
raise YunohostError('backup_archive_open_failed')
|
|
|
|
# Add files to the archive
|
|
try:
|
|
for path in self.manager.paths_to_backup:
|
|
# Add the "source" into the archive and transform the path into
|
|
# "dest"
|
|
tar.add(path['source'], arcname=path['dest'])
|
|
tar.close()
|
|
except IOError:
|
|
logger.error(m18n.n('backup_archive_writing_error'), exc_info=1)
|
|
raise YunohostError('backup_creation_failed')
|
|
|
|
# Move info file
|
|
shutil.copy(os.path.join(self.work_dir, 'info.json'),
|
|
os.path.join(ARCHIVES_PATH, self.name + '.info.json'))
|
|
|
|
# If backuped to a non-default location, keep a symlink of the archive
|
|
# to that location
|
|
link = os.path.join(ARCHIVES_PATH, self.name + '.tar.gz')
|
|
if not os.path.isfile(link):
|
|
os.symlink(self._archive_file, link)
|
|
|
|
def mount(self, restore_manager):
|
|
"""
|
|
Mount the archive. We avoid copy to be able to restore on system without
|
|
too many space.
|
|
|
|
Exceptions:
|
|
backup_archive_open_failed -- Raised if the archive can't be open
|
|
"""
|
|
super(TarBackupMethod, self).mount(restore_manager)
|
|
|
|
# Check the archive can be open
|
|
try:
|
|
tar = tarfile.open(self._archive_file, "r:gz")
|
|
except:
|
|
logger.debug("cannot open backup archive '%s'",
|
|
self._archive_file, exc_info=1)
|
|
raise YunohostError('backup_archive_open_failed')
|
|
tar.close()
|
|
|
|
# Mount the tarball
|
|
logger.debug(m18n.n("restore_extracting"))
|
|
tar = tarfile.open(self._archive_file, "r:gz")
|
|
tar.extract('info.json', path=self.work_dir)
|
|
|
|
try:
|
|
tar.extract('backup.csv', path=self.work_dir)
|
|
except KeyError:
|
|
# Old backup archive have no backup.csv file
|
|
pass
|
|
|
|
# Extract system parts backup
|
|
conf_extracted = False
|
|
|
|
system_targets = self.manager.targets.list("system", exclude=["Skipped"])
|
|
apps_targets = self.manager.targets.list("apps", exclude=["Skipped"])
|
|
|
|
for system_part in system_targets:
|
|
# Caution: conf_ynh_currenthost helpers put its files in
|
|
# conf/ynh
|
|
if system_part.startswith("conf_"):
|
|
if conf_extracted:
|
|
continue
|
|
system_part = "conf/"
|
|
conf_extracted = True
|
|
else:
|
|
system_part = system_part.replace("_", "/") + "/"
|
|
subdir_and_files = [
|
|
tarinfo for tarinfo in tar.getmembers()
|
|
if tarinfo.name.startswith(system_part)
|
|
]
|
|
tar.extractall(members=subdir_and_files, path=self.work_dir)
|
|
subdir_and_files = [
|
|
tarinfo for tarinfo in tar.getmembers()
|
|
if tarinfo.name.startswith("hooks/restore/")
|
|
]
|
|
tar.extractall(members=subdir_and_files, path=self.work_dir)
|
|
|
|
# Extract apps backup
|
|
for app in apps_targets:
|
|
subdir_and_files = [
|
|
tarinfo for tarinfo in tar.getmembers()
|
|
if tarinfo.name.startswith("apps/" + app)
|
|
]
|
|
tar.extractall(members=subdir_and_files, path=self.work_dir)
|
|
|
|
|
|
class BorgBackupMethod(BackupMethod):
|
|
|
|
@property
|
|
def method_name(self):
|
|
return 'borg'
|
|
|
|
def backup(self):
|
|
""" Backup prepared files with borg """
|
|
super(CopyBackupMethod, self).backup()
|
|
|
|
# TODO run borg create command
|
|
raise YunohostError('backup_borg_not_implemented')
|
|
|
|
def mount(self, mnt_path):
|
|
raise YunohostError('backup_borg_not_implemented')
|
|
|
|
|
|
class CustomBackupMethod(BackupMethod):
|
|
|
|
"""
|
|
This class use a bash script/hook "backup_method" to do the
|
|
backup/restore operations. A user can add his own hook inside
|
|
/etc/yunohost/hooks.d/backup_method/
|
|
"""
|
|
|
|
def __init__(self, repo=None, method=None, **kwargs):
|
|
super(CustomBackupMethod, self).__init__(repo)
|
|
self.args = kwargs
|
|
self.method = method
|
|
self._need_mount = None
|
|
|
|
@property
|
|
def method_name(self):
|
|
return 'borg'
|
|
|
|
def need_mount(self):
|
|
"""Call the backup_method hook to know if we need to organize files
|
|
|
|
Exceptions:
|
|
backup_custom_need_mount_error -- Raised if the hook failed
|
|
"""
|
|
if self._need_mount is not None:
|
|
return self._need_mount
|
|
|
|
ret = hook_callback('backup_method', [self.method],
|
|
args=self._get_args('need_mount'))
|
|
|
|
self._need_mount = True if ret['succeed'] else False
|
|
return self._need_mount
|
|
|
|
def backup(self):
|
|
"""
|
|
Launch a custom script to backup
|
|
|
|
Exceptions:
|
|
backup_custom_backup_error -- Raised if the custom script failed
|
|
"""
|
|
|
|
ret = hook_callback('backup_method', [self.method],
|
|
args=self._get_args('backup'))
|
|
if ret['failed']:
|
|
raise YunohostError('backup_custom_backup_error')
|
|
|
|
def mount(self, restore_manager):
|
|
"""
|
|
Launch a custom script to mount the custom archive
|
|
|
|
Exceptions:
|
|
backup_custom_mount_error -- Raised if the custom script failed
|
|
"""
|
|
super(CustomBackupMethod, self).mount(restore_manager)
|
|
ret = hook_callback('backup_method', [self.method],
|
|
args=self._get_args('mount'))
|
|
if ret['failed']:
|
|
raise YunohostError('backup_custom_mount_error')
|
|
|
|
def _get_args(self, action):
|
|
"""Return the arguments to give to the custom script"""
|
|
return [action, self.work_dir, self.name, self.repo, self.manager.size,
|
|
self.manager.description]
|
|
|
|
|
|
#
|
|
# "Front-end" #
|
|
#
|
|
|
|
def backup_create(name=None, description=None, methods=[],
|
|
output_directory=None, no_compress=False,
|
|
system=[], apps=[]):
|
|
"""
|
|
Create a backup local archive
|
|
|
|
Keyword arguments:
|
|
name -- Name of the backup archive
|
|
description -- Short description of the backup
|
|
method -- Method of backup to use
|
|
output_directory -- Output directory for the backup
|
|
no_compress -- Do not create an archive file
|
|
system -- List of system elements to backup
|
|
apps -- List of application names to backup
|
|
"""
|
|
|
|
# TODO: Add a 'clean' argument to clean output directory
|
|
|
|
#
|
|
# Validate / parse arguments #
|
|
#
|
|
|
|
# Validate there is no archive with the same name
|
|
if name and name in backup_list()['archives']:
|
|
raise YunohostError('backup_archive_name_exists')
|
|
|
|
# Validate output_directory option
|
|
if output_directory:
|
|
output_directory = os.path.abspath(output_directory)
|
|
|
|
# Check for forbidden folders
|
|
if output_directory.startswith(ARCHIVES_PATH) or \
|
|
re.match(r'^/(|(bin|boot|dev|etc|lib|root|run|sbin|sys|usr|var)(|/.*))$',
|
|
output_directory):
|
|
raise YunohostError('backup_output_directory_forbidden')
|
|
|
|
# Check that output directory is empty
|
|
if os.path.isdir(output_directory) and no_compress and \
|
|
os.listdir(output_directory):
|
|
raise YunohostError('backup_output_directory_not_empty')
|
|
elif no_compress:
|
|
raise YunohostError('backup_output_directory_required')
|
|
|
|
# Define methods (retro-compat)
|
|
if not methods:
|
|
if no_compress:
|
|
methods = ['copy']
|
|
else:
|
|
methods = ['tar'] # In future, borg will be the default actions
|
|
|
|
# If no --system or --apps given, backup everything
|
|
if system is None and apps is None:
|
|
system = []
|
|
apps = []
|
|
|
|
#
|
|
# Intialize #
|
|
#
|
|
|
|
# Create yunohost archives directory if it does not exists
|
|
_create_archive_dir()
|
|
|
|
# Prepare files to backup
|
|
if no_compress:
|
|
backup_manager = BackupManager(name, description,
|
|
work_dir=output_directory)
|
|
else:
|
|
backup_manager = BackupManager(name, description)
|
|
|
|
# Add backup methods
|
|
if output_directory:
|
|
methods = BackupMethod.create(methods, output_directory)
|
|
else:
|
|
methods = BackupMethod.create(methods)
|
|
|
|
for method in methods:
|
|
backup_manager.add(method)
|
|
|
|
# Add backup targets (system and apps)
|
|
backup_manager.set_system_targets(system)
|
|
backup_manager.set_apps_targets(apps)
|
|
|
|
#
|
|
# Collect files and put them in the archive #
|
|
#
|
|
|
|
# Collect files to be backup (by calling app backup script / system hooks)
|
|
backup_manager.collect_files()
|
|
|
|
# Apply backup methods on prepared files
|
|
logger.info(m18n.n("backup_actually_backuping"))
|
|
backup_manager.backup()
|
|
|
|
logger.success(m18n.n('backup_created'))
|
|
|
|
return {
|
|
'name': backup_manager.name,
|
|
'size': backup_manager.size,
|
|
'results': backup_manager.targets.results
|
|
}
|
|
|
|
|
|
def backup_restore(auth, name, system=[], apps=[], force=False):
|
|
"""
|
|
Restore from a local backup archive
|
|
|
|
Keyword argument:
|
|
name -- Name of the local backup archive
|
|
force -- Force restauration on an already installed system
|
|
system -- List of system parts to restore
|
|
apps -- List of application names to restore
|
|
"""
|
|
|
|
#
|
|
# Validate / parse arguments #
|
|
#
|
|
|
|
# If no --system or --apps given, restore everything
|
|
if system is None and apps is None:
|
|
system = []
|
|
apps = []
|
|
|
|
# TODO don't ask this question when restoring apps only and certain system
|
|
# parts
|
|
|
|
# Check if YunoHost is installed
|
|
if system is not None and os.path.isfile('/etc/yunohost/installed'):
|
|
logger.warning(m18n.n('yunohost_already_installed'))
|
|
if not force:
|
|
try:
|
|
# Ask confirmation for restoring
|
|
i = msignals.prompt(m18n.n('restore_confirm_yunohost_installed',
|
|
answers='y/N'))
|
|
except NotImplemented:
|
|
pass
|
|
else:
|
|
if i == 'y' or i == 'Y':
|
|
force = True
|
|
if not force:
|
|
raise YunohostError('restore_failed')
|
|
|
|
# TODO Partial app restore could not work if ldap is not restored before
|
|
# TODO repair mysql if broken and it's a complete restore
|
|
|
|
#
|
|
# Initialize #
|
|
#
|
|
|
|
restore_manager = RestoreManager(name)
|
|
|
|
restore_manager.set_system_targets(system)
|
|
restore_manager.set_apps_targets(apps)
|
|
|
|
restore_manager.assert_enough_free_space()
|
|
|
|
#
|
|
# Mount the archive then call the restore for each system part / app #
|
|
#
|
|
|
|
logger.info(m18n.n("backup_mount_archive_for_restore"))
|
|
restore_manager.mount()
|
|
restore_manager.restore()
|
|
|
|
# Check if something has been restored
|
|
if restore_manager.success:
|
|
logger.success(m18n.n('restore_complete'))
|
|
else:
|
|
raise YunohostError('restore_nothings_done')
|
|
|
|
return restore_manager.targets.results
|
|
|
|
|
|
def backup_list(with_info=False, human_readable=False):
|
|
"""
|
|
List available local backup archives
|
|
|
|
Keyword arguments:
|
|
with_info -- Show backup information for each archive
|
|
human_readable -- Print sizes in human readable format
|
|
|
|
"""
|
|
result = []
|
|
|
|
try:
|
|
# Retrieve local archives
|
|
archives = os.listdir(ARCHIVES_PATH)
|
|
except OSError:
|
|
logger.debug("unable to iterate over local archives", exc_info=1)
|
|
else:
|
|
# Iterate over local archives
|
|
for f in archives:
|
|
try:
|
|
name = f[:f.rindex('.tar.gz')]
|
|
except ValueError:
|
|
continue
|
|
result.append(name)
|
|
result.sort(key=lambda x: os.path.getctime(os.path.join(ARCHIVES_PATH, x + ".tar.gz")))
|
|
|
|
if result and with_info:
|
|
d = OrderedDict()
|
|
for a in result:
|
|
try:
|
|
d[a] = backup_info(a, human_readable=human_readable)
|
|
except YunohostError as e:
|
|
logger.warning('%s: %s' % (a, e.strerror))
|
|
|
|
result = d
|
|
|
|
return {'archives': result}
|
|
|
|
|
|
def backup_info(name, with_details=False, human_readable=False):
|
|
"""
|
|
Get info about a local backup archive
|
|
|
|
Keyword arguments:
|
|
name -- Name of the local backup archive
|
|
with_details -- Show additional backup information
|
|
human_readable -- Print sizes in human readable format
|
|
|
|
"""
|
|
archive_file = '%s/%s.tar.gz' % (ARCHIVES_PATH, name)
|
|
|
|
# Check file exist (even if it's a broken symlink)
|
|
if not os.path.lexists(archive_file):
|
|
raise YunohostError('backup_archive_name_unknown', name=name)
|
|
|
|
# If symlink, retrieve the real path
|
|
if os.path.islink(archive_file):
|
|
archive_file = os.path.realpath(archive_file)
|
|
|
|
# Raise exception if link is broken (e.g. on unmounted external storage)
|
|
if not os.path.exists(archive_file):
|
|
raise YunohostError('backup_archive_broken_link',
|
|
path=archive_file)
|
|
|
|
info_file = "%s/%s.info.json" % (ARCHIVES_PATH, name)
|
|
|
|
if not os.path.exists(info_file):
|
|
tar = tarfile.open(archive_file, "r:gz")
|
|
info_dir = info_file + '.d'
|
|
try:
|
|
tar.extract('info.json', path=info_dir)
|
|
except KeyError:
|
|
logger.debug("unable to retrieve '%s' inside the archive",
|
|
info_file, exc_info=1)
|
|
raise YunohostError('backup_invalid_archive')
|
|
else:
|
|
shutil.move(os.path.join(info_dir, 'info.json'), info_file)
|
|
finally:
|
|
tar.close()
|
|
os.rmdir(info_dir)
|
|
|
|
try:
|
|
with open(info_file) as f:
|
|
# Retrieve backup info
|
|
info = json.load(f)
|
|
except:
|
|
logger.debug("unable to load '%s'", info_file, exc_info=1)
|
|
raise YunohostError('backup_invalid_archive')
|
|
|
|
# Retrieve backup size
|
|
size = info.get('size', 0)
|
|
if not size:
|
|
tar = tarfile.open(archive_file, "r:gz")
|
|
size = reduce(lambda x, y: getattr(x, 'size', x) + getattr(y, 'size', y),
|
|
tar.getmembers())
|
|
tar.close()
|
|
if human_readable:
|
|
size = binary_to_human(size) + 'B'
|
|
|
|
result = {
|
|
'path': archive_file,
|
|
'created_at': datetime.utcfromtimestamp(info['created_at']),
|
|
'description': info['description'],
|
|
'size': size,
|
|
}
|
|
|
|
if with_details:
|
|
system_key = "system"
|
|
# Historically 'system' was 'hooks'
|
|
if "hooks" in info.keys():
|
|
system_key = "hooks"
|
|
|
|
result["apps"] = info["apps"]
|
|
result["system"] = info[system_key]
|
|
return result
|
|
|
|
|
|
def backup_delete(name):
|
|
"""
|
|
Delete a backup
|
|
|
|
Keyword arguments:
|
|
name -- Name of the local backup archive
|
|
|
|
"""
|
|
if name not in backup_list()["archives"]:
|
|
raise YunohostError('backup_archive_name_unknown',
|
|
name=name)
|
|
|
|
hook_callback('pre_backup_delete', args=[name])
|
|
|
|
archive_file = '%s/%s.tar.gz' % (ARCHIVES_PATH, name)
|
|
info_file = "%s/%s.info.json" % (ARCHIVES_PATH, name)
|
|
|
|
for backup_file in [archive_file, info_file]:
|
|
try:
|
|
os.remove(backup_file)
|
|
except:
|
|
logger.debug("unable to delete '%s'", backup_file, exc_info=1)
|
|
logger.warning(m18n.n('backup_delete_error', path=backup_file))
|
|
|
|
hook_callback('post_backup_delete', args=[name])
|
|
|
|
logger.success(m18n.n('backup_deleted'))
|
|
|
|
#
|
|
# Misc helpers #
|
|
#
|
|
|
|
|
|
def _create_archive_dir():
|
|
""" Create the YunoHost archives directory if doesn't exist """
|
|
if not os.path.isdir(ARCHIVES_PATH):
|
|
if os.path.lexists(ARCHIVES_PATH):
|
|
raise YunohostError('backup_output_symlink_dir_broken', path=ARCHIVES_PATH)
|
|
|
|
# Create the archive folder, with 'admin' as owner, such that
|
|
# people can scp archives out of the server
|
|
mkdir(ARCHIVES_PATH, mode=0o750, parents=True, uid="admin", gid="root")
|
|
|
|
|
|
def _call_for_each_path(self, callback, csv_path=None):
|
|
""" Call a callback for each path in csv """
|
|
if csv_path is None:
|
|
csv_path = self.csv_path
|
|
with open(csv_path, "r") as backup_file:
|
|
backup_csv = csv.DictReader(backup_file, fieldnames=['source', 'dest'])
|
|
for row in backup_csv:
|
|
callback(self, row['source'], row['dest'])
|
|
|
|
|
|
def free_space_in_directory(dirpath):
|
|
stat = os.statvfs(dirpath)
|
|
return stat.f_frsize * stat.f_bavail
|
|
|
|
|
|
def disk_usage(path):
|
|
# We don't do this in python with os.stat because we don't want
|
|
# to follow symlinks
|
|
|
|
du_output = subprocess.check_output(['du', '-sb', path])
|
|
return int(du_output.split()[0].decode('utf-8'))
|