# -*- coding: utf-8 -*-
#
# This file is part of INGInious. See the LICENSE and the COPYRIGHTS files for
# more information about the licensing of this file.
""" Custom installer for the web app """
import hashlib
import os
import tarfile
import tempfile
import urllib.request
from binascii import hexlify
import docker
from docker.errors import BuildError
from gridfs import GridFS
from pymongo import MongoClient
import inginious.common.custom_yaml as yaml
HEADER = '\033[95m'
INFO = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[33m'
WHITE = '\033[97m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
DOC = '\033[39m'
BACKGROUND_RED = '\033[101m'
[docs]class Installer:
""" Custom installer for the WebApp frontend """
def __init__(self, config_path=None):
self._config_path = config_path
#######################################
# Display functions #
#######################################
def _display_header(self, title):
""" Displays an header in the console """
print("")
print(BOLD + HEADER + "--- " + title + " ---" + ENDC)
def _display_warning(self, content):
""" Displays a warning in the console """
print(WARNING + "(WARN) " + content + ENDC)
def _display_info(self, content):
""" Displays an info message in the console """
print(INFO + "(INFO) " + content + ENDC)
def _display_question(self, content):
""" Displays a preamble to a question """
print(DOC + content + ENDC)
def _display_error(self, content):
""" Displays an error """
print(WHITE + BACKGROUND_RED + "(ERROR) " + content + ENDC)
def _display_big_warning(self, content):
""" Displays a BIG warning """
print("")
print(BOLD + WARNING + "--- WARNING ---" + ENDC)
print(WARNING + content + ENDC)
print("")
def _ask_with_default(self, question, default=""):
default = str(default)
answer = input(DOC + UNDERLINE + question + " [" + default + "]:" + ENDC + " ")
if answer == "":
answer = default
return answer
def _ask_boolean(self, question, default):
while True:
val = self._ask_with_default(question, ("yes" if default else "no")).lower()
if val in ["yes", "y", "1", "true", "t"]:
return True
elif val in ["no", "n", "0", "false", "f"]:
return False
self._display_question("Please answer 'yes' or 'no'.")
def _ask_integer(self, question, default):
while True:
try:
return int(self._ask_with_default(question, default))
except:
pass
#######################################
# Main function #
#######################################
[docs] def run(self):
""" Run the installator """
self._display_header("BACKEND CONFIGURATION")
options = {}
while True:
options = {}
backend = self.ask_backend()
if backend == "local":
self._display_info("Backend chosen: local. Testing the configuration.")
options = self._ask_local_config()
if not self.test_local_docker_conf():
self._display_error(
"An error occurred while testing the configuration. Please make sure you are able do run `docker info` in "
"your command line, and environment parameters like DOCKER_HOST are correctly set.")
if self._ask_boolean("Would you like to continue anyway?", False):
break
else:
break
else:
self._display_warning(
"Backend chosen: manual. As it is a really advanced feature, you will have to configure it yourself in "
"the configuration file, at the end of the setup process.")
options = {"backend": backend}
break
self._display_header("MONGODB CONFIGURATION")
mongo_opt = self.configure_mongodb()
options.update(mongo_opt)
self._display_header("TASK DIRECTORY")
task_directory_opt = self.configure_task_directory()
options.update(task_directory_opt)
self._display_header("CONTAINERS")
self.select_containers_to_build()
self._display_header("MISC")
misc_opt = self.configure_misc()
options.update(misc_opt)
database = self.try_mongodb_opts(options["mongo_opt"]["host"], options["mongo_opt"]["database"])
self._display_header("BACKUP DIRECTORY")
backup_directory_opt = self.configure_backup_directory()
options.update(backup_directory_opt)
self._display_header("AUTHENTIFICATION")
auth_opts = self.configure_authentication(database)
options.update(auth_opts)
self._display_info("You may want to add additional plugins to the configuration file.")
self._display_header("REMOTE DEBUGGING - IN BROWSER")
self._display_info(
"If you want to activate the remote debugging of task in the users' browser, you have to install separately "
"INGInious-xterm, which is available on Github, according to the parameters you have given for the hostname and the "
"port range given in the configuration of the remote debugging.")
self._display_info(
"You can leave the following question empty to disable this feature; remote debugging will still be available, "
"but not in the browser.")
webterm = self._ask_with_default(
"Please indicate the link to your installation of INGInious-xterm (for example: "
"https://your-hostname.com:8080).", "")
if webterm != "":
options["webterm"] = webterm
self._display_header("END")
file_dir = self._config_path or os.path.join(os.getcwd(), self.configuration_filename())
try:
yaml.dump(options, open(file_dir, "w"))
self._display_info("Successfully written the configuration file")
except:
self._display_error("Cannot write the configuration file on disk. Here is the content of the file")
print(yaml.dump(options))
#######################################
# Docker configuration #
#######################################
def _ask_local_config(self):
""" Ask some parameters about the local configuration """
options = {"backend": "local", "local-config": {}}
# Concurrency
while True:
concurrency = self._ask_with_default(
"Maximum concurrency (number of tasks running simultaneously). Leave it empty to use the number of "
"CPU of your host.", "")
if concurrency == "":
break
try:
concurrency = int(concurrency)
except:
self._display_error("Invalid number")
continue
if concurrency <= 0:
self._display_error("Invalid number")
continue
options["local-config"]["concurrency"] = concurrency
break
# Debug hostname
hostname = self._ask_with_default(
"What is the external hostname/address of your machine? You can leave this empty and let INGInious "
"autodetect it.", "")
if hostname != "":
options["local-config"]["debug_host"] = hostname
self._display_info(
"You can now enter the port range for the remote debugging feature of INGInious. Please verify that these "
"ports are open in your firewall. You can leave this parameters empty, the default is 64100-64200")
# Debug port range
port_range = None
while True:
start_port = self._ask_with_default("Beginning of the range", "")
if start_port != "":
try:
start_port = int(start_port)
except:
self._display_error("Invalid number")
continue
end_port = self._ask_with_default("End of the range", str(start_port + 100))
try:
end_port = int(end_port)
except:
self._display_error("Invalid number")
continue
if start_port > end_port:
self._display_error("Invalid range")
continue
port_range = str(start_port) + "-" + str(end_port)
else:
break
if port_range != None:
options["local-config"]["debug_ports"] = port_range
return options
[docs] def test_local_docker_conf(self):
""" Test to connect to a local Docker daemon """
try:
docker_connection = docker.from_env()
except Exception as e:
self._display_error("- Unable to connect to Docker. Error was %s" % str(e))
return False
try:
self._display_info("- Asking Docker some info")
if docker.utils.compare_version('1.24', docker_connection.version()['ApiVersion']) < 0:
self._display_error("- Docker version >= 1.12.0 is required.")
return False
except Exception as e:
self._display_error("- Unable to contact Docker. Error was %s" % str(e))
return False
self._display_info("- Successfully got info from Docker. Docker connection works.")
return True
[docs] def ask_backend(self):
""" Ask the user to choose the backend """
response = self._ask_boolean(
"Do you have a local docker daemon (on Linux), do you use docker-machine via a local machine, or do you use "
"Docker for macOS?", True)
if (response):
self._display_info("If you use docker-machine on macOS, please see "
"http://inginious.readthedocs.io/en/latest/install_doc/troubleshooting.html")
return "local"
else:
self._display_info(
"You will have to run inginious-backend and inginious-agent yourself. Please run the commands without argument "
"and/or read the documentation for more info")
return self._display_question("Please enter the address of your backend")
#######################################
# MONGODB CONFIGURATION #
#######################################
[docs] def try_mongodb_opts(self, host="localhost", database_name='INGInious'):
""" Try MongoDB configuration """
try:
mongo_client = MongoClient(host=host)
# Effective access only occurs when we call a method on the connexion
mongo_version = str(mongo_client.server_info()['version'])
self._display_info("Found mongodb server running version %s on %s." % (mongo_version, host))
except Exception as e:
self._display_warning("Cannot connect to MongoDB on host %s: %s" % (host, str(e)))
return None
try:
database = mongo_client[database_name]
# Effective access only occurs when we call a method on the database.
database.list_collection_names()
except Exception as e:
self._display_warning("Cannot access database %s: %s" % (database_name, str(e)))
return None
try:
# Effective access only occurs when we call a method on the gridfs object.
GridFS(database).find_one()
except Exception as e:
self._display_warning("Cannot access gridfs %s: %s" % (database_name, str(e)))
return None
return database
#######################################
# TASK DIRECTORY #
#######################################
#######################################
# CONTAINERS #
#######################################
def _build_container(self, name, folder):
self._display_info("Building container {}...".format(name))
docker_connection = docker.from_env()
docker_connection.images.build(path=folder, tag=name)
self._display_info("done.".format(name))
[docs] def select_containers_to_build(self):
if not self._ask_boolean("Build the default containers? This is highly recommended, and is required to build other containers.", True):
self._display_info("Skipping container building.")
return
try:
with tempfile.TemporaryDirectory() as tmpdirname:
self._display_info("Downloading the base container source directory...")
self._retrieve_and_extract_tarball("https://api.github.com/repos/UCL-INGI/INGInious/tarball", tmpdirname)
self._build_container("ingi/inginious-c-base", os.path.join(tmpdirname, "base-containers", "base"))
self._build_container("ingi/inginious-c-default", os.path.join(tmpdirname, "base-containers", "default"))
with tempfile.TemporaryDirectory() as tmpdirname:
self._display_info("Downloading the other containers source directory...")
self._retrieve_and_extract_tarball("https://api.github.com/repos/UCL-INGI/INGInious-containers/tarball", tmpdirname)
todo = {"ingi/inginious-c-base": None, "ingi/inginious-c-default": "ingi/inginious-c-base"}
available_containers = set(os.listdir(os.path.join(tmpdirname, 'grading')))
self._display_info("Done.")
def add_container(container):
if container in todo:
return
line_from = \
[l for l in open(os.path.join(tmpdirname, 'grading', container, 'Dockerfile')).read().split("\n") if
l.startswith("FROM")][0]
supercontainer = line_from.strip()[4:].strip().split(":")[0]
if supercontainer.startswith("ingi/") and supercontainer not in todo:
self._display_info("Container {} requires container {}, I'll build it too.".format(container, supercontainer))
add_container(supercontainer)
todo[container] = supercontainer if supercontainer.startswith("ingi/") else None
self._display_info("The following containers can be built:")
for container in available_containers:
self._display_info("\t"+container)
while True:
answer = self._ask_with_default("Indicate the name of a container to build, or press enter to continue")
if answer == "":
break
if answer not in available_containers:
self._display_warning("Unknown container. Please retry")
else:
self._display_info("Ok, I'll build container {}".format(answer))
add_container(answer)
done = {"ingi/inginious-c-base", "ingi/inginious-c-default"}
del todo["ingi/inginious-c-base"]
del todo["ingi/inginious-c-default"]
while len(todo) != 0:
todo_now = [x for x, y in todo.items() if y is None or y in done]
for x in todo_now:
del todo[x]
for container in todo_now:
try:
self._build_container("ingi/inginious-c-{}".format(container), os.path.join(tmpdirname, 'grading', container))
except BuildError:
self._display_error("An error occured while building the container. Please retry manually.")
except Exception as e:
self._display_error("An error occurred while copying the directory: {}".format(e))
#######################################
# MISC #
#######################################
[docs] def ldap_plugin(self):
""" Configures the LDAP plugin """
name = self._ask_with_default("Authentication method name (will be displayed on the login page)", "LDAP")
prefix = self._ask_with_default("Prefix to append to the username before db storage. Usefull when you have more than one auth method with "
"common usernames.", "")
ldap_host = self._ask_with_default("LDAP Host", "ldap.your.domain.com")
encryption = 'none'
while True:
encryption = self._ask_with_default("Encryption (either 'ssl', 'tls', or 'none')", 'none')
if encryption not in ['none', 'ssl', 'tls']:
self._display_error("Invalid value")
else:
break
base_dn = self._ask_with_default("Base DN", "ou=people,c=com")
request = self._ask_with_default("Request to find a user. '{}' will be replaced by the username", "uid={}")
require_cert = self._ask_boolean("Require certificate validation?", encryption is not None)
return {
"plugin_module": "inginious.frontend.plugins.auth.ldap_auth",
"host": ldap_host,
"encryption": encryption,
"base_dn": base_dn,
"request": request,
"prefix": prefix,
"name": name,
"require_cert": require_cert
}
[docs] def configuration_filename(self):
""" Returns the name of the configuration file """
return "configuration.yaml"
[docs] def support_remote_debugging(self):
""" Returns True if the frontend supports remote debugging, False else"""
return True
def _retrieve_and_extract_tarball(self, link, folder):
filename, _ = urllib.request.urlretrieve(link)
with tarfile.open(filename, mode="r:gz") as thetarfile:
members = thetarfile.getmembers()
commonpath = os.path.commonpath([tarinfo.name for tarinfo in members])
for member in members:
member.name = member.name[len(commonpath) + 1:]
if member.name:
thetarfile.extract(member, folder)