#!/usr/bin/env python
# camcops_server/camcops.py
"""
===============================================================================
Copyright (C) 2012-2018 Rudolf Cardinal (rudolf@pobox.com).
This file is part of CamCOPS.
CamCOPS is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
CamCOPS is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with CamCOPS. If not, see <http://www.gnu.org/licenses/>.
===============================================================================
"""
# SET UP LOGGING BEFORE WE IMPORT CAMCOPS MODULES, allowing them to log during
# imports (see e.g. cc_plot).
# Currently sets up colour logging even if under WSGI environment. This is fine
# for gunicorn from the command line; I'm less clear about whether the disk
# logs look polluted by ANSI codes; needs checking.
import logging
from cardinal_pythonlib.logs import (
BraceStyleAdapter,
main_only_quicksetup_rootlogger,
print_report_on_all_logs,
set_level_for_logger_and_its_handlers,
)
main_only_quicksetup_rootlogger(
logging.INFO, with_process_id=True, with_thread_id=True
)
log = BraceStyleAdapter(logging.getLogger(__name__))
log.info("CamCOPS starting")
# Main imports
from argparse import (
ArgumentParser,
ArgumentDefaultsHelpFormatter,
Namespace,
RawDescriptionHelpFormatter,
) # nopep8
import codecs # nopep8
import os # nopep8
import multiprocessing # nopep8
# from pprint import pformat # nopep8
import sys # nopep8
import tempfile # nopep8
from typing import Any, Dict, Optional, TYPE_CHECKING # nopep8
import unittest # nopep8
import cherrypy # nopep8
try:
from gunicorn.app.base import BaseApplication
except ImportError:
BaseApplication = None # e.g. on Windows: "ImportError: no module named 'fcntl'". # noqa
from pyramid.router import Router # nopep8
from wsgiref.simple_server import make_server # nopep8
from cardinal_pythonlib.argparse_func import ShowAllSubparserHelpAction # nopep8
from cardinal_pythonlib.classes import gen_all_subclasses # nopep8
from cardinal_pythonlib.debugging import pdb_run # nopep8
from cardinal_pythonlib.process import launch_external_file # nopep8
from cardinal_pythonlib.ui import ask_user, ask_user_password # nopep8
from cardinal_pythonlib.sqlalchemy.dialect import (
ALL_SQLA_DIALECTS,
SqlaDialectName,
) # nopep8
from cardinal_pythonlib.wsgi.constants import WsgiEnvVar # nopep8
from cardinal_pythonlib.wsgi.reverse_proxied_mw import (
ReverseProxiedConfig,
ReverseProxiedMiddleware,
) # nopep8
# Import this one early:
# noinspection PyUnresolvedReferences
import camcops_server.cc_modules.cc_all_models # import side effects (ensure all models registered) # noqa
from camcops_server.cc_modules.cc_alembic import (
create_database_from_scratch,
upgrade_database_to_head,
) # nopep8
from camcops_server.cc_modules.cc_baseconstants import (
ENVVAR_CONFIG_FILE,
DOCUMENTATION_INDEX_FILENAME,
) # nopep8
# noinspection PyUnresolvedReferences
import camcops_server.cc_modules.client_api # import side effects (register unit test) # nopep8
from camcops_server.cc_modules.cc_config import (
get_default_config_from_os_env, # nopep8
get_demo_apache_config,
get_demo_config,
get_demo_mysql_create_db,
get_demo_mysql_dump_script,
get_demo_supervisor_config,
)
from camcops_server.cc_modules.cc_constants import (
CAMCOPS_URL,
MINIMUM_PASSWORD_LENGTH,
USER_NAME_FOR_SYSTEM,
) # nopep8
from camcops_server.cc_modules.cc_hl7 import send_all_pending_hl7_messages # nopep8
from camcops_server.cc_modules.cc_pyramid import RouteCollection # nopep8
from camcops_server.cc_modules.cc_request import (
CamcopsRequest,
command_line_request_context,
pyramid_configurator_context,
) # nopep8
from camcops_server.cc_modules.cc_sqlalchemy import get_all_ddl # nopep8
from camcops_server.cc_modules.cc_task import Task # nopep8
from camcops_server.cc_modules.cc_unittest import (
DemoDatabaseTestCase,
DemoRequestTestCase,
ExtendedTestCase,
) # nopep8
from camcops_server.cc_modules.cc_user import (
SecurityLoginFailure,
set_password_directly,
User,
) # nopep8
from camcops_server.cc_modules.cc_version import CAMCOPS_SERVER_VERSION # nopep8
from camcops_server.cc_modules.merge_db import merge_camcops_db # nopep8
log.debug("All imports complete")
if TYPE_CHECKING:
# noinspection PyProtectedMember
from argparse import _SubParsersAction
# =============================================================================
# Check Python version (the shebang is not a guarantee)
# =============================================================================
# if sys.version_info[0] != 2 or sys.version_info[1] != 7:
# # ... sys.version_info.major (etc.) require Python 2.7 in any case!
# raise RuntimeError(
# "CamCOPS needs Python 2.7, and this Python version is: "
# + sys.version)
if sys.version_info[0] != 3:
raise RuntimeError(
"CamCOPS needs Python 3, and this Python version is: " + sys.version)
# =============================================================================
# Debugging options
# =============================================================================
DEBUG_LOG_CONFIG = False
DEBUG_RUN_WITH_PDB = False
if DEBUG_LOG_CONFIG or DEBUG_RUN_WITH_PDB:
log.warning("Debugging options enabled!")
# =============================================================================
# Other constants
# =============================================================================
DEFAULT_CONFIG_FILENAME = "/etc/camcops/camcops.conf"
DEFAULT_HOST = "127.0.0.1"
DEFAULT_MAX_THREADS = 100
# ... beware the default MySQL connection limit of 151;
# https://dev.mysql.com/doc/refman/5.7/en/too-many-connections.html
DEFAULT_PORT = 8000
URL_PATH_ROOT = '/'
# =============================================================================
# WSGI entry point
# =============================================================================
def ensure_database_is_ok() -> None:
config = get_default_config_from_os_env()
config.assert_database_ok()
[docs]def make_wsgi_app(debug_toolbar: bool = False,
reverse_proxied_config: ReverseProxiedConfig = None,
debug_reverse_proxy: bool = False) -> Router:
"""
Makes and returns a WSGI application, attaching all our special methods.
QUESTION: how do we access the WSGI environment (passed to the WSGI app)
from within a Pyramid request?
ANSWER:
.. code-block:: none
Configurator.make_wsgi_app() calls Router.__init__()
and returns: app = Router(...)
The WSGI framework uses: response = app(environ, start_response)
which therefore calls: Router.__call__(environ, start_response)
which does:
response = self.execution_policy(environ, self)
return response(environ, start_response)
So something LIKE this will be called:
Router.default_execution_policy(environ, router)
with router.request_context(environ) as request:
# ...
So the environ is handled by Router.request_context(environ)
which will call BaseRequest.__init__()
which does:
d = self.__dict__
d['environ'] = environ
so we should be able to use
request.environ # type: Dict[str, str]
"""
log.debug("Creating WSGI app")
# Make app
with pyramid_configurator_context(debug_toolbar=debug_toolbar) as config:
app = config.make_wsgi_app()
# Middleware above the Pyramid level
if reverse_proxied_config and reverse_proxied_config.necessary():
app = ReverseProxiedMiddleware(app=app,
config=reverse_proxied_config,
debug=debug_reverse_proxy)
log.debug("WSGI app created")
return app
def make_wsgi_app_from_argparse_args(args: Namespace) -> Router:
# ... matches add_wsgi_options()
reverse_proxied_config = ReverseProxiedConfig(
trusted_proxy_headers=args.trusted_proxy_headers,
http_host=args.proxy_http_host,
remote_addr=args.proxy_remote_addr,
script_name=(
args.proxy_script_name or
os.environ.get(WsgiEnvVar.SCRIPT_NAME, "")
),
server_port=args.proxy_server_port,
server_name=args.proxy_server_name,
url_scheme=args.proxy_url_scheme,
rewrite_path_info=args.proxy_rewrite_path_info,
)
return make_wsgi_app(debug_toolbar=args.debug_toolbar,
reverse_proxied_config=reverse_proxied_config,
debug_reverse_proxy=args.debug_reverse_proxy)
def test_serve_pyramid(application: Router,
host: str = DEFAULT_HOST,
port: int = DEFAULT_PORT) -> None:
ensure_database_is_ok()
server = make_server(host, port, application)
log.info("Serving on host={}, port={}".format(host, port))
server.serve_forever()
def join_url_fragments(*fragments: str) -> str:
# urllib.parse.urljoin doesn't do what we want
newfrags = [f[1:] if f.startswith("/") else f for f in fragments]
return "/".join(newfrags)
[docs]def serve_cherrypy(application: Router,
host: str,
port: int,
unix_domain_socket_filename: str,
threads_start: int,
threads_max: int, # -1 for no limit
server_name: str,
log_screen: bool,
ssl_certificate: Optional[str],
ssl_private_key: Optional[str],
root_path: str) -> None:
"""
Start CherryPy server
- Multithreading.
- Any platform.
"""
ensure_database_is_ok()
# Report on options
if unix_domain_socket_filename:
# If this is specified, it takes priority
log.info("Starting CherryPy server via UNIX domain socket at: {}",
unix_domain_socket_filename)
else:
log.info("Starting CherryPy server on host {}, port {}", host, port)
log.info("Within this web server instance, CamCOPS will be at: {}",
root_path)
log.info(
"... webview at: {}",
# urllib.parse.urljoin is useless for this
join_url_fragments(root_path, RouteCollection.HOME.path))
log.info(
"... tablet client API at: {}",
join_url_fragments(root_path, RouteCollection.CLIENT_API.path))
log.info("Thread pool starting size: {}", threads_start)
log.info("Thread pool max size: {}", threads_max)
# Set up CherryPy
cherrypy.config.update({
# See http://svn.cherrypy.org/trunk/cherrypy/_cpserver.py
'server.socket_host': host,
'server.socket_port': port,
'server.socket_file': unix_domain_socket_filename,
'server.thread_pool': threads_start,
'server.thread_pool_max': threads_max,
'server.server_name': server_name,
'server.log_screen': log_screen,
})
if ssl_certificate and ssl_private_key:
cherrypy.config.update({
'server.ssl_module': 'builtin',
'server.ssl_certificate': ssl_certificate,
'server.ssl_private_key': ssl_private_key,
})
# Mount WSGI application
cherrypy.tree.graft(application, root_path)
# Start server
try:
# log.debug("cherrypy.server.thread_pool: {}",
# cherrypy.server.thread_pool)
cherrypy.engine.start()
cherrypy.engine.block()
except KeyboardInterrupt:
cherrypy.engine.stop()
[docs]def serve_gunicorn(application: Router,
host: str,
port: int,
unix_domain_socket_filename: str,
num_workers: int,
ssl_certificate: Optional[str],
ssl_private_key: Optional[str],
reload: bool = False,
timeout_s: int = 30,
debug_show_gunicorn_options: bool = False) -> None:
"""
Start Gunicorn server
- Multiprocessing; this is a Good Thing particularly in Python; see e.g.
https://eli.thegreenplace.net/2012/01/16/python-parallelizing-cpu-bound-tasks-with-multiprocessing/ # noqa
http://www.dabeaz.com/python/UnderstandingGIL.pdf
- UNIX only.
- The Pyramid debug toolbar detects a multiprocessing web server and says
"shan't, because I use global state".
"""
if BaseApplication is None:
raise RuntimeError("Gunicorn does not run under Windows. "
"(It relies on the UNIX fork() facility.)")
ensure_database_is_ok()
# Report on options, and calculate Gunicorn versions
if unix_domain_socket_filename:
# If this is specified, it takes priority
log.info("Starting Gunicorn server via UNIX domain socket at: {}",
unix_domain_socket_filename)
bind = "unix:" + unix_domain_socket_filename
else:
log.info("Starting Gunicorn server on host {}, port {}", host, port)
bind = "{}:{}".format(host, port)
log.info("... using {} workers", num_workers)
# We encapsulate this class definition in the function, since it inherits
# from a class whose import will crash under Windows.
# http://docs.gunicorn.org/en/stable/custom.html
class StandaloneApplication(BaseApplication):
def __init__(self,
app_: Router,
options: Dict[str, Any] = None,
debug_show_known_settings: bool = False) -> None:
self.options = options or {} # type: Dict[str, Any]
self.application = app_
super().__init__()
if debug_show_known_settings:
# log.info("Gunicorn settings:\n{}", pformat(self.cfg.settings))
# ... which basically tells us to look in gunicorn/config.py
# at every class that inherits from Setting.
# Each has helpful documentation, as follows:
possible_keys = sorted(self.cfg.settings.keys())
for k in possible_keys:
v = self.cfg.settings[k]
log.info("{}:\n{}", k, v.desc)
def load_config(self):
# The Gunicorn example looks somewhat convoluted! Let's be simpler:
for key, value in self.options.items():
key_lower = key.lower()
if key_lower in self.cfg.settings and value is not None:
self.cfg.set(key_lower, value)
def load(self):
return self.application
opts = {
'bind': bind,
'certfile': ssl_certificate,
'keyfile': ssl_private_key,
'reload': reload,
'timeout': timeout_s,
'workers': num_workers,
}
app = StandaloneApplication(
application, opts,
debug_show_known_settings=debug_show_gunicorn_options)
app.run()
# =============================================================================
# Command-line functions
# =============================================================================
def launch_manual() -> None:
launch_external_file(DOCUMENTATION_INDEX_FILENAME)
def print_demo_camcops_config() -> None:
print(get_demo_config())
def print_demo_supervisor_config() -> None:
print(get_demo_supervisor_config())
def print_demo_apache_config() -> None:
print(get_demo_apache_config())
def print_demo_mysql_create_db() -> None:
print(get_demo_mysql_create_db())
def print_demo_mysql_dump_script() -> None:
print(get_demo_mysql_dump_script())
def print_database_title() -> None:
with command_line_request_context() as req:
print(req.database_title)
def generate_anonymisation_staging_db() -> None:
# generate_anonymisation_staging_db: *** BROKEN; REPLACE
db = pls.get_anonymisation_database() # may raise
ddfilename = pls.EXPORT_CRIS_DATA_DICTIONARY_TSV_FILE
classes = get_all_task_classes()
with codecs.open(ddfilename, mode="w", encoding="utf8") as f:
written_header = False
for cls in classes:
if cls.is_anonymous:
continue
# Drop, make and populate tables
cls.make_cris_tables(db)
# Add info to data dictionary
rows = cls.get_cris_dd_rows()
if not rows:
continue
if not written_header:
f.write(get_tsv_header_from_dict(rows[0]) + "\n")
written_header = True
for r in rows:
f.write(get_tsv_line_from_dict(r) + "\n")
db.commit()
log.info("Draft data dictionary written to {}".format(ddfilename))
def get_username_from_cli(req: CamcopsRequest,
prompt: str,
starting_username: str = "",
must_exist: bool = False,
must_not_exist: bool = False) -> str:
assert not (must_exist and must_not_exist)
first = True
while True:
if first:
username = starting_username
first = False
else:
username = ""
username = username or ask_user(prompt)
exists = User.user_exists(req, username)
if must_not_exist and exists:
log.error("... user already exists!")
continue
if must_exist and not exists:
log.error("... no such user!")
continue
if username == USER_NAME_FOR_SYSTEM:
log.error("... username {!r} is reserved".format(
USER_NAME_FOR_SYSTEM))
continue
return username
def get_new_password_from_cli(username: str) -> str:
while True:
password1 = ask_user_password("New password for user "
"{}".format(username))
if not password1 or len(password1) < MINIMUM_PASSWORD_LENGTH:
log.error("... passwords can't be blank or shorter than {} "
"characters".format(MINIMUM_PASSWORD_LENGTH))
continue
password2 = ask_user_password("New password for user {} "
"(again)".format(username))
if password1 != password2:
log.error("... passwords don't match; try again")
continue
return password1
[docs]def make_superuser(username: str = None) -> bool:
"""
Make a superuser from the command line.
"""
with command_line_request_context() as req:
username = get_username_from_cli(
req=req,
prompt="Username for new superuser (or to gain superuser status)",
starting_username=username,
)
existing_user = User.get_user_by_name(req.dbsession, username)
if existing_user:
log.info("Giving superuser status to {!r}".format(username))
existing_user.superuser = True
success = True
else:
log.info("Creating superuser {!r}".format(username))
password = get_new_password_from_cli(username=username)
success = User.create_superuser(req, username, password)
if success:
log.info("Success")
return True
else:
log.critical("Failed to create superuser")
return False
[docs]def reset_password(username: str = None) -> bool:
"""
Reset a password from the command line.
"""
with command_line_request_context() as req:
username = get_username_from_cli(
req=req,
prompt="Username to reset password for",
starting_username=username,
must_exist=True,
)
log.info("Resetting password for user {!r}".format(username))
password = get_new_password_from_cli(username)
success = set_password_directly(req, username, password)
if success:
log.info("Success")
else:
log.critical("Failure")
return success
[docs]def enable_user_cli(username: str = None) -> bool:
"""
Re-enable a locked user account from the command line.
"""
with command_line_request_context() as req:
if username is None:
username = get_username_from_cli(
req=req,
prompt="Username to unlock",
must_exist=True,
)
else:
if not User.user_exists(req, username):
log.critical("No such user: {!r}".format(username))
return False
SecurityLoginFailure.enable_user(req, username)
log.info("Enabled.")
return True
def send_hl7(show_queue_only: bool) -> None:
with command_line_request_context() as req:
send_all_pending_hl7_messages(req.config,
show_queue_only=show_queue_only)
# -----------------------------------------------------------------------------
# Test rig
# -----------------------------------------------------------------------------
[docs]def self_test(show_only: bool = False) -> None:
"""
Run all unit tests.
"""
with tempfile.TemporaryDirectory() as tmpdirname:
tmpconfigfilename = os.path.join(tmpdirname, "dummy_config.conf")
configtext = get_demo_config()
with open(tmpconfigfilename, "w") as file:
file.write(configtext)
# First, for safety:
os.environ[ENVVAR_CONFIG_FILE] = tmpconfigfilename
# ... we're going to be using a test (SQLite) database, but we want to
# be very sure that nothing writes to a real database! Also, we will
# want to read from this dummy config at some point.
# If you use this:
# loader = TestLoader()
# suite = loader.discover(CAMCOPS_SERVER_DIRECTORY)
# ... then it fails because submodules contain relative imports (e.g.
# "from ..cc_modules.something import x" and this gives "ValueError:
# attemped relative import beyond top-level package". As the unittest
# docs say, "In order to be compatible with test discovery, all of the
# test files must be modules or packages... importable from the
# top-level directory of the project".
#
# However, having imported everything, all our tests should be
# subclasses of TestCase... but so are some other things.
#
# So we have a choice:
# 1. manual test specification (yuk)
# 2. hack around TestCase.__subclasses__ to exclude "built-in" ones
# 3. abandon relative imports
# ... not a bad general idea (they often seem to cause problems!)
# ... however, the discovery process (a) fails with importing
# "alembic.versions", but more problematically, imports tasks
# twice, which gives errors like
# "sqlalchemy.exc.InvalidRequestError: Table 'ace3' is already
# defined for this MetaData instance."
# So, hack it is.
# noinspection PyProtectedMember
skip_testclass_subclasses = [
# The ugly hack: what you see from
# unittest.TestCase.__subclasses__() from a clean import:
unittest.case.FunctionTestCase, # built in
unittest.case._SubTest, # built in
unittest.loader._FailedTest, # built in
# plus our extras:
DemoDatabaseTestCase, # our base class
DemoRequestTestCase, # also a base class
ExtendedTestCase, # also a base class
]
suite = unittest.TestSuite()
for cls in gen_all_subclasses(unittest.TestCase):
# log.critical("Considering: {}", cls)
if cls in skip_testclass_subclasses:
continue
if not cls.__module__.startswith("camcops_server"):
# don't, for example, run cardinal_pythonlib self-tests
continue
log.info("Discovered test: {}", cls)
suite.addTest(unittest.makeSuite(cls))
if show_only:
return
runner = unittest.TextTestRunner()
runner.run(suite)
# =============================================================================
# Command-line processor
# =============================================================================
_REQNAMED = 'required named arguments'
# noinspection PyShadowingBuiltins
[docs]def add_sub(sp: "_SubParsersAction", cmd: str,
config_mandatory: Optional[bool] = False,
description: str = None,
help: str = None) -> ArgumentParser:
"""
help:
Used for the main help summary, i.e. "camcops --help".
description:
Used for the description in the detailed help, e.g.
"camcops docs --help". Defaults to "help".
config_mandatory:
None = don't ask for config
False = ask for it, but not mandatory
True = mandatory
"""
if description is None:
description = help
subparser = sp.add_parser(
cmd,
help=help,
description=description,
formatter_class=ArgumentDefaultsHelpFormatter
) # type: ArgumentParser
subparser.add_argument(
'-v', '--verbose', action='store_true',
help="Be verbose")
if config_mandatory:
cfg_help = "Configuration file"
else:
cfg_help = ("Configuration file (if not specified, the environment"
" variable {} is checked)".format(ENVVAR_CONFIG_FILE))
if config_mandatory is None:
pass
elif config_mandatory:
g = subparser.add_argument_group(_REQNAMED)
# https://stackoverflow.com/questions/24180527/argparse-required-arguments-listed-under-optional-arguments # noqa
g.add_argument("--config", required=True, help=cfg_help)
else:
subparser.add_argument("--config", help=cfg_help)
return subparser
# noinspection PyShadowingBuiltins
def add_req_named(sp: ArgumentParser, switch: str, help: str,
action: str = None) -> None:
# noinspection PyProtectedMember
reqgroup = next((g for g in sp._action_groups
if g.title == _REQNAMED), None)
if not reqgroup:
reqgroup = sp.add_argument_group(_REQNAMED)
reqgroup.add_argument(switch, required=True, action=action, help=help)
def add_wsgi_options(sp: ArgumentParser) -> None:
sp.add_argument(
"--trusted_proxy_headers", type=str, nargs="*",
help=(
"Trust these WSGI environment variables for when the server "
"is behind a reverse proxy (e.g. an Apache front-end web "
"server). Options: {!r}".format(
ReverseProxiedMiddleware.ALL_CANDIDATES)
)
)
sp.add_argument(
'--proxy_http_host', type=str, default=None,
help=(
"Option to set the WSGI HTTP host directly. "
"This affects the WSGI variable {w}. If not specified, "
"trusted variables within {v!r} will be used.".format(
w=WsgiEnvVar.HTTP_HOST,
v=ReverseProxiedMiddleware.CANDIDATES_HTTP_HOST,
)
)
)
sp.add_argument(
'--proxy_remote_addr', type=str, default=None,
help=(
"Option to set the WSGI remote address directly. "
"This affects the WSGI variable {w}. If not specified, "
"trusted variables within {v!r} will be used.".format(
w=WsgiEnvVar.REMOTE_ADDR,
v=ReverseProxiedMiddleware.CANDIDATES_REMOTE_ADDR,
)
)
)
sp.add_argument(
"--proxy_script_name", type=str, default=None,
help=(
"Path at which this script is mounted. Set this if you are "
"hosting this CamCOPS instance at a non-root path, unless you "
"set trusted WSGI headers instead. For example, "
"if you are running an Apache server and want this instance "
"of CamCOPS to appear at /somewhere/camcops, then (a) "
"configure your Apache instance to proxy requests to "
"/somewhere/camcops/... to this server (e.g. via an internal "
"TCP/IP port or UNIX socket) and specify this option. If this "
"option is not set, then the OS environment variable {sn} "
"will be checked as well, and if that is not set, trusted "
"variables within {v!r} will be used. This option affects the "
"WSGI variables {sn} and {pi}.".format(
sn=WsgiEnvVar.SCRIPT_NAME,
pi=WsgiEnvVar.PATH_INFO,
v=ReverseProxiedMiddleware.CANDIDATES_SCRIPT_NAME,
)
)
)
sp.add_argument(
'--proxy_server_port', type=int, default=None,
help=(
"Option to set the WSGI server port directly. "
"This affects the WSGI variable {w}. If not specified, "
"trusted variables within {v!r} will be used.".format(
w=WsgiEnvVar.SERVER_PORT,
v=ReverseProxiedMiddleware.CANDIDATES_SERVER_PORT,
)
)
)
sp.add_argument(
'--proxy_server_name', type=str, default=None,
help=(
"Option to set the WSGI server name directly. "
"This affects the WSGI variable {w}. If not specified, "
"trusted variables within {v!r} will be used.".format(
w=WsgiEnvVar.SERVER_NAME,
v=ReverseProxiedMiddleware.CANDIDATES_SERVER_NAME,
)
)
)
sp.add_argument(
'--proxy_url_scheme', type=str, default=None,
help=(
"Option to set the WSGI scheme (e.g. http, https) directly. "
"This affects the WSGI variable {w}. If not specified, "
"trusted variables within {v!r} will be used.".format(
w=WsgiEnvVar.WSGI_URL_SCHEME,
v=ReverseProxiedMiddleware.CANDIDATES_URL_SCHEME,
)
)
)
sp.add_argument(
'--proxy_rewrite_path_info', action="store_true",
help=(
"If SCRIPT_NAME is rewritten, this option causes PATH_INFO to "
"be rewritten, if it starts with SCRIPT_NAME, to strip off "
"SCRIPT_NAME. Appropriate for some front-end web browsers "
"with limited reverse proxying support (but do not use for "
"Apache with ProxyPass, because that rewrites incoming URLs "
"properly)."
)
)
sp.add_argument(
'--debug_reverse_proxy', action="store_true",
help="For --behind_reverse_proxy: show debugging information as "
"WSGI variables are rewritten."
)
sp.add_argument(
'--debug_toolbar', action="store_true",
help="Enable the Pyramid debug toolbar"
)
[docs]def camcops_main() -> None:
"""
Command-line entry point.
Note that we can't easily use delayed imports to speed up the help output,
because the help system has function calls embedded into it.
"""
# Fetch command-line options.
# -------------------------------------------------------------------------
# Base parser
# -------------------------------------------------------------------------
parser = ArgumentParser(
prog="camcops", # name the user will use to call it
description="""CamCOPS server version {}, by Rudolf Cardinal.
Use 'camcops <COMMAND> --help' for more detail on each command.""".format(
CAMCOPS_SERVER_VERSION),
formatter_class=RawDescriptionHelpFormatter,
# add_help=False # only do this if manually overriding the method
)
parser.add_argument(
'--allhelp',
action=ShowAllSubparserHelpAction,
help='show help for all commands and exit')
parser.add_argument(
"--version", action="version",
version="CamCOPS {}".format(CAMCOPS_SERVER_VERSION))
# -------------------------------------------------------------------------
# Subcommand subparser
# -------------------------------------------------------------------------
subparsers = parser.add_subparsers(
title="commands",
description="Valid CamCOPS commands are as follows.",
help='Specify one command.',
dest='command', # sorts out the help for the command being mandatory
# https://stackoverflow.com/questions/23349349/argparse-with-required-subparser # noqa
) # type: _SubParsersAction # noqa
subparsers.required = True # requires a command
# You can't use "add_subparsers" more than once.
# Subparser groups seem not yet to be supported:
# https://bugs.python.org/issue9341
# https://bugs.python.org/issue14037
# -------------------------------------------------------------------------
# Getting started commands
# -------------------------------------------------------------------------
docs_parser = add_sub(
subparsers, "docs", config_mandatory=None,
help="Launch the main documentation (CamCOPS manual)"
)
docs_parser.set_defaults(
func=lambda args: launch_manual())
democonfig_parser = add_sub(
subparsers, "demo_camcops_config", config_mandatory=None,
help="Print a demo CamCOPS config file")
democonfig_parser.set_defaults(
func=lambda args: print_demo_camcops_config())
demosupervisorconf_parser = add_sub(
subparsers, "demo_supervisor_config", config_mandatory=None,
help="Print a demo 'supervisor' config file for CamCOPS")
demosupervisorconf_parser.set_defaults(
func=lambda args: print_demo_supervisor_config())
demoapacheconf_parser = add_sub(
subparsers, "demo_apache_config", config_mandatory=None,
help="Print a demo Apache config file section for CamCOPS")
demoapacheconf_parser.set_defaults(
func=lambda args: print_demo_apache_config())
demo_mysql_create_db_parser = add_sub(
subparsers, "demo_mysql_create_db", config_mandatory=None,
help="Print demo instructions to create a MySQL database for CamCOPS")
demo_mysql_create_db_parser.set_defaults(
func=lambda args: print_demo_mysql_create_db())
demo_mysql_dump_script_parser = add_sub(
subparsers, "demo_mysql_dump_script", config_mandatory=None,
help="Print demo instructions to dump all current MySQL databases")
demo_mysql_dump_script_parser.set_defaults(
func=lambda args: print_demo_mysql_dump_script())
# -------------------------------------------------------------------------
# Database commands
# -------------------------------------------------------------------------
upgradedb_parser = add_sub(
subparsers, "upgrade_db", config_mandatory=True,
help="Upgrade database to most recent version (via Alembic)")
upgradedb_parser.set_defaults(
func=lambda args: upgrade_database_to_head(show_sql_only=False))
show_upgrade_sql_parser = add_sub(
subparsers, "show_upgrade_sql", config_mandatory=True,
help="Show SQL for upgrading database (to stdout)")
show_upgrade_sql_parser.set_defaults(
func=lambda args: upgrade_database_to_head(show_sql_only=True))
showdbtitle_parser = add_sub(
subparsers, "show_db_title",
help="Show database title")
showdbtitle_parser.set_defaults(
func=lambda args: print_database_title())
mergedb_parser = add_sub(
subparsers, "merge_db", config_mandatory=True,
help="Merge in data from an old or recent CamCOPS database")
mergedb_parser.add_argument(
'--report_every', type=int, default=10000,
help="Report progress every n rows")
mergedb_parser.add_argument(
'--echo', action="store_true",
help="Echo SQL to source database")
mergedb_parser.add_argument(
'--dummy_run', action="store_true",
help="Perform a dummy run only; do not alter destination database")
mergedb_parser.add_argument(
'--info_only', action="store_true",
help="Show table information only; don't do any work")
mergedb_parser.add_argument(
'--skip_hl7_logs', action="store_true",
help="Skip the HL7 message log table")
mergedb_parser.add_argument(
'--skip_audit_logs', action="store_true",
help="Skip the audit log table")
mergedb_parser.add_argument(
'--default_group_id', type=int, default=None,
help="Default group ID (integer) to apply to old records without one. "
"If none is specified, a new group will be created for such "
"records.")
mergedb_parser.add_argument(
'--default_group_name', type=str, default=None,
help="If default_group_id is not specified, use this group name. The "
"group will be looked up if it exists, and created if not.")
add_req_named(
mergedb_parser,
"--src",
help="Source database (specified as an SQLAlchemy URL). The contents "
"of this database will be merged into the database specified "
"in the config file.")
mergedb_parser.set_defaults(func=lambda args: merge_camcops_db(
src=args.src,
echo=args.echo,
report_every=args.report_every,
dummy_run=args.dummy_run,
info_only=args.info_only,
skip_hl7_logs=args.skip_hl7_logs,
skip_audit_logs=args.skip_audit_logs,
default_group_id=args.default_group_id,
default_group_name=args.default_group_name,
))
# WATCH OUT. There appears to be a bug somewhere in the way that the
# Pyramid debug toolbar registers itself with SQLAlchemy (see
# pyramid_debugtoolbar/panels/sqla.py; look for "before_cursor_execute"
# and "after_cursor_execute". Somehow, some connections (but not all) seem
# to get this event registered twice. The upshot is that the sequence can
# lead to an attempt to double-delete the debug toolbar's timer:
#
# _before_cursor_execute: <sqlalchemy.engine.base.Connection object at 0x7f5c1fa7c630>, 'SHOW CREATE TABLE `_hl7_run_log`', () # noqa
# _before_cursor_execute: <sqlalchemy.engine.base.Connection object at 0x7f5c1fa7c630>, 'SHOW CREATE TABLE `_hl7_run_log`', () # noqa
# ^^^ this is the problem: event called twice
# _after_cursor_execute: <sqlalchemy.engine.base.Connection object at 0x7f5c1fa7c630>, 'SHOW CREATE TABLE `_hl7_run_log`', () # noqa
# _after_cursor_execute: <sqlalchemy.engine.base.Connection object at 0x7f5c1fa7c630>, 'SHOW CREATE TABLE `_hl7_run_log`', () # noqa
# ^^^ and this is where the problem becomes evident
# Traceback (most recent call last):
# ...
# File "/home/rudolf/dev/venvs/camcops/lib/python3.5/site-packages/pyramid_debugtoolbar/panels/sqla.py", line 51, in _after_cursor_execute # noqa
# delattr(conn, 'pdtb_start_timer')
# AttributeError: pdtb_start_timer
#
# So the simplest thing is only to register the debug toolbar for stuff
# that might need it...
createdb_parser = add_sub(
subparsers, "create_db", config_mandatory=True,
help="Create CamCOPS database from scratch (AVOID; use the upgrade "
"facility instead)")
add_req_named(
createdb_parser,
'--confirm_create_db', action="store_true",
help="Must specify this too, as a safety measure")
createdb_parser.set_defaults(
func=lambda args: create_database_from_scratch(
cfg=get_default_config_from_os_env()
))
# db_group.add_argument(
# "-s", "--summarytables", action="store_true", default=False,
# help="Make summary tables")
# -------------------------------------------------------------------------
# User commands
# -------------------------------------------------------------------------
superuser_parser = add_sub(
subparsers, "make_superuser",
help="Make superuser, or give superuser status to an existing user")
superuser_parser.add_argument(
'--username',
help="Username of superuser to create/promote (if omitted, you will "
"be asked to type it in)")
superuser_parser.set_defaults(func=lambda args: make_superuser(
username=args.username
))
password_parser = add_sub(
subparsers, "reset_password",
help="Reset a user's password")
password_parser.add_argument(
'--username',
help="Username to change password for (if omitted, you will be asked "
"to type it in)")
password_parser.set_defaults(func=lambda args: reset_password(
username=args.username
))
enableuser_parser = add_sub(
subparsers, "enable_user",
help="Re-enable a locked user account")
enableuser_parser.add_argument(
'--username',
help="Username to enable (if omitted, you will be asked "
"to type it in)")
enableuser_parser.set_defaults(func=lambda args: enable_user_cli(
username=args.username
))
# -------------------------------------------------------------------------
# Export options
# -------------------------------------------------------------------------
ddl_parser = add_sub(
subparsers, "ddl",
help="Print database schema (data definition language; DDL)")
ddl_parser.add_argument(
'--dialect', type=str, default=SqlaDialectName.MYSQL,
help="SQL dialect (options: {})".format(", ".join(ALL_SQLA_DIALECTS)))
ddl_parser.set_defaults(
func=lambda args: print(get_all_ddl(dialect_name=args.dialect)))
hl7_parser = add_sub(
subparsers, "hl7",
help="Send pending HL7 messages and outbound files")
hl7_parser.set_defaults(
func=lambda args: send_hl7(show_queue_only=False))
showhl7queue_parser = add_sub(
subparsers, "show_hl7_queue",
help="View outbound HL7/file queue (without sending)")
showhl7queue_parser.set_defaults(
func=lambda args: send_hl7(show_queue_only=True))
# *** ANONYMOUS STAGING DATABASE DISABLED TEMPORARILY
# anonstaging_parser = add_sub(
# subparsers, "anonstaging",
# help="Generate/regenerate anonymisation staging database")
# anonstaging_parser.set_defaults(
# func=lambda args: generate_anonymisation_staging_db())
# -------------------------------------------------------------------------
# Test options
# -------------------------------------------------------------------------
showtests_parser = add_sub(
subparsers, "show_tests", config_mandatory=None,
help="Show available self-tests")
showtests_parser.set_defaults(func=lambda args: self_test(show_only=True))
selftest_parser = add_sub(
subparsers, "self_test", config_mandatory=None,
help="Test internal code")
selftest_parser.set_defaults(func=lambda args: self_test())
serve_pyr_parser = add_sub(
subparsers, "serve_pyramid",
help="Test web server (single-thread, single-process, HTTP-only, "
"Pyramid; for development use only")
serve_pyr_parser.add_argument(
'--host', type=str, default=DEFAULT_HOST,
help="Hostname to listen on")
serve_pyr_parser.add_argument(
'--port', type=int, default=DEFAULT_PORT,
help="Port to listen on")
add_wsgi_options(serve_pyr_parser)
serve_pyr_parser.set_defaults(func=lambda args: test_serve_pyramid(
application=make_wsgi_app_from_argparse_args(args),
host=args.host,
port=args.port
))
# -------------------------------------------------------------------------
# Web server options
# -------------------------------------------------------------------------
serve_cp_parser = add_sub(
subparsers, "serve_cherrypy",
help="Start web server (via CherryPy)")
serve_cp_parser.add_argument(
"--serve", action="store_true",
help="")
serve_cp_parser.add_argument(
'--host', type=str, default=DEFAULT_HOST,
help="hostname to listen on")
serve_cp_parser.add_argument(
'--port', type=int, default=DEFAULT_PORT,
help="port to listen on")
serve_cp_parser.add_argument(
'--unix_domain_socket', type=str, default="",
help="UNIX domain socket to listen on (overrides host/port if "
"specified)")
serve_cp_parser.add_argument(
"--server_name", type=str, default="localhost",
help="CherryPy's SERVER_NAME environ entry")
serve_cp_parser.add_argument(
"--threads_start", type=int, default=10,
help="Number of threads for server to start with")
serve_cp_parser.add_argument(
"--threads_max", type=int, default=DEFAULT_MAX_THREADS,
help="Maximum number of threads for server to use (-1 for no limit) "
"(BEWARE exceeding the permitted number of database connections)")
serve_cp_parser.add_argument(
"--ssl_certificate", type=str,
help="SSL certificate file "
"(e.g. /etc/ssl/certs/ssl-cert-snakeoil.pem)")
serve_cp_parser.add_argument(
"--ssl_private_key", type=str,
help="SSL private key file "
"(e.g. /etc/ssl/private/ssl-cert-snakeoil.key)")
serve_cp_parser.add_argument(
"--log_screen", dest="log_screen", action="store_true",
help="Log access requests etc. to terminal (default)")
serve_cp_parser.add_argument(
"--no_log_screen", dest="log_screen", action="store_false",
help="Don't log access requests etc. to terminal")
serve_cp_parser.set_defaults(log_screen=True)
serve_cp_parser.add_argument(
"--root_path", type=str, default=URL_PATH_ROOT,
help=(
"Root path to serve CRATE at, WITHIN this CherryPy web server "
"instance. (There is unlikely to be a reason to use something "
"other than '/'; do not confuse this with the mount point "
"within a wider, e.g. Apache, configuration, which is set "
"instead by the WSGI variable {}; see the "
"--trusted_proxy_headers and --proxy_script_name "
"options.)".format(WsgiEnvVar.SCRIPT_NAME)
)
)
add_wsgi_options(serve_cp_parser)
serve_cp_parser.set_defaults(func=lambda args: serve_cherrypy(
application=make_wsgi_app_from_argparse_args(args),
host=args.host,
port=args.port,
threads_start=args.threads_start,
threads_max=args.threads_max,
unix_domain_socket_filename=args.unix_domain_socket,
server_name=args.server_name,
log_screen=args.log_screen,
ssl_certificate=args.ssl_certificate,
ssl_private_key=args.ssl_private_key,
root_path=args.root_path,
))
cpu_count = multiprocessing.cpu_count()
serve_gu_parser = add_sub(
subparsers, "serve_gunicorn",
help="Start web server (via Gunicorn) (not available under Windows)")
serve_gu_parser.add_argument(
"--serve", action="store_true",
help="")
serve_gu_parser.add_argument(
'--host', type=str, default=DEFAULT_HOST,
help="hostname to listen on")
serve_gu_parser.add_argument(
'--port', type=int, default=DEFAULT_PORT,
help="port to listen on")
serve_gu_parser.add_argument(
'--unix_domain_socket', type=str, default="",
help="UNIX domain socket to listen on (overrides host/port if "
"specified)")
serve_gu_parser.add_argument(
"--num_workers", type=int, default=cpu_count * 2,
help="Number of worker processes for server to use")
serve_gu_parser.add_argument(
"--debug_reload", action="store_true",
help="Debugging option: reload Gunicorn upon code change")
serve_gu_parser.add_argument(
"--ssl_certificate", type=str,
help="SSL certificate file "
"(e.g. /etc/ssl/certs/ssl-cert-snakeoil.pem)")
serve_gu_parser.add_argument(
"--ssl_private_key", type=str,
help="SSL private key file "
"(e.g. /etc/ssl/private/ssl-cert-snakeoil.key)")
serve_gu_parser.add_argument(
"--timeout", type=int, default=30,
help="Gunicorn worker timeout (s)"
)
serve_gu_parser.add_argument(
"--debug_show_gunicorn_options", action="store_true",
help="Debugging option: show possible Gunicorn settings"
)
serve_gu_parser.set_defaults(log_screen=True)
add_wsgi_options(serve_gu_parser)
serve_gu_parser.set_defaults(func=lambda args: serve_gunicorn(
application=make_wsgi_app_from_argparse_args(args),
host=args.host,
port=args.port,
num_workers=args.num_workers,
unix_domain_socket_filename=args.unix_domain_socket,
ssl_certificate=args.ssl_certificate,
ssl_private_key=args.ssl_private_key,
reload=args.debug_reload,
timeout_s=args.timeout,
debug_show_gunicorn_options=args.debug_show_gunicorn_options,
))
# -------------------------------------------------------------------------
# OK; parse the arguments
# -------------------------------------------------------------------------
progargs = parser.parse_args()
# Initial log level (overridden later by config file but helpful for start)
loglevel = logging.DEBUG if progargs.verbose else logging.INFO
rootlogger = logging.getLogger()
set_level_for_logger_and_its_handlers(rootlogger, loglevel)
# Say hello
log.info("CamCOPS server version {}", CAMCOPS_SERVER_VERSION)
log.info("By Rudolf Cardinal. See {}", CAMCOPS_URL)
log.info("Using {} tasks", len(Task.all_subclasses_by_tablename()))
log.debug("Command-line arguments: {!r}", progargs)
if DEBUG_LOG_CONFIG:
print_report_on_all_logs()
# Finalize the config filename
if hasattr(progargs, 'config') and progargs.config:
# We want the the config filename in the environment from now on:
os.environ[ENVVAR_CONFIG_FILE] = progargs.config
cfg_name = os.environ.get(ENVVAR_CONFIG_FILE, None)
log.info("Using configuration file: {!r}", cfg_name)
if progargs.func is None:
raise NotImplementedError("Command-line function not implemented!")
success = progargs.func(progargs) # type: Optional[bool]
if success is None or success is True:
sys.exit(0)
else:
sys.exit(1)
# =============================================================================
# Command-line entry point
# =============================================================================
def main():
if DEBUG_RUN_WITH_PDB:
pdb_run(camcops_main)
else:
camcops_main()
if __name__ == '__main__':
main()