Coverage for cc_modules/cc_config.py : 64%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python
3# noinspection HttpUrlsUsage
4"""
5camcops_server/cc_modules/cc_config.py
7===============================================================================
9 Copyright (C) 2012-2020 Rudolf Cardinal (rudolf@pobox.com).
11 This file is part of CamCOPS.
13 CamCOPS is free software: you can redistribute it and/or modify
14 it under the terms of the GNU General Public License as published by
15 the Free Software Foundation, either version 3 of the License, or
16 (at your option) any later version.
18 CamCOPS is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 GNU General Public License for more details.
23 You should have received a copy of the GNU General Public License
24 along with CamCOPS. If not, see <https://www.gnu.org/licenses/>.
26===============================================================================
28**Read and represent a CamCOPS config file.**
30Also contains various types of demonstration config file (CamCOPS, but also
31``supervisord``, Apache, etc.) and demonstration helper scripts (e.g. MySQL).
33There are CONDITIONAL AND IN-FUNCTION IMPORTS HERE; see below. This is to
34minimize the number of modules loaded when this is used in the context of the
35client-side database script, rather than the webview.
37Moreover, it should not use SQLAlchemy objects directly; see ``celery.py``.
39In particular, I tried hard to use a "database-unaware" (unbound) SQLAlchemy
40ExportRecipient object. However, when the backend re-calls the config to get
41its recipients, we get errors like:
43.. code-block:: none
45 [2018-12-25 00:56:00,118: ERROR/ForkPoolWorker-7] Task camcops_server.cc_modules.celery_tasks.export_to_recipient_backend[ab2e2691-c2fa-4821-b8cd-2cbeb86ddc8f] raised unexpected: DetachedInstanceError('Instance <ExportRecipient at 0x7febbeeea7b8> is not bound to a Session; attribute refresh operation cannot proceed',)
46 Traceback (most recent call last):
47 File "/home/rudolf/dev/venvs/camcops/lib/python3.6/site-packages/celery/app/trace.py", line 382, in trace_task
48 R = retval = fun(*args, **kwargs)
49 File "/home/rudolf/dev/venvs/camcops/lib/python3.6/site-packages/celery/app/trace.py", line 641, in __protected_call__
50 return self.run(*args, **kwargs)
51 File "/home/rudolf/Documents/code/camcops/server/camcops_server/cc_modules/celery_tasks.py", line 103, in export_to_recipient_backend
52 schedule_via_backend=False)
53 File "/home/rudolf/Documents/code/camcops/server/camcops_server/cc_modules/cc_export.py", line 255, in export
54 req, recipient_names=recipient_names, all_recipients=all_recipients)
55 File "/home/rudolf/Documents/code/camcops/server/camcops_server/cc_modules/cc_config.py", line 1460, in get_export_recipients
56 valid_names = set(r.recipient_name for r in recipients)
57 File "/home/rudolf/Documents/code/camcops/server/camcops_server/cc_modules/cc_config.py", line 1460, in <genexpr>
58 valid_names = set(r.recipient_name for r in recipients)
59 File "/home/rudolf/dev/venvs/camcops/lib/python3.6/site-packages/sqlalchemy/orm/attributes.py", line 242, in __get__
60 return self.impl.get(instance_state(instance), dict_)
61 File "/home/rudolf/dev/venvs/camcops/lib/python3.6/site-packages/sqlalchemy/orm/attributes.py", line 594, in get
62 value = state._load_expired(state, passive)
63 File "/home/rudolf/dev/venvs/camcops/lib/python3.6/site-packages/sqlalchemy/orm/state.py", line 608, in _load_expired
64 self.manager.deferred_scalar_loader(self, toload)
65 File "/home/rudolf/dev/venvs/camcops/lib/python3.6/site-packages/sqlalchemy/orm/loading.py", line 813, in load_scalar_attributes
66 (state_str(state)))
67 sqlalchemy.orm.exc.DetachedInstanceError: Instance <ExportRecipient at 0x7febbeeea7b8> is not bound to a Session; attribute refresh operation cannot proceed (Background on this error at: http://sqlalche.me/e/bhk3)
69""" # noqa
71import codecs
72import collections
73import configparser
74import contextlib
75import datetime
76import os
77import logging
78import re
79from typing import Any, Dict, Generator, List, Optional, Union
81from cardinal_pythonlib.configfiles import (
82 get_config_parameter,
83 get_config_parameter_boolean,
84 get_config_parameter_loglevel,
85 get_config_parameter_multiline
86)
87from cardinal_pythonlib.docker import running_under_docker
88from cardinal_pythonlib.fileops import relative_filename_within_dir
89from cardinal_pythonlib.logs import BraceStyleAdapter
90from cardinal_pythonlib.randomness import create_base64encoded_randomness
91from cardinal_pythonlib.reprfunc import auto_repr
92from cardinal_pythonlib.sqlalchemy.alembic_func import (
93 get_current_and_head_revision,
94)
95from cardinal_pythonlib.sqlalchemy.engine_func import (
96 is_sqlserver,
97 is_sqlserver_2008_or_later,
98)
99from cardinal_pythonlib.sqlalchemy.logs import pre_disable_sqlalchemy_extra_echo_log # noqa
100from cardinal_pythonlib.sqlalchemy.schema import get_table_names
101from cardinal_pythonlib.sqlalchemy.session import get_safe_url_from_engine
102from cardinal_pythonlib.wsgi.reverse_proxied_mw import ReverseProxiedMiddleware
103import celery.schedules
104from sqlalchemy.engine import create_engine
105from sqlalchemy.engine.base import Engine
106from sqlalchemy.orm import sessionmaker
107from sqlalchemy.orm import Session as SqlASession
109from camcops_server.cc_modules.cc_baseconstants import (
110 ALEMBIC_BASE_DIR,
111 ALEMBIC_CONFIG_FILENAME,
112 ALEMBIC_VERSION_TABLE,
113 ENVVAR_CONFIG_FILE,
114 LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR,
115 ON_READTHEDOCS,
116)
117from camcops_server.cc_modules.cc_cache import cache_region_static, fkg
118from camcops_server.cc_modules.cc_constants import (
119 CONFIG_FILE_EXPORT_SECTION,
120 CONFIG_FILE_SERVER_SECTION,
121 CONFIG_FILE_SITE_SECTION,
122 ConfigDefaults,
123 ConfigParamExportGeneral,
124 ConfigParamExportRecipient,
125 ConfigParamServer,
126 ConfigParamSite,
127 DockerConstants,
128)
129from camcops_server.cc_modules.cc_exportrecipientinfo import (
130 ExportRecipientInfo,
131)
132from camcops_server.cc_modules.cc_exception import raise_runtime_error
133from camcops_server.cc_modules.cc_filename import (
134 PatientSpecElementForFilename,
135)
136from camcops_server.cc_modules.cc_language import POSSIBLE_LOCALES
137from camcops_server.cc_modules.cc_pyramid import MASTER_ROUTE_CLIENT_API
138from camcops_server.cc_modules.cc_snomed import (
139 get_all_task_snomed_concepts,
140 get_icd9_snomed_concepts_from_xml,
141 get_icd10_snomed_concepts_from_xml,
142 SnomedConcept,
143)
144from camcops_server.cc_modules.cc_validators import (
145 validate_export_recipient_name,
146 validate_group_name,
147)
148from camcops_server.cc_modules.cc_version_string import (
149 CAMCOPS_SERVER_VERSION_STRING,
150)
152log = BraceStyleAdapter(logging.getLogger(__name__))
154pre_disable_sqlalchemy_extra_echo_log()
156# =============================================================================
157# Constants
158# =============================================================================
160VALID_RECIPIENT_NAME_REGEX = r"^[\w_-]+$"
161# ... because we'll use them for filenames, amongst other things
162# https://stackoverflow.com/questions/10944438/
163# https://regexr.com/
165# Windows paths: irrelevant, as Windows doesn't run supervisord
166DEFAULT_LINUX_CAMCOPS_CONFIG = "/etc/camcops/camcops.conf"
167DEFAULT_LINUX_CAMCOPS_BASE_DIR = "/usr/share/camcops"
168DEFAULT_LINUX_CAMCOPS_VENV_DIR = os.path.join(
169 DEFAULT_LINUX_CAMCOPS_BASE_DIR, "venv")
170DEFAULT_LINUX_CAMCOPS_VENV_BIN_DIR = os.path.join(
171 DEFAULT_LINUX_CAMCOPS_VENV_DIR, "bin")
172DEFAULT_LINUX_CAMCOPS_EXECUTABLE = os.path.join(
173 DEFAULT_LINUX_CAMCOPS_VENV_BIN_DIR, "camcops_server")
174DEFAULT_LINUX_CAMCOPS_STATIC_DIR = os.path.join(
175 DEFAULT_LINUX_CAMCOPS_VENV_DIR,
176 "lib", "python3.6", "site-packages", "camcops_server", "static")
177DEFAULT_LINUX_LOGDIR = "/var/log/supervisor"
178DEFAULT_LINUX_USER = "www-data" # Ubuntu default
181# =============================================================================
182# Helper functions
183# =============================================================================
185def warn_if_not_within_docker_dir(param_name: str,
186 filespec: str,
187 permit_cfg: bool = False,
188 permit_venv: bool = False,
189 permit_tmp: bool = False,
190 param_contains_not_is: bool = False) -> None:
191 """
192 If the specified filename isn't within a relevant directory that will be
193 used by CamCOPS when operating within a Docker Compose application, warn
194 the user.
196 Args:
197 param_name:
198 Name of the parameter in the CamCOPS config file.
199 filespec:
200 Filename (or filename-like thing) to check.
201 permit_cfg:
202 Permit the file to be in the configuration directory.
203 permit_venv:
204 Permit the file to be in the virtual environment directory.
205 permit_tmp:
206 Permit the file to be in the shared temporary space.
207 param_contains_not_is:
208 The parameter "contains", not "is", the filename.
209 """
210 if not filespec:
211 return
212 is_phrase = "contains" if param_contains_not_is else "is"
213 permitted_dirs = [] # type: List[str]
214 if permit_cfg:
215 permitted_dirs.append(DockerConstants.CONFIG_DIR)
216 if permit_venv:
217 permitted_dirs.append(DockerConstants.VENV_DIR)
218 if permit_tmp:
219 permitted_dirs.append(DockerConstants.TMP_DIR)
220 ok = any(
221 relative_filename_within_dir(filespec, d)
222 for d in permitted_dirs
223 )
224 if not ok:
225 log.warning(
226 f"Config parameter {param_name} {is_phrase} {filespec!r}, "
227 f"which is not within the permitted Docker directories "
228 f"{permitted_dirs!r}"
229 )
232def warn_if_not_docker_value(param_name: str,
233 actual_value: Any,
234 required_value: Any) -> None:
235 """
236 Warn the user if a parameter does not match the specific value required
237 when operating under Docker.
239 Args:
240 param_name:
241 Name of the parameter in the CamCOPS config file.
242 actual_value:
243 Value in the config file.
244 required_value:
245 Value that should be used.
246 """
247 if actual_value != required_value:
248 log.warning(
249 f"Config parameter {param_name} is {actual_value!r}, "
250 f"but should be {required_value!r} when running inside Docker"
251 )
254def warn_if_not_present(param_name: str, value: Any) -> None:
255 """
256 Warn the user if a parameter is not set (None, or an empty string), for
257 when operating under Docker.
259 Args:
260 param_name:
261 Name of the parameter in the CamCOPS config file.
262 value:
263 Value in the config file.
264 """
265 if value is None or value == "":
266 log.warning(
267 f"Config parameter {param_name} is not specified, "
268 f"but should be specified when running inside Docker"
269 )
272# =============================================================================
273# Demo config
274# =============================================================================
276# Cosmetic demonstration constants:
277DEFAULT_DB_READONLY_USER = 'QQQ_USERNAME_REPLACE_ME'
278DEFAULT_DB_READONLY_PASSWORD = 'PPP_PASSWORD_REPLACE_ME'
279DUMMY_INSTITUTION_URL = 'https://www.mydomain/'
282def get_demo_config(for_docker: bool = False) -> str:
283 """
284 Returns a demonstration config file based on the specified parameters.
286 Args:
287 for_docker:
288 Adjust defaults for the Docker environment.
289 """
290 # ...
291 # http://www.debian.org/doc/debian-policy/ch-opersys.html#s-writing-init
292 # https://people.canonical.com/~cjwatson/ubuntu-policy/policy.html/ch-opersys.html # noqa
293 session_cookie_secret = create_base64encoded_randomness(num_bytes=64)
295 cd = ConfigDefaults(docker=for_docker)
296 return f"""
297# Demonstration CamCOPS server configuration file.
298#
299# Created by CamCOPS server version {CAMCOPS_SERVER_VERSION_STRING}.
300# See help at https://camcops.readthedocs.io/.
301#
302# Using defaults for Docker environment: {for_docker}
304# =============================================================================
305# CamCOPS site
306# =============================================================================
308[{CONFIG_FILE_SITE_SECTION}]
310# -----------------------------------------------------------------------------
311# Database connection
312# -----------------------------------------------------------------------------
314{ConfigParamSite.DB_URL} = {cd.demo_db_url}
315{ConfigParamSite.DB_ECHO} = {cd.DB_ECHO}
317# -----------------------------------------------------------------------------
318# URLs and paths
319# -----------------------------------------------------------------------------
321{ConfigParamSite.LOCAL_INSTITUTION_URL} = {DUMMY_INSTITUTION_URL}
322{ConfigParamSite.LOCAL_LOGO_FILE_ABSOLUTE} = {cd.LOCAL_LOGO_FILE_ABSOLUTE}
323{ConfigParamSite.CAMCOPS_LOGO_FILE_ABSOLUTE} = {cd.CAMCOPS_LOGO_FILE_ABSOLUTE}
325{ConfigParamSite.EXTRA_STRING_FILES} = {cd.EXTRA_STRING_FILES}
326{ConfigParamSite.RESTRICTED_TASKS} =
327{ConfigParamSite.LANGUAGE} = {cd.LANGUAGE}
329{ConfigParamSite.SNOMED_TASK_XML_FILENAME} =
330{ConfigParamSite.SNOMED_ICD9_XML_FILENAME} =
331{ConfigParamSite.SNOMED_ICD10_XML_FILENAME} =
333{ConfigParamSite.WKHTMLTOPDF_FILENAME} =
335# -----------------------------------------------------------------------------
336# Login and session configuration
337# -----------------------------------------------------------------------------
339{ConfigParamSite.SESSION_COOKIE_SECRET} = camcops_autogenerated_secret_{session_cookie_secret}
340{ConfigParamSite.SESSION_TIMEOUT_MINUTES} = {cd.SESSION_TIMEOUT_MINUTES}
341{ConfigParamSite.PASSWORD_CHANGE_FREQUENCY_DAYS} = {cd.PASSWORD_CHANGE_FREQUENCY_DAYS}
342{ConfigParamSite.LOCKOUT_THRESHOLD} = {cd.LOCKOUT_THRESHOLD}
343{ConfigParamSite.LOCKOUT_DURATION_INCREMENT_MINUTES} = {cd.LOCKOUT_DURATION_INCREMENT_MINUTES}
344{ConfigParamSite.DISABLE_PASSWORD_AUTOCOMPLETE} = {cd.DISABLE_PASSWORD_AUTOCOMPLETE}
346# -----------------------------------------------------------------------------
347# Suggested filenames for saving PDFs from the web view
348# -----------------------------------------------------------------------------
350{ConfigParamSite.PATIENT_SPEC_IF_ANONYMOUS} = {cd.PATIENT_SPEC_IF_ANONYMOUS}
351{ConfigParamSite.PATIENT_SPEC} = {{{PatientSpecElementForFilename.SURNAME}}}_{{{PatientSpecElementForFilename.FORENAME}}}_{{{PatientSpecElementForFilename.ALLIDNUMS}}}
353{ConfigParamSite.TASK_FILENAME_SPEC} = CamCOPS_{{patient}}_{{created}}_{{tasktype}}-{{serverpk}}.{{filetype}}
354{ConfigParamSite.TRACKER_FILENAME_SPEC} = CamCOPS_{{patient}}_{{now}}_tracker.{{filetype}}
355{ConfigParamSite.CTV_FILENAME_SPEC} = CamCOPS_{{patient}}_{{now}}_clinicaltextview.{{filetype}}
357# -----------------------------------------------------------------------------
358# E-mail options
359# -----------------------------------------------------------------------------
361{ConfigParamSite.EMAIL_HOST} = mysmtpserver.mydomain
362{ConfigParamSite.EMAIL_PORT} = {cd.EMAIL_PORT}
363{ConfigParamSite.EMAIL_USE_TLS} = {cd.EMAIL_USE_TLS}
364{ConfigParamSite.EMAIL_HOST_USERNAME} = myusername
365{ConfigParamSite.EMAIL_HOST_PASSWORD} = mypassword
366{ConfigParamSite.EMAIL_FROM} = CamCOPS computer <noreply@myinstitution.mydomain>
367{ConfigParamSite.EMAIL_SENDER} =
368{ConfigParamSite.EMAIL_REPLY_TO} = CamCOPS clinical administrator <admin@myinstitution.mydomain>
370# -----------------------------------------------------------------------------
371# User download options
372# -----------------------------------------------------------------------------
374{ConfigParamSite.PERMIT_IMMEDIATE_DOWNLOADS} = {cd.PERMIT_IMMEDIATE_DOWNLOADS}
375{ConfigParamSite.USER_DOWNLOAD_DIR} = {cd.USER_DOWNLOAD_DIR}
376{ConfigParamSite.USER_DOWNLOAD_FILE_LIFETIME_MIN} = {cd.USER_DOWNLOAD_FILE_LIFETIME_MIN}
377{ConfigParamSite.USER_DOWNLOAD_MAX_SPACE_MB} = {cd.USER_DOWNLOAD_MAX_SPACE_MB}
379# -----------------------------------------------------------------------------
380# Debugging options
381# -----------------------------------------------------------------------------
383{ConfigParamSite.WEBVIEW_LOGLEVEL} = {cd.WEBVIEW_LOGLEVEL_TEXTFORMAT}
384{ConfigParamSite.CLIENT_API_LOGLEVEL} = {cd.CLIENT_API_LOGLEVEL_TEXTFORMAT}
385{ConfigParamSite.ALLOW_INSECURE_COOKIES} = {cd.ALLOW_INSECURE_COOKIES}
388# =============================================================================
389# Web server options
390# =============================================================================
392[{CONFIG_FILE_SERVER_SECTION}]
394# -----------------------------------------------------------------------------
395# Common web server options
396# -----------------------------------------------------------------------------
398{ConfigParamServer.HOST} = {cd.HOST}
399{ConfigParamServer.PORT} = {cd.PORT}
400{ConfigParamServer.UNIX_DOMAIN_SOCKET} =
402# If you host CamCOPS behind Apache, it’s likely that you’ll want Apache to
403# handle HTTPS and CamCOPS to operate unencrypted behind a reverse proxy, in
404# which case don’t set SSL_CERTIFICATE or SSL_PRIVATE_KEY.
405{ConfigParamServer.SSL_CERTIFICATE} =
406{ConfigParamServer.SSL_PRIVATE_KEY} =
407{ConfigParamServer.STATIC_CACHE_DURATION_S} = {cd.STATIC_CACHE_DURATION_S}
409# -----------------------------------------------------------------------------
410# WSGI options
411# -----------------------------------------------------------------------------
413{ConfigParamServer.DEBUG_REVERSE_PROXY} = {cd.DEBUG_REVERSE_PROXY}
414{ConfigParamServer.DEBUG_TOOLBAR} = {cd.DEBUG_TOOLBAR}
415{ConfigParamServer.SHOW_REQUESTS} = {cd.SHOW_REQUESTS}
416{ConfigParamServer.SHOW_REQUEST_IMMEDIATELY} = {cd.SHOW_REQUEST_IMMEDIATELY}
417{ConfigParamServer.SHOW_RESPONSE} = {cd.SHOW_RESPONSE}
418{ConfigParamServer.SHOW_TIMING} = {cd.SHOW_TIMING}
419{ConfigParamServer.PROXY_HTTP_HOST} =
420{ConfigParamServer.PROXY_REMOTE_ADDR} =
421{ConfigParamServer.PROXY_REWRITE_PATH_INFO} = {cd.PROXY_REWRITE_PATH_INFO}
422{ConfigParamServer.PROXY_SCRIPT_NAME} =
423{ConfigParamServer.PROXY_SERVER_NAME} =
424{ConfigParamServer.PROXY_SERVER_PORT} =
425{ConfigParamServer.PROXY_URL_SCHEME} =
426{ConfigParamServer.TRUSTED_PROXY_HEADERS} =
427 HTTP_X_FORWARDED_HOST
428 HTTP_X_FORWARDED_SERVER
429 HTTP_X_FORWARDED_PORT
430 HTTP_X_FORWARDED_PROTO
431 HTTP_X_FORWARDED_FOR
432 HTTP_X_SCRIPT_NAME
434# -----------------------------------------------------------------------------
435# CherryPy options
436# -----------------------------------------------------------------------------
438{ConfigParamServer.CHERRYPY_SERVER_NAME} = {cd.CHERRYPY_SERVER_NAME}
439{ConfigParamServer.CHERRYPY_THREADS_START} = {cd.CHERRYPY_THREADS_START}
440{ConfigParamServer.CHERRYPY_THREADS_MAX} = {cd.CHERRYPY_THREADS_MAX}
441{ConfigParamServer.CHERRYPY_LOG_SCREEN} = {cd.CHERRYPY_LOG_SCREEN}
442{ConfigParamServer.CHERRYPY_ROOT_PATH} = {cd.CHERRYPY_ROOT_PATH}
444# -----------------------------------------------------------------------------
445# Gunicorn options
446# -----------------------------------------------------------------------------
448{ConfigParamServer.GUNICORN_NUM_WORKERS} = {cd.GUNICORN_NUM_WORKERS}
449{ConfigParamServer.GUNICORN_DEBUG_RELOAD} = {cd.GUNICORN_DEBUG_RELOAD}
450{ConfigParamServer.GUNICORN_TIMEOUT_S} = {cd.GUNICORN_TIMEOUT_S}
451{ConfigParamServer.DEBUG_SHOW_GUNICORN_OPTIONS} = {cd.DEBUG_SHOW_GUNICORN_OPTIONS}
454# =============================================================================
455# Export options
456# =============================================================================
458[{CONFIG_FILE_EXPORT_SECTION}]
460{ConfigParamExportGeneral.CELERY_BEAT_EXTRA_ARGS} =
461{ConfigParamExportGeneral.CELERY_BEAT_SCHEDULE_DATABASE} = {cd.CELERY_BEAT_SCHEDULE_DATABASE}
462{ConfigParamExportGeneral.CELERY_BROKER_URL} = {cd.CELERY_BROKER_URL}
463{ConfigParamExportGeneral.CELERY_WORKER_EXTRA_ARGS} =
464 --maxtasksperchild=1000
465{ConfigParamExportGeneral.CELERY_EXPORT_TASK_RATE_LIMIT} = 100/m
466{ConfigParamExportGeneral.EXPORT_LOCKDIR} = {cd.EXPORT_LOCKDIR}
468{ConfigParamExportGeneral.RECIPIENTS} =
470{ConfigParamExportGeneral.SCHEDULE_TIMEZONE} = {cd.SCHEDULE_TIMEZONE}
471{ConfigParamExportGeneral.SCHEDULE} =
474# =============================================================================
475# Details for each export recipient
476# =============================================================================
478# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
479# Example recipient
480# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
481 # Example (disabled because it's not in the {ConfigParamExportGeneral.RECIPIENTS} list above)
483[recipient:recipient_A]
485 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
486 # How to export
487 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
489{ConfigParamExportRecipient.TRANSMISSION_METHOD} = hl7
490{ConfigParamExportRecipient.PUSH} = true
491{ConfigParamExportRecipient.TASK_FORMAT} = pdf
492{ConfigParamExportRecipient.XML_FIELD_COMMENTS} = {cd.XML_FIELD_COMMENTS}
494 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
495 # What to export
496 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
498{ConfigParamExportRecipient.ALL_GROUPS} = false
499{ConfigParamExportRecipient.GROUPS} =
500 myfirstgroup
501 mysecondgroup
502{ConfigParamExportRecipient.TASKS} =
504{ConfigParamExportRecipient.START_DATETIME_UTC} =
505{ConfigParamExportRecipient.END_DATETIME_UTC} =
506{ConfigParamExportRecipient.FINALIZED_ONLY} = {cd.FINALIZED_ONLY}
507{ConfigParamExportRecipient.INCLUDE_ANONYMOUS} = {cd.INCLUDE_ANONYMOUS}
508{ConfigParamExportRecipient.PRIMARY_IDNUM} = 1
509{ConfigParamExportRecipient.REQUIRE_PRIMARY_IDNUM_MANDATORY_IN_POLICY} = {cd.REQUIRE_PRIMARY_IDNUM_MANDATORY_IN_POLICY}
511 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
512 # Options applicable to database exports
513 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
515{ConfigParamExportRecipient.DB_URL} = some_sqlalchemy_url
516{ConfigParamExportRecipient.DB_ECHO} = {cd.DB_ECHO}
517{ConfigParamExportRecipient.DB_INCLUDE_BLOBS} = {cd.DB_INCLUDE_BLOBS}
518{ConfigParamExportRecipient.DB_ADD_SUMMARIES} = {cd.DB_ADD_SUMMARIES}
519{ConfigParamExportRecipient.DB_PATIENT_ID_PER_ROW} = {cd.DB_PATIENT_ID_PER_ROW}
521 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
522 # Options applicable to e-mail exports
523 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
525{ConfigParamExportRecipient.EMAIL_TO} =
526 Perinatal Psychiatry Admin <perinatal@myinstitution.mydomain>
528{ConfigParamExportRecipient.EMAIL_CC} =
529 Dr Alice Bradford <alice.bradford@myinstitution.mydomain>
530 Dr Charles Dogfoot <charles.dogfoot@myinstitution.mydomain>
532{ConfigParamExportRecipient.EMAIL_BCC} =
533 superuser <root@myinstitution.mydomain>
535{ConfigParamExportRecipient.EMAIL_PATIENT_SPEC_IF_ANONYMOUS} = anonymous
536{ConfigParamExportRecipient.EMAIL_PATIENT_SPEC} = {{{PatientSpecElementForFilename.SURNAME}}}, {{{PatientSpecElementForFilename.FORENAME}}}, {{{PatientSpecElementForFilename.ALLIDNUMS}}}
537{ConfigParamExportRecipient.EMAIL_SUBJECT} = CamCOPS task for {{patient}}, created {{created}}: {{tasktype}}, PK {{serverpk}}
538{ConfigParamExportRecipient.EMAIL_BODY_IS_HTML} = false
539{ConfigParamExportRecipient.EMAIL_BODY} =
540 Please find attached a new CamCOPS task for manual filing to the electronic
541 patient record of
543 {{patient}}
545 Task type: {{tasktype}}
546 Created: {{created}}
547 CamCOPS server primary key: {{serverpk}}
549 Yours faithfully,
551 The CamCOPS computer.
553{ConfigParamExportRecipient.EMAIL_KEEP_MESSAGE} = {cd.HL7_KEEP_MESSAGE}
555 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
556 # Options applicable to HL7
557 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
559{ConfigParamExportRecipient.HL7_HOST} = myhl7server.mydomain
560{ConfigParamExportRecipient.HL7_PORT} = {cd.HL7_PORT}
561{ConfigParamExportRecipient.HL7_PING_FIRST} = {cd.HL7_PING_FIRST}
562{ConfigParamExportRecipient.HL7_NETWORK_TIMEOUT_MS} = {cd.HL7_NETWORK_TIMEOUT_MS}
563{ConfigParamExportRecipient.HL7_KEEP_MESSAGE} = {cd.HL7_KEEP_MESSAGE}
564{ConfigParamExportRecipient.HL7_KEEP_REPLY} = {cd.HL7_KEEP_REPLY}
565{ConfigParamExportRecipient.HL7_DEBUG_DIVERT_TO_FILE} = {cd.HL7_DEBUG_DIVERT_TO_FILE}
566{ConfigParamExportRecipient.HL7_DEBUG_TREAT_DIVERTED_AS_SENT} = {cd.HL7_DEBUG_TREAT_DIVERTED_AS_SENT}
568 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
569 # Options applicable to file transfers/attachments
570 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
572{ConfigParamExportRecipient.FILE_PATIENT_SPEC} = {{surname}}_{{forename}}_{{idshortdesc1}}{{idnum1}}
573{ConfigParamExportRecipient.FILE_PATIENT_SPEC_IF_ANONYMOUS} = {cd.FILE_PATIENT_SPEC_IF_ANONYMOUS}
574{ConfigParamExportRecipient.FILE_FILENAME_SPEC} = /my_nfs_mount/mypath/CamCOPS_{{patient}}_{{created}}_{{tasktype}}-{{serverpk}}.{{filetype}}
575{ConfigParamExportRecipient.FILE_MAKE_DIRECTORY} = {cd.FILE_MAKE_DIRECTORY}
576{ConfigParamExportRecipient.FILE_OVERWRITE_FILES} = {cd.FILE_OVERWRITE_FILES}
577{ConfigParamExportRecipient.FILE_EXPORT_RIO_METADATA} = {cd.FILE_EXPORT_RIO_METADATA}
578{ConfigParamExportRecipient.FILE_SCRIPT_AFTER_EXPORT} =
580 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
581 # Extra options for RiO metadata for file-based export
582 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
584{ConfigParamExportRecipient.RIO_IDNUM} = 2
585{ConfigParamExportRecipient.RIO_UPLOADING_USER} = CamCOPS
586{ConfigParamExportRecipient.RIO_DOCUMENT_TYPE} = CC
588 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
589 # Extra options for REDCap export
590 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
592{ConfigParamExportRecipient.REDCAP_API_URL} = https://domain.of.redcap.server/api/
593{ConfigParamExportRecipient.REDCAP_API_KEY} = myapikey
594{ConfigParamExportRecipient.REDCAP_FIELDMAP_FILENAME} = /location/of/fieldmap.xml
596 """.strip() # noqa
599# =============================================================================
600# Demo configuration files, other than the CamCOPS config file itself
601# =============================================================================
603DEFAULT_SOCKET_FILENAME = "/run/camcops/camcops.socket"
606def get_demo_supervisor_config() -> str:
607 """
608 Returns a demonstration ``supervisord`` config file based on the
609 specified parameters.
610 """
611 redirect_stderr = "true"
612 autostart = "true"
613 autorestart = "true"
614 startsecs = "30"
615 stopwaitsecs = "60"
616 return f"""
617# =============================================================================
618# Demonstration 'supervisor' (supervisord) config file for CamCOPS.
619# Created by CamCOPS version {CAMCOPS_SERVER_VERSION_STRING}.
620# =============================================================================
621# See https://camcops.readthedocs.io/en/latest/administrator/server_configuration.html#start-camcops
623[program:camcops_server]
625command = {DEFAULT_LINUX_CAMCOPS_EXECUTABLE} serve_gunicorn
626 --config {DEFAULT_LINUX_CAMCOPS_CONFIG}
628directory = {DEFAULT_LINUX_CAMCOPS_BASE_DIR}
629environment = MPLCONFIGDIR="{LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR}"
630user = {DEFAULT_LINUX_USER}
631stdout_logfile = {DEFAULT_LINUX_LOGDIR}/camcops_server.log
632redirect_stderr = {redirect_stderr}
633autostart = {autostart}
634autorestart = {autorestart}
635startsecs = {startsecs}
636stopwaitsecs = {stopwaitsecs}
638[program:camcops_workers]
640command = {DEFAULT_LINUX_CAMCOPS_EXECUTABLE} launch_workers
641 --config {DEFAULT_LINUX_CAMCOPS_CONFIG}
643directory = {DEFAULT_LINUX_CAMCOPS_BASE_DIR}
644environment = MPLCONFIGDIR="{LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR}"
645user = {DEFAULT_LINUX_USER}
646stdout_logfile = {DEFAULT_LINUX_LOGDIR}/camcops_workers.log
647redirect_stderr = {redirect_stderr}
648autostart = {autostart}
649autorestart = {autorestart}
650startsecs = {startsecs}
651stopwaitsecs = {stopwaitsecs}
653[program:camcops_scheduler]
655command = {DEFAULT_LINUX_CAMCOPS_EXECUTABLE} launch_scheduler
656 --config {DEFAULT_LINUX_CAMCOPS_CONFIG}
658directory = {DEFAULT_LINUX_CAMCOPS_BASE_DIR}
659environment = MPLCONFIGDIR="{LINUX_DEFAULT_MATPLOTLIB_CACHE_DIR}"
660user = {DEFAULT_LINUX_USER}
661stdout_logfile = {DEFAULT_LINUX_LOGDIR}/camcops_scheduler.log
662redirect_stderr = {redirect_stderr}
663autostart = {autostart}
664autorestart = {autorestart}
665startsecs = {startsecs}
666stopwaitsecs = {stopwaitsecs}
668[group:camcops]
670programs = camcops_server, camcops_workers, camcops_scheduler
672 """.strip() # noqa
675def get_demo_apache_config(
676 rootpath: str = "camcops", # no slash
677 specimen_internal_port: int = None,
678 specimen_socket_file: str = DEFAULT_SOCKET_FILENAME) -> str:
679 """
680 Returns a demo Apache HTTPD config file section applicable to CamCOPS.
681 """
682 cd = ConfigDefaults()
683 specimen_internal_port = specimen_internal_port or cd.PORT
684 urlbase = "/" + rootpath
685 # noinspection HttpUrlsUsage
686 return f"""
687# Demonstration Apache config file section for CamCOPS.
688# Created by CamCOPS version {CAMCOPS_SERVER_VERSION_STRING}.
689#
690# Under Ubuntu, the Apache config will be somewhere in /etc/apache2/
691# Under CentOS, the Apache config will be somewhere in /etc/httpd/
692#
693# This section should go within the <VirtualHost> directive for the secure
694# (SSL, HTTPS) part of the web site.
696<VirtualHost *:443>
697 # ...
699 # =========================================================================
700 # CamCOPS
701 # =========================================================================
702 # Apache operates on the principle that the first match wins. So, if we
703 # want to serve CamCOPS but then override some of its URLs to serve static
704 # files faster, we define the static stuff first.
706 # ---------------------------------------------------------------------
707 # 1. Serve static files
708 # ---------------------------------------------------------------------
709 # a) offer them at the appropriate URL
710 # b) provide permission
711 # c) disable ProxyPass for static files
713 # CHANGE THIS: aim the alias at your own institutional logo.
715 Alias {urlbase}/static/logo_local.png {DEFAULT_LINUX_CAMCOPS_STATIC_DIR}/logo_local.png
717 # We move from more specific to less specific aliases; the first match
718 # takes precedence. (Apache will warn about conflicting aliases if
719 # specified in a wrong, less-to-more-specific, order.)
721 Alias {urlbase}/static/ {DEFAULT_LINUX_CAMCOPS_STATIC_DIR}/
723 <Directory {DEFAULT_LINUX_CAMCOPS_STATIC_DIR}>
724 Require all granted
726 # ... for old Apache versions (e.g. 2.2), use instead:
727 # Order allow,deny
728 # Allow from all
729 </Directory>
731 # Don't ProxyPass the static files; we'll serve them via Apache.
733 ProxyPassMatch ^{urlbase}/static/ !
735 # ---------------------------------------------------------------------
736 # 2. Proxy requests to the CamCOPS web server and back; allow access
737 # ---------------------------------------------------------------------
738 # ... either via an internal TCP/IP port (e.g. 1024 or higher, and NOT
739 # accessible to users);
740 # ... or, better, via a Unix socket, e.g. {specimen_socket_file}
741 #
742 # NOTES
743 #
744 # - When you ProxyPass {urlbase}, you should browse to
745 #
746 # https://YOURSITE{urlbase}
747 #
748 # and point your tablet devices to
749 #
750 # https://YOURSITE{urlbase}{MASTER_ROUTE_CLIENT_API}
751 #
752 # - Don't specify trailing slashes for the ProxyPass and
753 # ProxyPassReverse directives.
754 # If you do, http://host/camcops will fail though
755 # http://host/camcops/ will succeed.
756 #
757 # - An alternative fix is to enable mod_rewrite (e.g. sudo a2enmod
758 # rewrite), then add these commands:
759 #
760 # RewriteEngine on
761 # RewriteRule ^/{rootpath}$ {rootpath}/ [L,R=301]
762 #
763 # which will redirect requests without the trailing slash to a
764 # version with the trailing slash.
765 #
766 # - Ensure that you put the CORRECT PROTOCOL (http, https) in the rules
767 # below.
768 #
769 # - For ProxyPass options, see https://httpd.apache.org/docs/2.2/mod/mod_proxy.html#proxypass
770 #
771 # - Include "retry=0" to stop Apache disabling the connection for
772 # while on failure.
773 # - Consider adding a "timeout=<seconds>" option if the back-end is
774 # slow and causing timeouts.
775 #
776 # - CamCOPS MUST BE TOLD about its location and protocol, because that
777 # information is critical for synthesizing URLs, but is stripped out
778 # by the reverse proxy system. There are two ways:
779 #
780 # (i) specifying headers or WSGI environment variables, such as
781 # the HTTP(S) headers X-Forwarded-Proto and X-Script-Name below
782 # (and telling CamCOPS to trust them via its
783 # TRUSTED_PROXY_HEADERS setting);
784 #
785 # (ii) specifying other options to "camcops_server", including
786 # PROXY_SCRIPT_NAME, PROXY_URL_SCHEME; see the help for the
787 # CamCOPS config.
788 #
789 # So:
790 #
791 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
792 # (a) Reverse proxy
793 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
794 #
795 # #####################################################################
796 # PORT METHOD
797 # #####################################################################
798 # Note the use of "http" (reflecting the backend), not https (like the
799 # front end).
801 # ProxyPass {urlbase} http://127.0.0.1:{specimen_internal_port} retry=0 timeout=300
802 # ProxyPassReverse {urlbase} http://127.0.0.1:{specimen_internal_port}
804 # #####################################################################
805 # UNIX SOCKET METHOD (Apache 2.4.9 and higher)
806 # #####################################################################
807 # This requires Apache 2.4.9, and passes after the '|' character a URL
808 # that determines the Host: value of the request; see
809 # ://httpd.apache.org/docs/trunk/mod/mod_proxy.html#proxypass
810 #
811 # The general syntax is:
812 #
813 # ProxyPass /URL_USER_SEES unix:SOCKETFILE|PROTOCOL://HOST/EXTRA_URL_FOR_BACKEND retry=0
814 #
815 # Note that:
816 #
817 # - the protocol should be http, not https (Apache deals with the
818 # HTTPS part and passes HTTP on)
819 # - the EXTRA_URL_FOR_BACKEND needs to be (a) unique for each
820 # instance or Apache will use a single worker for multiple
821 # instances, and (b) blank for the backend's benefit. Since those
822 # two conflict when there's >1 instance, there's a problem.
823 # - Normally, HOST is given as localhost. It may be that this problem
824 # is solved by using a dummy unique value for HOST:
825 # https://bz.apache.org/bugzilla/show_bug.cgi?id=54101#c1
826 #
827 # If your Apache version is too old, you will get the error
828 #
829 # "AH00526: Syntax error on line 56 of /etc/apache2/sites-enabled/SOMETHING:
830 # ProxyPass URL must be absolute!"
831 #
832 # If you get this error:
833 #
834 # AH01146: Ignoring parameter 'retry=0' for worker 'unix:/tmp/.camcops_gunicorn.sock|https://localhost' because of worker sharing
835 # https://wiki.apache.org/httpd/ListOfErrors
836 #
837 # ... then your URLs are overlapping and should be redone or sorted;
838 # see http://httpd.apache.org/docs/2.4/mod/mod_proxy.html#workers
839 #
840 # The part that must be unique for each instance, with no part a
841 # leading substring of any other, is THIS_BIT in:
842 #
843 # ProxyPass /URL_USER_SEES unix:SOCKETFILE|http://localhost/THIS_BIT retry=0
844 #
845 # If you get an error like this:
846 #
847 # AH01144: No protocol handler was valid for the URL /SOMEWHERE. If you are using a DSO version of mod_proxy, make sure the proxy submodules are included in the configuration using LoadModule.
848 #
849 # Then do this:
850 #
851 # sudo a2enmod proxy proxy_http
852 # sudo apache2ctl restart
853 #
854 # If you get an error like this:
855 #
856 # ... [proxy_http:error] [pid 32747] (103)Software caused connection abort: [client 109.151.49.173:56898] AH01102: error reading status line from remote server httpd-UDS:0
857 # [proxy:error] [pid 32747] [client 109.151.49.173:56898] AH00898: Error reading from remote server returned by /camcops_bruhl/webview
858 #
859 # then check you are specifying http://, not https://, in the ProxyPass
860 #
861 # Other information sources:
862 #
863 # - https://emptyhammock.com/projects/info/pyweb/webconfig.html
865 ProxyPass {urlbase} unix:{specimen_socket_file}|http://dummy1 retry=0 timeout=300
866 ProxyPassReverse {urlbase} unix:{specimen_socket_file}|http://dummy1
868 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
869 # (b) Allow proxy over SSL.
870 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
871 # Without this, you will get errors like:
872 # ... SSL Proxy requested for wombat:443 but not enabled [Hint: SSLProxyEngine]
873 # ... failed to enable ssl support for 0.0.0.0:0 (httpd-UDS)
875 SSLProxyEngine on
877 <Location /camcops>
879 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
880 # (c) Allow access
881 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
883 Require all granted
885 # ... for old Apache versions (e.g. 2.2), use instead:
886 #
887 # Order allow,deny
888 # Allow from all
890 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
891 # (d) Tell the proxied application that we are using HTTPS, and
892 # where the application is installed
893 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
894 # ... https://stackoverflow.com/questions/16042647
895 #
896 # Enable mod_headers (e.g. "sudo a2enmod headers") and set:
898 RequestHeader set X-Forwarded-Proto https
899 RequestHeader set X-Script-Name {urlbase}
901 # ... then ensure the TRUSTED_PROXY_HEADERS setting in the CamCOPS
902 # config file includes:
903 #
904 # HTTP_X_FORWARDED_HOST
905 # HTTP_X_FORWARDED_SERVER
906 # HTTP_X_FORWARDED_PORT
907 # HTTP_X_FORWARDED_PROTO
908 # HTTP_X_SCRIPT_NAME
909 #
910 # (X-Forwarded-For, X-Forwarded-Host, and X-Forwarded-Server are
911 # supplied by Apache automatically.)
913 </Location>
915 #==========================================================================
916 # SSL security (for HTTPS)
917 #==========================================================================
919 # You will also need to install your SSL certificate; see the
920 # instructions that came with it. You get a certificate by creating a
921 # certificate signing request (CSR). You enter some details about your
922 # site, and a software tool makes (1) a private key, which you keep
923 # utterly private, and (2) a CSR, which you send to a Certificate
924 # Authority (CA) for signing. They send back a signed certificate, and
925 # a chain of certificates leading from yours to a trusted root CA.
926 #
927 # You can create your own (a 'snake-oil' certificate), but your tablets
928 # and browsers will not trust it, so this is a bad idea.
929 #
930 # Once you have your certificate: edit and uncomment these lines:
932 # SSLEngine on
934 # SSLCertificateKeyFile /etc/ssl/private/my.private.key
936 # ... a private file that you made before creating the certificate
937 # request, and NEVER GAVE TO ANYBODY, and NEVER WILL (or your
938 # security is broken and you need a new certificate).
940 # SSLCertificateFile /etc/ssl/certs/my.public.cert
942 # ... signed and supplied to you by the certificate authority (CA),
943 # from the public certificate you sent to them.
945 # SSLCertificateChainFile /etc/ssl/certs/my-institution.ca-bundle
947 # ... made from additional certificates in a chain, supplied to you by
948 # the CA. For example, mine is univcam.ca-bundle, made with the
949 # command:
950 #
951 # cat TERENASSLCA.crt UTNAddTrustServer_CA.crt AddTrustExternalCARoot.crt > univcam.ca-bundle
953</VirtualHost>
955 """.strip() # noqa
958# =============================================================================
959# Helper functions
960# =============================================================================
962def raise_missing(section: str, parameter: str) -> None:
963 msg = (
964 f"Config file: missing/blank parameter {parameter} "
965 f"in section [{section}]"
966 )
967 raise_runtime_error(msg)
970# =============================================================================
971# CrontabEntry
972# =============================================================================
974class CrontabEntry(object):
975 """
976 Class to represent a ``crontab``-style entry.
977 """
978 def __init__(self,
979 line: str = None,
980 minute: Union[str, int, List[int]] = "*",
981 hour: Union[str, int, List[int]] = "*",
982 day_of_week: Union[str, int, List[int]] = "*",
983 day_of_month: Union[str, int, List[int]] = "*",
984 month_of_year: Union[str, int, List[int]] = "*",
985 content: str = None) -> None:
986 """
987 Args:
988 line:
989 line of the form ``m h dow dom moy content content content``.
990 minute:
991 crontab "minute" entry
992 hour:
993 crontab "hour" entry
994 day_of_week:
995 crontab "day_of_week" entry
996 day_of_month:
997 crontab "day_of_month" entry
998 month_of_year:
999 crontab "month_of_year" entry
1000 content:
1001 crontab "thing to run" entry
1003 If ``line`` is specified, it is used. Otherwise, the components are
1004 used; the default for each of them is ``"*"``, meaning "all". Thus, for
1005 example, you can specify ``minute="*/5"`` and that is sufficient to
1006 mean "every 5 minutes".
1007 """
1008 has_line = line is not None
1009 has_components = bool(minute and hour and day_of_week and
1010 day_of_month and month_of_year and content)
1011 assert has_line or has_components, (
1012 "Specify either a crontab line or all the time components"
1013 )
1014 if has_line:
1015 line = line.split("#")[0].strip() # everything before a '#'
1016 components = line.split() # split on whitespace
1017 assert len(components) >= 6, (
1018 "Must specify 5 time components and then contents"
1019 )
1020 minute, hour, day_of_week, day_of_month, month_of_year = (
1021 components[0:5]
1022 )
1023 content = " ".join(components[5:])
1025 self.minute = minute
1026 self.hour = hour
1027 self.day_of_week = day_of_week
1028 self.day_of_month = day_of_month
1029 self.month_of_year = month_of_year
1030 self.content = content
1032 def __repr__(self) -> str:
1033 return auto_repr(self, sort_attrs=False)
1035 def __str__(self) -> str:
1036 return (
1037 f"{self.minute} {self.hour} {self.day_of_week} "
1038 f"{self.day_of_month} {self.month_of_year} {self.content}"
1039 )
1041 def get_celery_schedule(self) -> celery.schedules.crontab:
1042 """
1043 Returns the corresponding Celery schedule.
1045 Returns:
1046 a :class:`celery.schedules.crontab`
1048 Raises:
1049 :exc:`celery.schedules.ParseException` if the input can't be parsed
1050 """
1051 return celery.schedules.crontab(
1052 minute=self.minute,
1053 hour=self.hour,
1054 day_of_week=self.day_of_week,
1055 day_of_month=self.day_of_month,
1056 month_of_year=self.month_of_year,
1057 )
1060# =============================================================================
1061# Configuration class. (It gets cached on a per-process basis.)
1062# =============================================================================
1064class CamcopsConfig(object):
1065 """
1066 Class representing the CamCOPS configuration.
1067 """
1069 def __init__(self,
1070 config_filename: str,
1071 config_text: str = None) -> None:
1072 """
1073 Initialize by reading the config file.
1075 Args:
1076 config_filename:
1077 Filename of the config file (usual method)
1078 config_text:
1079 Text contents of the config file (alternative method for
1080 special circumstances); overrides ``config_filename``
1081 """
1082 def _get_str(section: str, paramname: str,
1083 default: str = None) -> Optional[str]:
1084 return get_config_parameter(
1085 parser, section, paramname, str, default)
1087 def _get_bool(section: str, paramname: str, default: bool) -> bool:
1088 return get_config_parameter_boolean(
1089 parser, section, paramname, default)
1091 def _get_int(section: str, paramname: str,
1092 default: int = None) -> Optional[int]:
1093 return get_config_parameter(
1094 parser, section, paramname, int, default)
1096 def _get_multiline(section: str, paramname: str) -> List[str]:
1097 # http://stackoverflow.com/questions/335695/lists-in-configparser
1098 return get_config_parameter_multiline(
1099 parser, section, paramname, [])
1101 def _get_multiline_ignoring_comments(section: str,
1102 paramname: str) -> List[str]:
1103 # Returns lines with any trailing comments removed, and any
1104 # comment-only lines removed.
1105 lines = _get_multiline(section, paramname)
1106 return list(filter(None,
1107 (x.split("#")[0].strip() for x in lines if x)))
1109 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1110 # Learn something about our environment
1111 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1112 self.running_under_docker = running_under_docker()
1114 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1115 # Open config file
1116 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1117 self.camcops_config_filename = config_filename
1118 parser = configparser.ConfigParser()
1120 if config_text:
1121 log.info("Reading config from supplied string")
1122 parser.read_string(config_text)
1123 else:
1124 if not config_filename:
1125 raise AssertionError(
1126 f"Environment variable {ENVVAR_CONFIG_FILE} not specified "
1127 f"(and no command-line alternative given)")
1128 log.info("Reading from config file: {!r}", config_filename)
1129 with codecs.open(config_filename, "r", "utf8") as file:
1130 parser.read_file(file)
1132 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1133 # Main section (in alphabetical order)
1134 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1135 s = CONFIG_FILE_SITE_SECTION
1136 cs = ConfigParamSite
1137 cd = ConfigDefaults()
1139 self.allow_insecure_cookies = _get_bool(
1140 s, cs.ALLOW_INSECURE_COOKIES, cd.ALLOW_INSECURE_COOKIES)
1142 self.camcops_logo_file_absolute = _get_str(
1143 s, cs.CAMCOPS_LOGO_FILE_ABSOLUTE, cd.CAMCOPS_LOGO_FILE_ABSOLUTE)
1144 self.ctv_filename_spec = _get_str(s, cs.CTV_FILENAME_SPEC)
1146 self.db_url = parser.get(s, cs.DB_URL)
1147 # ... no default: will fail if not provided
1148 self.db_echo = _get_bool(s, cs.DB_ECHO, cd.DB_ECHO)
1149 self.client_api_loglevel = get_config_parameter_loglevel(
1150 parser, s, cs.CLIENT_API_LOGLEVEL, cd.CLIENT_API_LOGLEVEL)
1151 logging.getLogger("camcops_server.cc_modules.client_api")\
1152 .setLevel(self.client_api_loglevel)
1153 # ... MUTABLE GLOBAL STATE (if relatively unimportant); todo: fix
1155 self.disable_password_autocomplete = _get_bool(
1156 s, cs.DISABLE_PASSWORD_AUTOCOMPLETE,
1157 cd.DISABLE_PASSWORD_AUTOCOMPLETE)
1159 self.email_host = _get_str(s, cs.EMAIL_HOST, "")
1160 self.email_port = _get_int(s, cs.EMAIL_PORT, cd.EMAIL_PORT)
1161 self.email_use_tls = _get_bool(s, cs.EMAIL_USE_TLS, cd.EMAIL_USE_TLS)
1162 self.email_host_username = _get_str(s, cs.EMAIL_HOST_USERNAME, "")
1163 self.email_host_password = _get_str(s, cs.EMAIL_HOST_PASSWORD, "")
1165 self.email_from = _get_str(s, cs.EMAIL_FROM, "")
1166 self.email_sender = _get_str(s, cs.EMAIL_SENDER, "")
1167 self.email_reply_to = _get_str(s, cs.EMAIL_REPLY_TO, "")
1169 self.extra_string_files = _get_multiline(s, cs.EXTRA_STRING_FILES)
1171 self.language = _get_str(s, cs.LANGUAGE, cd.LANGUAGE)
1172 if self.language not in POSSIBLE_LOCALES:
1173 log.warning(f"Invalid language {self.language!r}, "
1174 f"switching to {cd.LANGUAGE!r}")
1175 self.language = cd.LANGUAGE
1176 self.local_institution_url = _get_str(
1177 s, cs.LOCAL_INSTITUTION_URL, cd.LOCAL_INSTITUTION_URL)
1178 self.local_logo_file_absolute = _get_str(
1179 s, cs.LOCAL_LOGO_FILE_ABSOLUTE, cd.LOCAL_LOGO_FILE_ABSOLUTE)
1180 self.lockout_threshold = _get_int(
1181 s, cs.LOCKOUT_THRESHOLD, cd.LOCKOUT_THRESHOLD)
1182 self.lockout_duration_increment_minutes = _get_int(
1183 s, cs.LOCKOUT_DURATION_INCREMENT_MINUTES,
1184 cd.LOCKOUT_DURATION_INCREMENT_MINUTES)
1186 self.password_change_frequency_days = _get_int(
1187 s, cs.PASSWORD_CHANGE_FREQUENCY_DAYS,
1188 cd.PASSWORD_CHANGE_FREQUENCY_DAYS)
1189 self.patient_spec_if_anonymous = _get_str(
1190 s, cs.PATIENT_SPEC_IF_ANONYMOUS, cd.PATIENT_SPEC_IF_ANONYMOUS)
1191 self.patient_spec = _get_str(s, cs.PATIENT_SPEC)
1192 self.permit_immediate_downloads = _get_bool(
1193 s, cs.PERMIT_IMMEDIATE_DOWNLOADS,
1194 cd.PERMIT_IMMEDIATE_DOWNLOADS)
1195 # currently not configurable, but easy to add in the future:
1196 self.plot_fontsize = cd.PLOT_FONTSIZE
1198 self.restricted_tasks = {} # type: Dict[str, List[str]]
1199 # ... maps XML task names to lists of authorized group names
1200 restricted_tasks = _get_multiline(s, cs.RESTRICTED_TASKS)
1201 for rt_line in restricted_tasks:
1202 rt_line = rt_line.split("#")[0].strip()
1203 # ... everything before a '#'
1204 if not rt_line: # comment line
1205 continue
1206 try:
1207 xml_taskname, groupnames = rt_line.split(":")
1208 except ValueError:
1209 raise ValueError(
1210 f"Restricted tasks line not in the format "
1211 f"'xml_taskname: groupname1, groupname2, ...'. Line was:\n"
1212 f"{rt_line!r}"
1213 )
1214 xml_taskname = xml_taskname.strip()
1215 if xml_taskname in self.restricted_tasks:
1216 raise ValueError(f"Duplicate restricted task specification "
1217 f"for {xml_taskname!r}")
1218 groupnames = [x.strip() for x in groupnames.split(",")]
1219 for gn in groupnames:
1220 validate_group_name(gn)
1221 self.restricted_tasks[xml_taskname] = groupnames
1223 self.session_timeout_minutes = _get_int(
1224 s, cs.SESSION_TIMEOUT_MINUTES, cd.SESSION_TIMEOUT_MINUTES)
1225 self.session_cookie_secret = _get_str(s, cs.SESSION_COOKIE_SECRET)
1226 self.session_timeout = datetime.timedelta(
1227 minutes=self.session_timeout_minutes)
1228 self.snomed_task_xml_filename = _get_str(
1229 s, cs.SNOMED_TASK_XML_FILENAME)
1230 self.snomed_icd9_xml_filename = _get_str(
1231 s, cs.SNOMED_ICD9_XML_FILENAME)
1232 self.snomed_icd10_xml_filename = _get_str(
1233 s, cs.SNOMED_ICD10_XML_FILENAME)
1235 self.task_filename_spec = _get_str(s, cs.TASK_FILENAME_SPEC)
1236 self.tracker_filename_spec = _get_str(s, cs.TRACKER_FILENAME_SPEC)
1238 self.user_download_dir = _get_str(s, cs.USER_DOWNLOAD_DIR, "")
1239 self.user_download_file_lifetime_min = _get_int(
1240 s, cs.USER_DOWNLOAD_FILE_LIFETIME_MIN,
1241 cd.USER_DOWNLOAD_FILE_LIFETIME_MIN)
1242 self.user_download_max_space_mb = _get_int(
1243 s, cs.USER_DOWNLOAD_MAX_SPACE_MB,
1244 cd.USER_DOWNLOAD_MAX_SPACE_MB)
1246 self.webview_loglevel = get_config_parameter_loglevel(
1247 parser, s, cs.WEBVIEW_LOGLEVEL, cd.WEBVIEW_LOGLEVEL)
1248 logging.getLogger().setLevel(self.webview_loglevel) # root logger
1249 # ... MUTABLE GLOBAL STATE (if relatively unimportant); todo: fix
1250 self.wkhtmltopdf_filename = _get_str(s, cs.WKHTMLTOPDF_FILENAME)
1252 # More validity checks for the main section:
1253 if not self.patient_spec_if_anonymous:
1254 raise_missing(s, cs.PATIENT_SPEC_IF_ANONYMOUS)
1255 if not self.patient_spec:
1256 raise_missing(s, cs.PATIENT_SPEC)
1257 if not self.session_cookie_secret:
1258 raise_missing(s, cs.SESSION_COOKIE_SECRET)
1259 if not self.task_filename_spec:
1260 raise_missing(s, cs.TASK_FILENAME_SPEC)
1261 if not self.tracker_filename_spec:
1262 raise_missing(s, cs.TRACKER_FILENAME_SPEC)
1263 if not self.ctv_filename_spec:
1264 raise_missing(s, cs.CTV_FILENAME_SPEC)
1266 # To prevent errors:
1267 del s
1268 del cs
1270 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1271 # Web server/WSGI section
1272 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1273 ws = CONFIG_FILE_SERVER_SECTION
1274 cw = ConfigParamServer
1276 self.cherrypy_log_screen = _get_bool(ws, cw.CHERRYPY_LOG_SCREEN,
1277 cd.CHERRYPY_LOG_SCREEN)
1278 self.cherrypy_root_path = _get_str(
1279 ws, cw.CHERRYPY_ROOT_PATH, cd.CHERRYPY_ROOT_PATH)
1280 self.cherrypy_server_name = _get_str(
1281 ws, cw.CHERRYPY_SERVER_NAME, cd.CHERRYPY_SERVER_NAME)
1282 self.cherrypy_threads_max = _get_int(
1283 ws, cw.CHERRYPY_THREADS_MAX, cd.CHERRYPY_THREADS_MAX)
1284 self.cherrypy_threads_start = _get_int(
1285 ws, cw.CHERRYPY_THREADS_START, cd.CHERRYPY_THREADS_START)
1286 self.debug_reverse_proxy = _get_bool(ws, cw.DEBUG_REVERSE_PROXY,
1287 cd.DEBUG_REVERSE_PROXY)
1288 self.debug_show_gunicorn_options = _get_bool(
1289 ws, cw.DEBUG_SHOW_GUNICORN_OPTIONS, cd.DEBUG_SHOW_GUNICORN_OPTIONS)
1290 self.debug_toolbar = _get_bool(ws, cw.DEBUG_TOOLBAR, cd.DEBUG_TOOLBAR)
1291 self.gunicorn_debug_reload = _get_bool(
1292 ws, cw.GUNICORN_DEBUG_RELOAD, cd.GUNICORN_DEBUG_RELOAD)
1293 self.gunicorn_num_workers = _get_int(
1294 ws, cw.GUNICORN_NUM_WORKERS, cd.GUNICORN_NUM_WORKERS)
1295 self.gunicorn_timeout_s = _get_int(
1296 ws, cw.GUNICORN_TIMEOUT_S, cd.GUNICORN_TIMEOUT_S)
1297 self.host = _get_str(ws, cw.HOST, cd.HOST)
1298 self.port = _get_int(ws, cw.PORT, cd.PORT)
1299 self.proxy_http_host = _get_str(ws, cw.PROXY_HTTP_HOST)
1300 self.proxy_remote_addr = _get_str(ws, cw.PROXY_REMOTE_ADDR)
1301 self.proxy_rewrite_path_info = _get_bool(
1302 ws, cw.PROXY_REWRITE_PATH_INFO, cd.PROXY_REWRITE_PATH_INFO)
1303 self.proxy_script_name = _get_str(ws, cw.PROXY_SCRIPT_NAME)
1304 self.proxy_server_name = _get_str(ws, cw.PROXY_SERVER_NAME)
1305 self.proxy_server_port = _get_int(ws, cw.PROXY_SERVER_PORT)
1306 self.proxy_url_scheme = _get_str(ws, cw.PROXY_URL_SCHEME)
1307 self.show_request_immediately = _get_bool(
1308 ws, cw.SHOW_REQUEST_IMMEDIATELY, cd.SHOW_REQUEST_IMMEDIATELY)
1309 self.show_requests = _get_bool(ws, cw.SHOW_REQUESTS, cd.SHOW_REQUESTS)
1310 self.show_response = _get_bool(ws, cw.SHOW_RESPONSE, cd.SHOW_RESPONSE)
1311 self.show_timing = _get_bool(ws, cw.SHOW_TIMING, cd.SHOW_TIMING)
1312 self.ssl_certificate = _get_str(ws, cw.SSL_CERTIFICATE)
1313 self.ssl_private_key = _get_str(ws, cw.SSL_PRIVATE_KEY)
1314 self.static_cache_duration_s = _get_int(ws, cw.STATIC_CACHE_DURATION_S,
1315 cd.STATIC_CACHE_DURATION_S)
1316 self.trusted_proxy_headers = _get_multiline(
1317 ws, cw.TRUSTED_PROXY_HEADERS)
1318 self.unix_domain_socket = _get_str(ws, cw.UNIX_DOMAIN_SOCKET)
1320 for tph in self.trusted_proxy_headers:
1321 if tph not in ReverseProxiedMiddleware.ALL_CANDIDATES:
1322 raise ValueError(
1323 f"Invalid {cw.TRUSTED_PROXY_HEADERS} value specified: "
1324 f"was {tph!r}, options are "
1325 f"{ReverseProxiedMiddleware.ALL_CANDIDATES}")
1327 del ws
1328 del cw
1330 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1331 # Export section
1332 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1333 es = CONFIG_FILE_EXPORT_SECTION
1334 ce = ConfigParamExportGeneral
1336 self.celery_beat_extra_args = _get_multiline(
1337 es, ce.CELERY_BEAT_EXTRA_ARGS)
1338 self.celery_beat_schedule_database = _get_str(
1339 es, ce.CELERY_BEAT_SCHEDULE_DATABASE)
1340 if not self.celery_beat_schedule_database:
1341 raise_missing(es, ce.CELERY_BEAT_SCHEDULE_DATABASE)
1342 self.celery_broker_url = _get_str(
1343 es, ce.CELERY_BROKER_URL, cd.CELERY_BROKER_URL)
1344 self.celery_worker_extra_args = _get_multiline(
1345 es, ce.CELERY_WORKER_EXTRA_ARGS)
1346 self.celery_export_task_rate_limit = _get_str(
1347 es, ce.CELERY_EXPORT_TASK_RATE_LIMIT)
1349 self.export_lockdir = _get_str(es, ce.EXPORT_LOCKDIR)
1350 if not self.export_lockdir:
1351 raise_missing(es, ConfigParamExportGeneral.EXPORT_LOCKDIR)
1353 self.export_recipient_names = _get_multiline_ignoring_comments(
1354 CONFIG_FILE_EXPORT_SECTION, ce.RECIPIENTS)
1355 duplicates = [name for name, count in
1356 collections.Counter(self.export_recipient_names).items()
1357 if count > 1]
1358 if duplicates:
1359 raise ValueError(
1360 f"Duplicate export recipients specified: {duplicates!r}")
1361 for recip_name in self.export_recipient_names:
1362 if re.match(VALID_RECIPIENT_NAME_REGEX, recip_name) is None:
1363 raise ValueError(
1364 f"Recipient names must be alphanumeric or _- only; was "
1365 f"{recip_name!r}")
1366 if len(set(self.export_recipient_names)) != len(self.export_recipient_names): # noqa
1367 raise ValueError("Recipient names contain duplicates")
1368 self._export_recipients = [] # type: List[ExportRecipientInfo]
1369 self._read_export_recipients(parser)
1371 self.schedule_timezone = _get_str(
1372 es, ce.SCHEDULE_TIMEZONE, cd.SCHEDULE_TIMEZONE)
1374 self.crontab_entries = [] # type: List[CrontabEntry]
1375 crontab_lines = _get_multiline(es, ce.SCHEDULE)
1376 for crontab_line in crontab_lines:
1377 crontab_line = crontab_line.split("#")[0].strip()
1378 # ... everything before a '#'
1379 if not crontab_line: # comment line
1380 continue
1381 crontab_entry = CrontabEntry(line=crontab_line)
1382 if crontab_entry.content not in self.export_recipient_names:
1383 raise ValueError(
1384 f"{ce.SCHEDULE} setting exists for non-existent recipient "
1385 f"{crontab_entry.content}")
1386 self.crontab_entries.append(crontab_entry)
1388 del es
1389 del ce
1391 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1392 # Other attributes
1393 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1394 self._sqla_engine = None
1396 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1397 # Docker checks
1398 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1399 if self.running_under_docker:
1400 log.info("Docker environment detected")
1402 # Values expected to be fixed
1403 warn_if_not_docker_value(
1404 param_name=ConfigParamExportGeneral.CELERY_BROKER_URL,
1405 actual_value=self.celery_broker_url,
1406 required_value=DockerConstants.CELERY_BROKER_URL
1407 )
1408 warn_if_not_docker_value(
1409 param_name=ConfigParamServer.HOST,
1410 actual_value=self.host,
1411 required_value=DockerConstants.HOST
1412 )
1414 # Values expected to be present
1415 #
1416 # - Re SSL certificates: reconsidered. People may want to run
1417 # internal plain HTTP but then an Apache front end, and they
1418 # wouldn't appreciate the warnings.
1419 #
1420 # warn_if_not_present(
1421 # param_name=ConfigParamServer.SSL_CERTIFICATE,
1422 # value=self.ssl_certificate
1423 # )
1424 # warn_if_not_present(
1425 # param_name=ConfigParamServer.SSL_PRIVATE_KEY,
1426 # value=self.ssl_private_key
1427 # )
1429 # Config-related files
1430 warn_if_not_within_docker_dir(
1431 param_name=ConfigParamServer.SSL_CERTIFICATE,
1432 filespec=self.ssl_certificate,
1433 permit_cfg=True
1434 )
1435 warn_if_not_within_docker_dir(
1436 param_name=ConfigParamServer.SSL_PRIVATE_KEY,
1437 filespec=self.ssl_private_key,
1438 permit_cfg=True
1439 )
1440 warn_if_not_within_docker_dir(
1441 param_name=ConfigParamSite.LOCAL_LOGO_FILE_ABSOLUTE,
1442 filespec=self.local_logo_file_absolute,
1443 permit_cfg=True,
1444 permit_venv=True
1445 )
1446 warn_if_not_within_docker_dir(
1447 param_name=ConfigParamSite.CAMCOPS_LOGO_FILE_ABSOLUTE,
1448 filespec=self.camcops_logo_file_absolute,
1449 permit_cfg=True,
1450 permit_venv=True
1451 )
1452 for esf in self.extra_string_files:
1453 warn_if_not_within_docker_dir(
1454 param_name=ConfigParamSite.EXTRA_STRING_FILES,
1455 filespec=esf,
1456 permit_cfg=True,
1457 permit_venv=True,
1458 param_contains_not_is=True
1459 )
1460 warn_if_not_within_docker_dir(
1461 param_name=ConfigParamSite.SNOMED_ICD9_XML_FILENAME,
1462 filespec=self.snomed_icd9_xml_filename,
1463 permit_cfg=True,
1464 permit_venv=True
1465 )
1466 warn_if_not_within_docker_dir(
1467 param_name=ConfigParamSite.SNOMED_ICD10_XML_FILENAME,
1468 filespec=self.snomed_icd10_xml_filename,
1469 permit_cfg=True,
1470 permit_venv=True
1471 )
1472 warn_if_not_within_docker_dir(
1473 param_name=ConfigParamSite.SNOMED_TASK_XML_FILENAME,
1474 filespec=self.snomed_task_xml_filename,
1475 permit_cfg=True,
1476 permit_venv=True
1477 )
1479 # Temporary/scratch space that needs to be shared between Docker
1480 # containers
1481 warn_if_not_within_docker_dir(
1482 param_name=ConfigParamSite.USER_DOWNLOAD_DIR,
1483 filespec=self.user_download_dir,
1484 permit_tmp=True
1485 )
1486 warn_if_not_within_docker_dir(
1487 param_name=ConfigParamExportGeneral.CELERY_BEAT_SCHEDULE_DATABASE, # noqa
1488 filespec=self.celery_beat_schedule_database,
1489 permit_tmp=True
1490 )
1491 warn_if_not_within_docker_dir(
1492 param_name=ConfigParamExportGeneral.EXPORT_LOCKDIR,
1493 filespec=self.export_lockdir,
1494 permit_tmp=True
1495 )
1497 # -------------------------------------------------------------------------
1498 # Database functions
1499 # -------------------------------------------------------------------------
1501 def get_sqla_engine(self) -> Engine:
1502 """
1503 Returns an SQLAlchemy :class:`Engine`.
1505 I was previously misinterpreting the appropriate scope of an Engine.
1506 I thought: create one per request.
1507 But the Engine represents the connection *pool*.
1508 So if you create them all the time, you get e.g. a
1509 'Too many connections' error.
1511 "The appropriate scope is once per [database] URL per application,
1512 at the module level."
1514 - https://groups.google.com/forum/#!topic/sqlalchemy/ZtCo2DsHhS4
1515 - https://stackoverflow.com/questions/8645250/how-to-close-sqlalchemy-connection-in-mysql
1517 Now, our CamcopsConfig instance is cached, so there should be one of
1518 them overall. See get_config() below.
1520 Therefore, making the engine a member of this class should do the
1521 trick, whilst avoiding global variables.
1522 """ # noqa
1523 if self._sqla_engine is None:
1524 self._sqla_engine = create_engine(
1525 self.db_url,
1526 echo=self.db_echo,
1527 pool_pre_ping=True,
1528 # pool_size=0, # no limit (for parallel testing, which failed)
1529 )
1530 log.debug("Created SQLAlchemy engine for URL {}",
1531 get_safe_url_from_engine(self._sqla_engine))
1532 return self._sqla_engine
1534 @property
1535 @cache_region_static.cache_on_arguments(function_key_generator=fkg)
1536 def get_all_table_names(self) -> List[str]:
1537 """
1538 Returns all table names from the database.
1539 """
1540 log.debug("Fetching database table names")
1541 engine = self.get_sqla_engine()
1542 return get_table_names(engine=engine)
1544 def get_dbsession_raw(self) -> SqlASession:
1545 """
1546 Returns a raw SQLAlchemy Session.
1547 Avoid this -- use :func:`get_dbsession_context` instead.
1548 """
1549 engine = self.get_sqla_engine()
1550 maker = sessionmaker(bind=engine)
1551 dbsession = maker() # type: SqlASession
1552 return dbsession
1554 @contextlib.contextmanager
1555 def get_dbsession_context(self) -> Generator[SqlASession, None, None]:
1556 """
1557 Context manager to provide an SQLAlchemy session that will COMMIT
1558 once we've finished, or perform a ROLLBACK if there was an exception.
1559 """
1560 dbsession = self.get_dbsession_raw()
1561 # noinspection PyBroadException
1562 try:
1563 yield dbsession
1564 dbsession.commit()
1565 except Exception:
1566 dbsession.rollback()
1567 raise
1568 finally:
1569 dbsession.close()
1571 def _assert_valid_database_engine(self) -> None:
1572 """
1573 Assert that our backend database is a valid type.
1575 Specifically, we prohibit:
1577 - SQL Server versions before 2008: they don't support timezones
1578 and we need that.
1579 """
1580 engine = self.get_sqla_engine()
1581 if not is_sqlserver(engine):
1582 return
1583 assert is_sqlserver_2008_or_later(engine), (
1584 "If you use Microsoft SQL Server as the back-end database for a "
1585 "CamCOPS server, it must be at least SQL Server 2008. Older "
1586 "versions do not have time zone awareness."
1587 )
1589 def _assert_database_is_at_head(self) -> None:
1590 """
1591 Assert that the current database is at its head (most recent) revision,
1592 by comparing its Alembic version number (written into the Alembic
1593 version table of the database) to the most recent Alembic revision in
1594 our ``camcops_server/alembic/versions`` directory.
1595 """
1596 current, head = get_current_and_head_revision(
1597 database_url=self.db_url,
1598 alembic_config_filename=ALEMBIC_CONFIG_FILENAME,
1599 alembic_base_dir=ALEMBIC_BASE_DIR,
1600 version_table=ALEMBIC_VERSION_TABLE,
1601 )
1602 if current == head:
1603 log.debug("Database is at correct (head) revision of {}", current)
1604 else:
1605 raise_runtime_error(
1606 f"Database structure is at version {current} but should be at "
1607 f"version {head}. CamCOPS will not start. Please use the "
1608 f"'upgrade_db' command to fix this.")
1610 def assert_database_ok(self) -> None:
1611 """
1612 Asserts that our database engine is OK and our database structure is
1613 correct.
1614 """
1615 self._assert_valid_database_engine()
1616 self._assert_database_is_at_head()
1618 # -------------------------------------------------------------------------
1619 # SNOMED-CT functions
1620 # -------------------------------------------------------------------------
1622 def get_task_snomed_concepts(self) -> Dict[str, SnomedConcept]:
1623 """
1624 Returns all SNOMED-CT concepts for tasks.
1626 Returns:
1627 dict: maps lookup strings to :class:`SnomedConcept` objects
1628 """
1629 if not self.snomed_task_xml_filename:
1630 return {}
1631 return get_all_task_snomed_concepts(self.snomed_task_xml_filename)
1633 def get_icd9cm_snomed_concepts(self) -> Dict[str, List[SnomedConcept]]:
1634 """
1635 Returns all SNOMED-CT concepts for ICD-9-CM codes supported by CamCOPS.
1637 Returns:
1638 dict: maps ICD-9-CM codes to :class:`SnomedConcept` objects
1639 """
1640 if not self.snomed_icd9_xml_filename:
1641 return {}
1642 return get_icd9_snomed_concepts_from_xml(self.snomed_icd9_xml_filename)
1644 def get_icd10_snomed_concepts(self) -> Dict[str, List[SnomedConcept]]:
1645 """
1646 Returns all SNOMED-CT concepts for ICD-10-CM codes supported by
1647 CamCOPS.
1649 Returns:
1650 dict: maps ICD-10 codes to :class:`SnomedConcept` objects
1651 """
1652 if not self.snomed_icd10_xml_filename:
1653 return {}
1654 return get_icd10_snomed_concepts_from_xml(
1655 self.snomed_icd10_xml_filename)
1657 # -------------------------------------------------------------------------
1658 # Export functions
1659 # -------------------------------------------------------------------------
1661 def _read_export_recipients(
1662 self,
1663 parser: configparser.ConfigParser = None) -> None:
1664 """
1665 Loads
1666 :class:`camcops_server.cc_modules.cc_exportrecipientinfo.ExportRecipientInfo`
1667 objects from the config file. Stores them in
1668 ``self._export_recipients``.
1670 Note that these objects are **not** associated with a database session.
1672 Args:
1673 parser: optional :class:`configparser.ConfigParser` object.
1674 """
1675 self._export_recipients = [] # type: List[ExportRecipientInfo]
1676 for recip_name in self.export_recipient_names:
1677 log.debug("Loading export config for recipient {!r}", recip_name)
1678 try:
1679 validate_export_recipient_name(recip_name)
1680 except ValueError as e:
1681 raise ValueError(f"Bad recipient name {recip_name!r}: {e}")
1682 recipient = ExportRecipientInfo.read_from_config(
1683 self, parser=parser, recipient_name=recip_name)
1684 self._export_recipients.append(recipient)
1686 def get_all_export_recipient_info(self) -> List["ExportRecipientInfo"]:
1687 """
1688 Returns all export recipients (in their "database unaware" form)
1689 specified in the config.
1691 Returns:
1692 list: of
1693 :class:`camcops_server.cc_modules.cc_exportrecipientinfo.ExportRecipientInfo`
1694 """ # noqa
1695 return self._export_recipients
1697 # -------------------------------------------------------------------------
1698 # File-based locks
1699 # -------------------------------------------------------------------------
1701 def get_export_lockfilename_db(self, recipient_name: str) -> str:
1702 """
1703 Returns a full path to a lockfile suitable for locking for a
1704 whole-database export to a particular export recipient.
1706 Args:
1707 recipient_name: name of the recipient
1709 Returns:
1710 a filename
1711 """
1712 filename = f"camcops_export_db_{recipient_name}"
1713 # ".lock" is appended automatically by the lockfile package
1714 return os.path.join(self.export_lockdir, filename)
1716 def get_export_lockfilename_task(self, recipient_name: str,
1717 basetable: str, pk: int) -> str:
1718 """
1719 Returns a full path to a lockfile suitable for locking for a
1720 single-task export to a particular export recipient.
1722 Args:
1723 recipient_name: name of the recipient
1724 basetable: task base table name
1725 pk: server PK of the task
1727 Returns:
1728 a filename
1729 """
1730 filename = f"camcops_export_task_{recipient_name}_{basetable}_{pk}"
1731 # ".lock" is appended automatically by the lockfile package
1732 return os.path.join(self.export_lockdir, filename)
1734 def get_master_export_recipient_lockfilename(self) -> str:
1735 """
1736 When we are modifying export recipients, we check "is this information
1737 the same as the current version in the database", and if not, we write
1738 fresh information to the database. If lots of processes do that at the
1739 same time, we have a problem (usually a database deadlock) -- hence
1740 this lock.
1742 Returns:
1743 a filename
1744 """
1745 filename = "camcops_master_export_recipient"
1746 # ".lock" is appended automatically by the lockfile package
1747 return os.path.join(self.export_lockdir, filename)
1749 def get_celery_beat_pidfilename(self) -> str:
1750 """
1751 Process ID file (pidfile) used by ``celery beat --pidfile ...``.
1752 """
1753 filename = "camcops_celerybeat.pid"
1754 return os.path.join(self.export_lockdir, filename)
1757# =============================================================================
1758# Get config filename from an appropriate environment (WSGI or OS)
1759# =============================================================================
1761def get_config_filename_from_os_env() -> str:
1762 """
1763 Returns the config filename to use, from our operating system environment
1764 variable.
1766 (We do NOT trust the WSGI environment for this.)
1767 """
1768 config_filename = os.environ.get(ENVVAR_CONFIG_FILE)
1769 if not config_filename:
1770 raise AssertionError(
1771 f"OS environment did not provide the required "
1772 f"environment variable {ENVVAR_CONFIG_FILE}")
1773 return config_filename
1776# =============================================================================
1777# Cached instances
1778# =============================================================================
1780@cache_region_static.cache_on_arguments(function_key_generator=fkg)
1781def get_config(config_filename: str) -> CamcopsConfig:
1782 """
1783 Returns a :class:`camcops_server.cc_modules.cc_config.CamcopsConfig` from
1784 the specified config filename.
1786 Cached.
1787 """
1788 return CamcopsConfig(config_filename)
1791# =============================================================================
1792# Get default config
1793# =============================================================================
1795def get_default_config_from_os_env() -> CamcopsConfig:
1796 """
1797 Returns the :class:`camcops_server.cc_modules.cc_config.CamcopsConfig`
1798 representing the config filename that we read from our operating system
1799 environment variable.
1800 """
1801 if ON_READTHEDOCS:
1802 return CamcopsConfig(config_filename="", config_text=get_demo_config())
1803 else:
1804 return get_config(get_config_filename_from_os_env())