Source code for camcops_server.cc_modules.client_api

#!/usr/bin/env python
# camcops_server/client_api.py

"""
===============================================================================

    Copyright (C) 2012-2018 Rudolf Cardinal (rudolf@pobox.com).

    This file is part of CamCOPS.

    CamCOPS is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    CamCOPS is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with CamCOPS. If not, see <http://www.gnu.org/licenses/>.

===============================================================================

We use primarily SQLAlchemy Core here (in contrast to the ORM used elsewhere).

"""

# =============================================================================
# Imports
# =============================================================================

import logging
# from pprint import pformat
import time
from typing import (Any, Dict, Iterable, List, Optional, Sequence, Tuple,
                    TYPE_CHECKING)
import unittest

from cardinal_pythonlib.convert import (
    base64_64format_encode,
    hex_xformat_encode,
)
from cardinal_pythonlib.datetimefunc import coerce_to_pendulum, format_datetime
from cardinal_pythonlib.logs import (
    BraceStyleAdapter,
    main_only_quicksetup_rootlogger,
)
from cardinal_pythonlib.pyramid.responses import TextResponse
from cardinal_pythonlib.sql.literals import sql_quote_string
from cardinal_pythonlib.sqlalchemy.core_query import (
    exists_in_table,
    fetch_all_first_values,
)
from cardinal_pythonlib.text import escape_newlines, unescape_newlines
from pendulum import DateTime as Pendulum
from pyramid.view import view_config
from pyramid.response import Response
from pyramid.security import NO_PERMISSION_REQUIRED
from semantic_version import Version
from sqlalchemy.engine.result import ResultProxy
from sqlalchemy.exc import IntegrityError
from sqlalchemy.sql.expression import exists, select, text, update
from sqlalchemy.sql.schema import Table

from camcops_server.cc_modules import cc_audit  # avoids "audit" name clash
from .cc_all_models import CLIENT_TABLE_MAP, RESERVED_FIELDS
from .cc_blob import Blob
from .cc_cache import cache_region_static, fkg
from .cc_client_api_core import (
    AllowedTablesFieldNames,
    exception_description,
    ExtraStringFieldNames,
    fail_server_error,
    fail_unsupported_operation,
    fail_user_error,
    IgnoringAntiqueTableException,
    require_keys,
    ServerErrorException,
    TabletParam,
    UserErrorException,
)
from .cc_constants import (
    CLIENT_DATE_FIELD,
    DateFormat,
    ERA_NOW,
    FP_ID_NUM,
    FP_ID_DESC,
    FP_ID_SHORT_DESC,
    MOVE_OFF_TABLET_FIELD,
    NUMBER_OF_IDNUMS_DEFUNCT,  # allowed; for old tablet versions
    TABLET_ID_FIELD,
)
from .cc_convert import decode_values, encode_single_value
from .cc_device import Device
from .cc_dirtytables import DirtyTable
from .cc_group import Group
from .cc_patient import Patient
from .cc_patientidnum import fake_tablet_id_for_patientidnum, PatientIdNum
from .cc_pyramid import Routes
from .cc_request import CamcopsRequest
from .cc_specialnote import SpecialNote
from .cc_task import all_task_tables_with_min_client_version
from .cc_unittest import DemoDatabaseTestCase
from .cc_version import CAMCOPS_SERVER_VERSION_STRING, MINIMUM_TABLET_VERSION

if TYPE_CHECKING:
    from pyramid.request import Request

log = BraceStyleAdapter(logging.getLogger(__name__))


# =============================================================================
# Constants
# =============================================================================

COPE_WITH_DELETED_PATIENT_DESCRIPTIONS = True
# ... as of client 2.0.0, ID descriptions are no longer duplicated.
# As of server 2.0.0, the fields still exist in the database, but the reporting
# and consistency check has been removed. In the next version of the server,
# the fields will be removed, and then the server should cope with old clients,
# at least for a while.

DUPLICATE_FAILED = "Failed to duplicate record"
INSERT_FAILED = "Failed to insert record"

# REGEX_INVALID_TABLE_FIELD_CHARS = re.compile("[^a-zA-Z0-9_]")
# ... the ^ within the [] means the expression will match any character NOT in
# the specified range

DEVICE_STORED_VAR_TABLENAME_DEFUNCT = "storedvars"
# ... old table, no longer in use, that Titanium clients used to upload.
# We recognize and ignore it now so that old clients can still work.

SILENTLY_IGNORE_TABLENAMES = [DEVICE_STORED_VAR_TABLENAME_DEFUNCT]

IGNORING_ANTIQUE_TABLE_MESSAGE = (
    "Ignoring user request to upload antique/defunct table, but reporting "
    "success to the client"
)

SUCCESS_CODE = "1"
FAILURE_CODE = "0"


# =============================================================================
# Cached information
# =============================================================================

@cache_region_static.cache_on_arguments(function_key_generator=fkg)
def all_tables_with_min_client_version() -> Dict[str, Version]:
    d = all_task_tables_with_min_client_version()
    d[Blob.__tablename__] = MINIMUM_TABLET_VERSION
    d[Patient.__tablename__] = MINIMUM_TABLET_VERSION
    d[PatientIdNum.__tablename__] = MINIMUM_TABLET_VERSION
    # log.critical("{}", pformat(d))
    return d


# =============================================================================
# Validators
# =============================================================================

[docs]def ensure_valid_table_name(req: CamcopsRequest, tablename: str) -> None: """ Ensures a table name doesn't contain bad characters, isn't a reserved table that the user is prohibited from accessing, and is a valid table name that's in the database. Raises UserErrorException upon failure. ... 2017-10-08: shortcut to all that: it's OK if it's listed as a valid client table. ... 2018-01-16 (v2.2.0): check also that client version is OK """ if tablename not in CLIENT_TABLE_MAP: fail_user_error("Invalid client table name: {}".format(tablename)) tables_versions = all_tables_with_min_client_version() assert tablename in tables_versions client_version = req.tabletsession.tablet_version_ver minimum_client_version = tables_versions[tablename] if client_version < minimum_client_version: fail_user_error( "Client CamCOPS version {cv} is less than the version ({minver}) " "required to handle table {t}".format( cv=client_version, minver=minimum_client_version, t=tablename ) )
[docs]def ensure_valid_field_name(table: Table, fieldname: str) -> None: """ Ensures a field name contains only valid characters, and isn't a reserved fieldname that the user isn't allowed to access. Raises UserErrorException upon failure. ... 2017-10-08: shortcut: it's OK if it's a column name for a particular table. """ # if fieldname in RESERVED_FIELDS: if fieldname.startswith("_"): # all reserved fields start with _ # ... but not all fields starting with "_" are reserved; e.g. # "_move_off_tablet" is allowed. if fieldname in RESERVED_FIELDS: fail_user_error("Reserved field name for table {!r}: {!r}".format( table.name, fieldname)) if fieldname not in table.columns.keys(): fail_user_error("Invalid field name for table {!r}: {!r}".format( table.name, fieldname))
# Note that the reserved-field check is case-sensitive, but so is the # "present in table" check. So for a malicious uploader trying to use, for # example, "_PK", this would not be picked up as a reserved field (so would # pass that check) but then wouldn't be recognized as a valid field (so # would fail). # ============================================================================= # Extracting information from the POST request # =============================================================================
[docs]def get_str_var(req: CamcopsRequest, var: str, mandatory: bool = True) -> Optional[str]: """ Retrieves a string variable from CamcopsRequest, raising an error that gets passed to the client device if it's mandatory and missing. Args: req: CamcopsRequest var: name of variable to retrieve mandatory: if True, script aborts if variable missing Returns: value """ val = req.get_str_param(var, default=None) if mandatory and val is None: fail_user_error("Must provide the variable: {}".format(var)) return val
def get_int_var(req: CamcopsRequest, var: str, mandatory: bool = True) -> int: s = get_str_var(req, var, mandatory) try: return int(s) except (TypeError, ValueError): fail_user_error("Variable {} is not a valid integer; was {!r}".format( var, s))
[docs]def get_table_from_req(req: CamcopsRequest, var: str) -> Table: """ Retrieves a table name from a CGI form and checks it's a valid client table. """ tablename = get_str_var(req, var, mandatory=True) if tablename in SILENTLY_IGNORE_TABLENAMES: raise IgnoringAntiqueTableException( "Ignoring table {}".format(tablename)) ensure_valid_table_name(req, tablename) return CLIENT_TABLE_MAP[tablename]
[docs]def get_tables_from_post_var(req: CamcopsRequest, var: str, mandatory: bool = True) -> List[Table]: """ Gets a list of tables from a CGI form variable, and ensures all are valid. """ cstables = get_str_var(req, var, mandatory=mandatory) if not cstables: return [] # can't have any commas in table names, so it's OK to use a simple # split() command tablenames = [x.strip() for x in cstables.split(",")] tables = [] # type: List[Table] for tn in tablenames: if tn in SILENTLY_IGNORE_TABLENAMES: log.warning(IGNORING_ANTIQUE_TABLE_MESSAGE) continue ensure_valid_table_name(req, tn) tables.append(CLIENT_TABLE_MAP[tn]) return tables
[docs]def get_single_field_from_post_var(req: CamcopsRequest, table: Table, var: str, mandatory: bool = True) -> str: """ Retrieves a field name from a the request and checks it's not a bad fieldname. """ field = get_str_var(req, var, mandatory=mandatory) ensure_valid_field_name(table, field) return field
[docs]def get_fields_from_post_var(req: CamcopsRequest, table: Table, var: str, mandatory: bool = True) -> List[str]: """ Get a comma-separated list of field names from a request and checks that all are acceptable. Returns a list of fieldnames. """ csfields = get_str_var(req, var, mandatory=mandatory) if not csfields: return [] # can't have any commas in fields, so it's OK to use a simple # split() command fields = [x.strip() for x in csfields.split(",")] for f in fields: ensure_valid_field_name(table, f) return fields
[docs]def get_values_from_post_var(req: CamcopsRequest, var: str, mandatory: bool = True) -> List[Any]: """ Retrieves a list of values from a CSV-separated list of SQL values stored in a CGI form (including e.g. NULL, numbers, quoted strings, and special handling for base-64/hex-encoded BLOBs.) """ csvalues = get_str_var(req, var, mandatory=mandatory) if not csvalues: return [] return decode_values(csvalues)
[docs]def get_fields_and_values(req: CamcopsRequest, table: Table, fields_var: str, values_var: str, mandatory: bool = True) -> Dict[str, Any]: """ Gets fieldnames and matching values from two variables in a request. """ fields = get_fields_from_post_var(req, table, fields_var, mandatory=mandatory) values = get_values_from_post_var(req, values_var, mandatory=mandatory) if len(fields) != len(values): fail_user_error( "Number of fields ({f}) doesn't match number of values " "({v})".format(f=len(fields), v=len(values)) ) return dict(list(zip(fields, values)))
# ============================================================================= # Sending stuff to the client # =============================================================================
[docs]def get_server_id_info(req: CamcopsRequest) -> Dict[str, str]: """Returns a reply for the tablet giving details of the server.""" group = Group.get_group_by_id(req.dbsession, req.user.upload_group_id) reply = { TabletParam.DATABASE_TITLE: req.database_title, TabletParam.ID_POLICY_UPLOAD: group.upload_policy or "", TabletParam.ID_POLICY_FINALIZE: group.finalize_policy or "", TabletParam.SERVER_CAMCOPS_VERSION: CAMCOPS_SERVER_VERSION_STRING, } for n in req.valid_which_idnums: nstr = str(n) reply[TabletParam.ID_DESCRIPTION_PREFIX + nstr] = \ req.get_id_desc(n, "") reply[TabletParam.ID_SHORT_DESCRIPTION_PREFIX + nstr] = \ req.get_id_shortdesc(n, "") return reply
[docs]def get_select_reply(fields: Sequence[str], rows: Sequence[Sequence[Any]]) -> Dict[str, str]: """ Return format: .. code-block:: none nfields:X fields:X nrecords:X record0:VALUES_AS_CSV_LIST_OF_ENCODED_SQL_VALUES ... record{n}:VALUES_AS_CSV_LIST_OF_ENCODED_SQL_VALUES """ nrecords = len(rows) reply = { TabletParam.NFIELDS: len(fields), TabletParam.FIELDS: ",".join(fields), TabletParam.NRECORDS: nrecords, } for r in range(nrecords): row = rows[r] encodedvalues = [] # type: List[str] for val in row: encodedvalues.append(encode_single_value(val)) reply[TabletParam.RECORD_PREFIX + str(r)] = ",".join(encodedvalues) return reply
# ============================================================================= # CamCOPS table functions # =============================================================================
[docs]def get_server_pks_of_active_records(req: CamcopsRequest, table: Table) -> List[int]: """ Gets server PKs of active records (_current and in the 'NOW' era) for the specified device/table. """ # noinspection PyProtectedMember query = ( select([table.c._pk]) .where(table.c._device_id == req.tabletsession.device_id) .where(table.c._current) .where(table.c._era == ERA_NOW) ) return fetch_all_first_values(req.dbsession, query)
[docs]def record_exists(req: CamcopsRequest, table: Table, clientpk_name: str, clientpk_value: Any) -> Tuple[bool, Optional[int]]: """ Checks if a record exists, using the device's perspective of a table/client PK combination. Returns (exists, serverpk), where exists is Boolean. If exists is False, serverpk will be None. """ # log.critical("record_exists: checking table={}, {}={}", # table.name, clientpk_name, clientpk_value) # noinspection PyProtectedMember query = ( select([table.c._pk]) .where(table.c._device_id == req.tabletsession.device_id) .where(table.c._current) .where(table.c._era == ERA_NOW) .where(table.c[clientpk_name] == clientpk_value) ) pklist = fetch_all_first_values(req.dbsession, query) rec_exists = bool(len(pklist) >= 1) serverpk = pklist[0] if rec_exists else None return rec_exists, serverpk
# Consider a warning/failure if we have >1 row meeting these criteria. # Not currently checked for.
[docs]def client_pks_that_exist(req: CamcopsRequest, table: Table, clientpk_name: str, clientpk_values: List[int]) -> Dict[int, int]: """ Searches for client PK values (for this device, current, and 'now') matching the input list. Returns a dictionary of {clientpk: serverpk} values for those that do. """ # noinspection PyProtectedMember query = ( select([table.c[clientpk_name], table.c._pk]) .where(table.c._device_id == req.tabletsession.device_id) .where(table.c._current) .where(table.c._era == ERA_NOW) .where(table.c[clientpk_name].in_(clientpk_values)) ) rows = req.dbsession.execute(query) clientpkdict = {} # type: Dict[int, int] for client_pk, server_pk in rows: clientpkdict[client_pk] = server_pk return clientpkdict
[docs]def get_server_pks_of_specified_records(req: CamcopsRequest, table: Table, wheredict: Dict) -> List[int]: """ Retrieves server PKs for a table, for a given device, given a set of 'where' conditions specified in wheredict (as field/value combinations, joined with AND). """ # noinspection PyProtectedMember query = ( select([table.c._pk]) .where(table.c._device_id == req.tabletsession.device_id) .where(table.c._current) .where(table.c._era == ERA_NOW) ) for fieldname, value in wheredict.items(): query = query.where(table.c[fieldname] == value) return fetch_all_first_values(req.dbsession, query)
[docs]def record_identical_full(req: CamcopsRequest, table: Table, serverpk: int, wheredict: Dict) -> bool: """ If a record with the specified server PK exists in the specified table having all its values matching the field/value combinations in wheredict (joined with AND), returns True. Otherwise, returns False. Used to detect if an incoming record matches the database record. CURRENTLY UNUSED. """ # noinspection PyProtectedMember criteria = [table.c._pk == serverpk] for fieldname, value in wheredict.items(): criteria.append(table.c[fieldname] == value) return exists_in_table(req.dbsession, table, *criteria)
[docs]def record_identical_by_date(req: CamcopsRequest, table: Table, serverpk: int, client_date_value: Pendulum) -> bool: """ Shortcut to detecting a record being identical. Returns true if the record (defined by its table/server PK) has a CLIENT_DATE_FIELD field that matches that of the incoming record. As long as the tablet always updates the CLIENT_DATE_FIELD when it saves a record, and the clock on the device doesn't go backwards by a certain exact millisecond-precision value, this is a valid method. """ # noinspection PyProtectedMember criteria = [ table.c._pk == serverpk, table.c[CLIENT_DATE_FIELD] == client_date_value, ] return exists_in_table(req.dbsession, table, *criteria)
[docs]def upload_record_core(req: CamcopsRequest, table: Table, clientpk_name: str, valuedict: Dict, recordnum: int) -> Tuple[Optional[int], int]: """ Uploads a record. Deals with IDENTICAL, NEW, and MODIFIED records. """ ts = req.tabletsession require_keys(valuedict, [clientpk_name, CLIENT_DATE_FIELD, MOVE_OFF_TABLET_FIELD]) clientpk_value = valuedict[clientpk_name] found, oldserverpk = record_exists(req, table, clientpk_name, clientpk_value) newserverpk = None if found: client_date_value = valuedict[CLIENT_DATE_FIELD] if record_identical_by_date(req, table, oldserverpk, client_date_value): # log.critical("... record is identical") # IDENTICAL. No action needed... # UNLESS MOVE_OFF_TABLET_FIELDNAME is set if valuedict[MOVE_OFF_TABLET_FIELD]: flag_record_for_preservation(req, table, oldserverpk) # log.debug("Table {table}, uploaded record {recordnum}: " # "identical but moving off tablet", # table=table, recordnum=recordnum) else: # log.debug("Table {table}, uploaded record {recordnum}: " # "identical", table=table, recordnum=recordnum) pass else: # MODIFIED # log.critical("... record is modified") if table == Patient.__tablename__: if ts.cope_with_deleted_patient_descriptors: # Old tablets (pre-2.0.0) will upload copies of the ID # descriptions with the patient. To cope with that, we # remove those here: for n in range(1, NUMBER_OF_IDNUMS_DEFUNCT + 1): nstr = str(n) fn_desc = FP_ID_DESC + nstr fn_shortdesc = FP_ID_SHORT_DESC + nstr valuedict.pop(fn_desc, None) # remove item, if exists valuedict.pop(fn_shortdesc, None) if ts.cope_with_old_idnums: # Insert records into the new ID number table from the old # patient table: for which_idnum in range(1, NUMBER_OF_IDNUMS_DEFUNCT + 1): nstr = str(which_idnum) fn_idnum = FP_ID_NUM + nstr idnum_value = valuedict.pop(fn_idnum, None) # ... and remove it from our new Patient record patient_id = valuedict.get("id", None) if idnum_value is None or patient_id is None: continue mark_table_dirty(req, PatientIdNum.__table__) _, _ = upload_record_core( req=req, table=PatientIdNum.__table__, clientpk_name='id', valuedict={ 'id': fake_tablet_id_for_patientidnum( patient_id=patient_id, which_idnum=which_idnum ), # ... guarantees a pseudo client PK 'patient_id': patient_id, 'which_idnum': which_idnum, 'idnum_value': idnum_value, CLIENT_DATE_FIELD: client_date_value, MOVE_OFF_TABLET_FIELD: valuedict[MOVE_OFF_TABLET_FIELD], # noqa }, recordnum=recordnum ) # Now, how to deal with deletion, i.e. records missing # from the tablet? # See our caller, upload_table(). newserverpk = insert_record(req, table, valuedict, oldserverpk) flag_modified(req, table, oldserverpk, newserverpk) # log.debug("Table {table}, record {recordnum}: modified", # table=table.name, recordnum=recordnum) else: # log.critical("... record is new") # NEW newserverpk = insert_record(req, table, valuedict, None) # log.debug("Table {table}, record {recordnum}: new", # table=table.name, recordnum=recordnum) return oldserverpk, newserverpk
[docs]def insert_record(req: CamcopsRequest, table: Table, valuedict: Dict, predecessor_pk: Optional[int]) -> int: """ Inserts a record, or raises an exception if that fails. """ mark_table_dirty(req, table) ts = req.tabletsession valuedict.update({ "_device_id": ts.device_id, "_era": ERA_NOW, "_current": 0, "_addition_pending": 1, "_removal_pending": 0, "_predecessor_pk": predecessor_pk, "_camcops_version": ts.tablet_version_str, "_group_id": req.user.upload_group_id, }) rp = req.dbsession.execute( table.insert().values(valuedict) ) # type: ResultProxy return rp.inserted_primary_key
[docs]def duplicate_record(req: CamcopsRequest, table: Table, serverpk: int) -> int: """ Duplicates the record defined by the table/serverpk combination. Will raise an exception if the insert fails. Otherwise... The old record then NEEDS MODIFICATION by flag_modified(). The new record NEEDS MODIFICATION by update_new_copy_of_record(). """ mark_table_dirty(req, table) # Fetch the existing record. # noinspection PyProtectedMember query = ( select([text('*')]) .select_from(table) .where(table.c._pk == serverpk) ) rp = req.dbsession.execute(query) # type: ResultProxy row = rp.fetchone() if not row: raise ServerErrorException( "Tried to fetch nonexistent record: table {t}, PK {pk}".format( t=table.name, pk=serverpk)) valuedict = dict(row) # Remove the PK from what we insert back (that will be autogenerated) valuedict.pop("_pk", None) # ... or del d["_pk"]; http://stackoverflow.com/questions/5447494 # Perform the insert insert_rp = req.dbsession.execute( table.insert().values(valuedict) ) # type: ResultProxy return insert_rp.inserted_primary_key
[docs]def update_new_copy_of_record(req: CamcopsRequest, table: Table, serverpk: int, valuedict: Dict, predecessor_pk: int) -> None: """ Following duplicate_record(), use this to modify the new copy (defined by the table/serverpk combination). """ valuedict.update(dict( _current=0, _addition_pending=1, _predecessor_pk=predecessor_pk, _camcops_version=req.tabletsession.tablet_version_str )) # noinspection PyProtectedMember req.dbsession.execute( update(table) .where(table.c._pk == serverpk) .values(valuedict) )
# ============================================================================= # Batch (atomic) upload and preserving # =============================================================================
[docs]def get_batch_details_start_if_needed(req: CamcopsRequest) \ -> Tuple[Optional[Pendulum], Optional[bool]]: """ Gets a (upload_batch_utc, currently_preserving) tuple. upload_batch_utc: the batchtime; UTC date/time of the current upload batch. currently_preserving: Boolean; whether preservation (shifting to an older era) is currently taking place. SIDE EFFECT: if the username is different from the username that started a previous upload batch for this device, we restart the upload batch (thus rolling back previous pending changes). """ query = ( select([Device.ongoing_upload_batch_utc, Device.uploading_user_id, Device.currently_preserving]) .select_from(Device.__table__) .where(Device.id == req.tabletsession.device_id) ) row = req.dbsession.execute(query).fetchone() if not row: return None, None upload_batch_utc, uploading_user_id, currently_preserving = row if not upload_batch_utc or uploading_user_id != req.user_id: # SIDE EFFECT: if the username changes, we restart (and thus roll back # previous pending changes) start_device_upload_batch(req) return req.now_utc, False # log.debug("get_batch_details_start_if_needed: upload_batch_utc = {!r}", # upload_batch_utc) return upload_batch_utc, currently_preserving
[docs]def start_device_upload_batch(req: CamcopsRequest) -> None: """ Starts an upload batch for a device. """ rollback_all(req) req.dbsession.execute( update(Device.__table__) .where(Device.id == req.tabletsession.device_id) .values(last_upload_batch_utc=req.now_utc, ongoing_upload_batch_utc=req.now_utc, uploading_user_id=req.tabletsession.user_id) )
[docs]def end_device_upload_batch(req: CamcopsRequest, batchtime: Pendulum, preserving: bool) -> None: """ Ends an upload batch, committing all changes made thus far. """ commit_all(req, batchtime, preserving) req.dbsession.execute( update(Device.__table__) .where(Device.id == req.tabletsession.device_id) .values(ongoing_upload_batch_utc=None, uploading_user_id=None, currently_preserving=0) )
[docs]def start_preserving(req: CamcopsRequest) -> None: """ Starts preservation (the process of moving records from the NOW era to an older era, so they can be removed safely from the tablet). """ req.dbsession.execute( update(Device.__table__) .where(Device.id == req.tabletsession.device_id) .values(currently_preserving=1) )
[docs]def mark_table_dirty(req: CamcopsRequest, table: Table) -> None: """ Marks a table as having been modified during the current upload. """ dbsession = req.dbsession ts = req.tabletsession table_already_dirty = exists_in_table( dbsession, DirtyTable.__table__, DirtyTable.device_id == ts.device_id, DirtyTable.tablename == table.name ) if not table_already_dirty: dbsession.execute( DirtyTable.__table__.insert() .values(device_id=ts.device_id, tablename=table.name) )
[docs]def get_dirty_tables(req: CamcopsRequest) -> List[Table]: """ Returns tables marked as dirty for this device. """ query = ( select([DirtyTable.tablename]) .where(DirtyTable.device_id == req.tabletsession.device_id) ) tablenames = fetch_all_first_values(req.dbsession, query) return [CLIENT_TABLE_MAP[tn] for tn in tablenames]
[docs]def flag_deleted(req: CamcopsRequest, table: Table, pklist: Iterable[int]) -> None: """ Marks record(s) as deleted, specified by a list of server PKs. """ mark_table_dirty(req, table) # noinspection PyProtectedMember req.dbsession.execute( update(table) .where(table.c._pk.in_(pklist)) .values(_removal_pending=1, _successor_pk=None) )
[docs]def flag_all_records_deleted(req: CamcopsRequest, table: Table) -> None: """ Marks all records in a table as deleted (that are current and in the current era). """ mark_table_dirty(req, table) # noinspection PyProtectedMember req.dbsession.execute( update(table) .where(table.c._device_id == req.tabletsession.device_id) .where(table.c._current) .where(table.c._era == ERA_NOW) .values(_removal_pending=1, _successor_pk=None) )
[docs]def flag_deleted_where_clientpk_not(req: CamcopsRequest, table: Table, clientpk_name: str, clientpk_values: Sequence[Any]) -> None: """ Marks for deletion all current/current-era records for a device, defined by a list of client-side PK values. """ mark_table_dirty(req, table) # noinspection PyProtectedMember req.dbsession.execute( update(table) .where(table.c._device_id == req.tabletsession.device_id) .where(table.c._current) .where(table.c._era == ERA_NOW) .where(table.c[clientpk_name].notin_(clientpk_values)) .values(_removal_pending=1, _successor_pk=None) )
[docs]def flag_modified(req: CamcopsRequest, table: Table, pk: int, successor_pk: int) -> None: """ Marks a record as old, storing its successor's details. """ mark_table_dirty(req, table) # noinspection PyProtectedMember req.dbsession.execute( update(table) .where(table.c._pk == pk) .values(_removal_pending=1, _successor_pk=successor_pk) )
[docs]def flag_record_for_preservation(req: CamcopsRequest, table: Table, pk: int) -> None: """ Marks a record for preservation (moving off the tablet, changing its era details). """ # noinspection PyProtectedMember req.dbsession.execute( update(table) .where(table.c._pk == pk) .values(_move_off_tablet=1) )
[docs]def commit_all(req: CamcopsRequest, batchtime: Pendulum, preserving: bool) -> None: """ Commits additions, removals, and preservations for all tables. """ tables = get_dirty_tables(req) auditsegments = [] for table in tables: n_added, n_removed, n_preserved = commit_table( req, batchtime, preserving, table, clear_dirty=False) auditsegments.append( "{tablename} ({n_added},{n_removed},{n_preserved})".format( tablename=table.name, n_added=n_added, n_removed=n_removed, n_preserved=n_preserved, ) ) clear_dirty_tables(req) details = "Upload [table (n_added,n_removed,n_preserved)]: {}".format( ", ".join(auditsegments) ) audit(req, details)
[docs]def commit_table(req: CamcopsRequest, batchtime: Pendulum, preserving: bool, table: Table, clear_dirty: bool = True) -> Tuple[int, int, int]: """ Commits additions, removals, and preservations for one table. """ # log.debug("commit_table: {}", table.name) user_id = req.user_id device_id = req.tabletsession.device_id exacttime = req.now # Additions # noinspection PyProtectedMember addition_rp = req.dbsession.execute( update(table) .where(table.c._device_id == device_id) .where(table.c._addition_pending) .values(_current=1, _addition_pending=0, _adding_user_id=user_id, _when_added_exact=exacttime, _when_added_batch_utc=batchtime) ) # type: ResultProxy n_added = addition_rp.rowcount # Removals # noinspection PyProtectedMember removal_rp = req.dbsession.execute( update(table) .where(table.c._device_id == device_id) .where(table.c._removal_pending) .values(_current=0, _removal_pending=0, _removing_user_id=user_id, _when_removed_exact=exacttime, _when_removed_batch_utc=batchtime) ) # type: ResultProxy n_removed = removal_rp.rowcount # Preservation new_era = format_datetime(batchtime, DateFormat.ERA) if preserving: # Preserve all relevant records # noinspection PyProtectedMember preserve_rp = req.dbsession.execute( update(table) .where(table.c._device_id == device_id) .where(table.c._era == ERA_NOW) .values(_era=new_era, _preserving_user_id=user_id, _move_off_tablet=0) ) # type: ResultProxy n_preserved = preserve_rp.rowcount # Also preserve/finalize any corresponding special notes (2015-02-01) req.dbsession.execute( update(SpecialNote.__table__) .where(SpecialNote.basetable == table.name) .where(SpecialNote.device_id == device_id) .where(SpecialNote.era == ERA_NOW) .values(era=new_era) ) else: # Preserve any individual records # noinspection PyProtectedMember preserve_rp = req.dbsession.execute( update(table) .where(table.c._device_id == device_id) .where(table.c._move_off_tablet) .values(_era=new_era, _preserving_user_id=user_id, _move_off_tablet=0) ) # type: ResultProxy n_preserved = preserve_rp.rowcount # Also preserve/finalize any corresponding special notes (2015-02-01) # noinspection PyProtectedMember req.dbsession.execute( update(SpecialNote.__table__) .where(SpecialNote.basetable == table.name) .where(SpecialNote.device_id == device_id) .where(SpecialNote.era == ERA_NOW) .where(exists().select_from(table) .where(table.c.id == SpecialNote.task_id) .where(table.c._device_id == SpecialNote.device_id) .where(table.c._era == new_era)) .values(era=new_era) ) # Remove individually from list of dirty tables? if clear_dirty: req.dbsession.execute( DirtyTable.__table__.delete() .where(DirtyTable.device_id == device_id) .where(DirtyTable.tablename == table.name) ) # ... otherwise a call to clear_dirty_tables() must be made. return n_added, n_removed, n_preserved
[docs]def rollback_all(req: CamcopsRequest) -> None: """ Rolls back all pending changes for a device. """ tables = get_dirty_tables(req) for table in tables: rollback_table(req, table) clear_dirty_tables(req)
[docs]def rollback_table(req: CamcopsRequest, table: Table) -> None: """ Rolls back changes for an individual table for a device. """ device_id = req.tabletsession.device_id # Pending additions # noinspection PyProtectedMember req.dbsession.execute( table.delete() .where(table.c._device_id == device_id) .where(table.c._addition_pending) ) # Pending deletions # noinspection PyProtectedMember req.dbsession.execute( update(table) .where(table.c._device_id == device_id) .where(table.c._removal_pending) .values(_removal_pending=0, _when_removed_exact=None, _when_removed_batch_utc=None, _removing_user_id=None, _successor_pk=None) ) # Record-specific preservation (set by flag_record_for_preservation()) # noinspection PyProtectedMember req.dbsession.execute( update(table) .where(table.c._device_id == device_id) .values(_move_off_tablet=0) )
[docs]def clear_dirty_tables(req: CamcopsRequest) -> None: """ Clears the dirty-table list for a device. """ req.dbsession.execute( DirtyTable.__table__.delete() .where(DirtyTable.device_id == req.tabletsession.device_id) )
# ============================================================================= # Audit functions # =============================================================================
[docs]def audit(req: CamcopsRequest, details: str, patient_server_pk: int = None, tablename: str = None, server_pk: int = None) -> None: """ Audit something. """ # Add parameters and pass on: cc_audit.audit( req=req, details=details, patient_server_pk=patient_server_pk, table=tablename, server_pk=server_pk, device_id=req.tabletsession.device_id, # added remote_addr=req.remote_addr, # added user_id=req.user_id, # added from_console=False, # added from_dbclient=True # added )
# ============================================================================= # Action processors: allowed to any user # ============================================================================= # If they return None, the framework uses the operation name as the reply in # the success message. Not returning anything is the same as returning None. # Authentication is performed in advance of these.
[docs]def check_device_registered(req: CamcopsRequest) -> None: """ Check that a device is registered, or raise UserErrorException. """ req.tabletsession.ensure_device_registered()
# ============================================================================= # Action processors that require REGISTRATION privilege # =============================================================================
[docs]def register(req: CamcopsRequest) -> Dict[str, Any]: """ Register a device with the server. """ dbsession = req.dbsession ts = req.tabletsession device_friendly_name = get_str_var(req, TabletParam.DEVICE_FRIENDLY_NAME, mandatory=False) device_exists = exists_in_table( dbsession, Device.__table__, Device.name == ts.device_name ) if device_exists: # device already registered, but accept re-registration dbsession.execute( update(Device.__table__) .where(Device.name == ts.device_name) .values(friendly_name=device_friendly_name, camcops_version=ts.tablet_version_str, registered_by_user_id=req.user_id, when_registered_utc=req.now_utc) ) else: # new registration try: dbsession.execute( Device.__table__.insert() .values(name=ts.device_name, friendly_name=device_friendly_name, camcops_version=ts.tablet_version_str, registered_by_user_id=req.user_id, when_registered_utc=req.now_utc) ) except IntegrityError: fail_user_error(INSERT_FAILED) ts.reload_device() audit( req, "register, device_id={}, friendly_name={}".format( ts.device_id, device_friendly_name), tablename=Device.__tablename__ ) return get_server_id_info(req)
[docs]def get_extra_strings(req: CamcopsRequest) -> Dict[str, str]: """ Fetch all local extra strings from the server. """ fields = [ExtraStringFieldNames.TASK, ExtraStringFieldNames.NAME, ExtraStringFieldNames.VALUE] rows = req.get_all_extra_strings() reply = get_select_reply(fields, rows) audit(req, "get_extra_strings") return reply
# noinspection PyUnusedLocal
[docs]def get_allowed_tables(req: CamcopsRequest) -> Dict[str, str]: """ Returns the names of all possible tables on the server, each paired with the minimum client (tablet) version that will be accepted for each table. (Typically these are all the same as the minimum global tablet version.) Uses the SELECT-like syntax. """ tables_versions = all_tables_with_min_client_version() fields = [AllowedTablesFieldNames.TABLENAME, AllowedTablesFieldNames.MIN_CLIENT_VERSION] rows = [[k, str(v)] for k, v in tables_versions.items()] reply = get_select_reply(fields, rows) audit(req, "get_allowed_tables") return reply
# ============================================================================= # Action processors that require UPLOAD privilege # ============================================================================= # noinspection PyUnusedLocal
[docs]def check_upload_user_and_device(req: CamcopsRequest) -> None: """ Stub function for the operation to check that a user is valid. """ pass # don't need to do anything!
# noinspection PyUnusedLocal
[docs]def get_id_info(req: CamcopsRequest) -> Dict: """ Fetch server ID information. """ return get_server_id_info(req)
[docs]def start_upload(req: CamcopsRequest) -> None: """ Begin an upload. """ start_device_upload_batch(req)
[docs]def end_upload(req: CamcopsRequest) -> None: """ Ends an upload and commits changes. """ batchtime, preserving = get_batch_details_start_if_needed(req) # ensure it's the same user finishing as starting! end_device_upload_batch(req, batchtime, preserving)
[docs]def upload_table(req: CamcopsRequest) -> str: """ Upload a table. Incoming information in the POST request includes a CSV list of fields, a count of the number of records being provided, and a set of variables named record0 ... record{nrecords}, each containing a CSV list of SQL-encoded values. Typically used for smaller tables, i.e. most except for BLOBs. """ table = get_table_from_req(req, TabletParam.TABLE) fields = get_fields_from_post_var(req, table, TabletParam.FIELDS) nrecords = get_int_var(req, TabletParam.NRECORDS) nfields = len(fields) if nfields < 1: fail_user_error("{}={}: can't be less than 1".format( TabletParam.FIELDS, nfields)) if nrecords < 0: fail_user_error("{}={}: can't be less than 0".format( TabletParam.NRECORDS, nrecords)) _, _ = get_batch_details_start_if_needed(req) ts = req.tabletsession if ts.explicit_pkname_for_upload_table: # q.v. # New client: tells us the PK name explicitly. clientpk_name = get_single_field_from_post_var(req, table, TabletParam.PKNAME) else: # Old client. Either (a) old Titanium client, in which the client PK # is in fields[0], or (b) an early C++ client, in which there was no # guaranteed order (and no explicit PK name was sent). However, in # either case, the client PK name was (is) always "id". clientpk_name = TABLET_ID_FIELD ensure_valid_field_name(table, clientpk_name) server_pks_uploaded = [] n_new = 0 n_modified = 0 n_identical = 0 server_active_record_pks = get_server_pks_of_active_records(req, table) mark_table_dirty(req, table) for r in range(nrecords): recname = TabletParam.RECORD_PREFIX + str(r) values = get_values_from_post_var(req, recname) nvalues = len(values) if nvalues != nfields: errmsg = ( "Number of fields in field list ({nfields}) doesn't match " "number of values in record {r} ({nvalues})".format( nfields=nfields, r=r, nvalues=nvalues) ) log.warning(errmsg + "\nfields: {!r}\n" "values: {!r}", fields, values) fail_user_error(errmsg) valuedict = dict(zip(fields, values)) # log.critical("table {!r}, record {}: {!r}", table.name, r, valuedict) # CORE: CALLS upload_record_core oldserverpk, newserverpk = upload_record_core( req, table, clientpk_name, valuedict, r) if oldserverpk is not None: server_pks_uploaded.append(oldserverpk) if newserverpk is None: n_identical += 1 else: n_modified += 1 else: n_new += 1 # Now deal with any ABSENT (not in uploaded data set) conditions. server_pks_for_deletion = [x for x in server_active_record_pks if x not in server_pks_uploaded] n_deleted = len(server_pks_for_deletion) if n_deleted > 0: flag_deleted(req, table, server_pks_for_deletion) # Special for old tablets: if req.tabletsession.cope_with_old_idnums and table == Patient.__table__: mark_table_dirty(req, PatientIdNum.__table__) # Mark patient ID numbers for deletion if their parent Patient is # similarly being marked for deletion # noinspection PyProtectedMember req.dbsession.execute( update(PatientIdNum.__table__) .where(PatientIdNum._device_id == Patient._device_id) .where(PatientIdNum._era == ERA_NOW) .where(PatientIdNum.patient_id == Patient.id) .where(Patient._pk.in_(server_pks_for_deletion)) .where(Patient._era == ERA_NOW) # shouldn't be in doubt! .values(_removal_pending=1, _successor_pk=None) ) # Success # log.debug( # "Table {t}: server_active_record_pks: {a}; server_pks_uploaded: {u}; " # "server_pks_for_deletion: {d}; " # "number of missing records (deleted): {nd}".format( # t=table.name, # a=server_active_record_pks, # u=server_pks_uploaded, # d=server_pks_for_deletion, # nd=n_deleted # ) # ) # Auditing occurs at commit_all. log.info("Upload successful; {n} records uploaded to table {t} " "({new} new, {mod} modified, {i} identical, {nd} deleted)", n=nrecords, t=table.name, new=n_new, mod=n_modified, i=n_identical, nd=n_deleted) return "Table {} upload successful".format(table.name)
[docs]def upload_record(req: CamcopsRequest) -> str: """ Upload an individual record. (Typically used for BLOBs.) Incoming POST information includes a CSV list of fields and a CSV list of values. """ table = get_table_from_req(req, TabletParam.TABLE) clientpk_name = get_single_field_from_post_var(req, table, TabletParam.PKNAME) valuedict = get_fields_and_values(req, table, TabletParam.FIELDS, TabletParam.VALUES) require_keys(valuedict, [clientpk_name, CLIENT_DATE_FIELD]) clientpk_value = valuedict[clientpk_name] wheredict = {clientpk_name: clientpk_value} serverpks = get_server_pks_of_specified_records(req, table, wheredict) if not serverpks: # Insert insert_record(req, table, valuedict, None) log.info("upload-insert") return "UPLOAD-INSERT" else: # Update oldserverpk = serverpks[0] client_date_value = valuedict[CLIENT_DATE_FIELD] rec_exists = record_identical_by_date(req, table, oldserverpk, client_date_value) if rec_exists: log.info("upload-update: skipping existing record") else: newserverpk = duplicate_record(req, table, oldserverpk) flag_modified(req, table, oldserverpk, newserverpk) update_new_copy_of_record(req, table, newserverpk, valuedict, oldserverpk) log.info("upload-update") return "UPLOAD-UPDATE"
# Auditing occurs at commit_all.
[docs]def upload_empty_tables(req: CamcopsRequest) -> str: """ The tablet supplies a list of tables that are empty at its end, and we will 'wipe' all appropriate tables; this reduces the number of HTTP requests. """ tables = get_tables_from_post_var(req, TabletParam.TABLES) _, _ = get_batch_details_start_if_needed(req) for table in tables: flag_all_records_deleted(req, table) log.info("upload_empty_tables") # Auditing occurs at commit_all. return "UPLOAD-EMPTY-TABLES"
[docs]def start_preservation(req: CamcopsRequest) -> str: """ Marks this upload batch as one in which all records will be preserved (i.e. moved from NOW-era to an older era, so they can be deleted safely from the tablet). Without this, individual records can still be marked for preservation if their MOVE_OFF_TABLET_FIELD field (_move_off_tablet) is set; see upload_record and its functions. """ _, _ = get_batch_details_start_if_needed(req) start_preserving(req) log.info("start_preservation successful") # Auditing occurs at commit_all. return "STARTPRESERVATION"
[docs]def delete_where_key_not(req: CamcopsRequest) -> str: """ Marks records for deletion, for a device/table, where the client PK is not in a specified list. """ table = get_table_from_req(req, TabletParam.TABLE) clientpk_name = get_single_field_from_post_var( req, table, TabletParam.PKNAME) clientpk_values = get_values_from_post_var(req, TabletParam.PKVALUES) _, _ = get_batch_details_start_if_needed(req) flag_deleted_where_clientpk_not(req, table, clientpk_name, clientpk_values) # Auditing occurs at commit_all. # log.info("delete_where_key_not successful; table {} trimmed", table) return "Trimmed"
[docs]def which_keys_to_send(req: CamcopsRequest) -> str: """ Intended use: "For my device, and a specified table, here are my client- side PKs (as a CSV list), and the modification dates for each corresponding record (as a CSV list). Please tell me which records have mismatching dates on the server, i.e. those that I need to re-upload." Used particularly for BLOBs, to reduce traffic, i.e. so we don't have to send a lot of BLOBs. """ try: table = get_table_from_req(req, TabletParam.TABLE) except IgnoringAntiqueTableException: raise IgnoringAntiqueTableException("") clientpk_name = get_single_field_from_post_var(req, table, TabletParam.PKNAME) clientpk_values = get_values_from_post_var(req, TabletParam.PKVALUES, mandatory=False) # ... should be autoconverted to int, but we check below client_dates = get_values_from_post_var(req, TabletParam.DATEVALUES, mandatory=False) # ... will be in string format npkvalues = len(clientpk_values) ndatevalues = len(client_dates) if npkvalues != ndatevalues: fail_user_error( "Number of PK values ({npk}) doesn't match number of dates " "({nd})".format(npk=npkvalues, nd=ndatevalues)) client_pk_to_datetime = {} # type: Dict[int, Pendulum] for i in range(npkvalues): cpkv = clientpk_values[i] if not isinstance(cpkv, int): fail_user_error("Bad (non-integer) client PK: {!r}".format(cpkv)) try: dt = coerce_to_pendulum(client_dates[i]) if dt is None: fail_user_error("Missing date/time for client " "PK {}".format(cpkv)) client_pk_to_datetime[cpkv] = dt except ValueError: fail_user_error("Bad date/time: {!r}".format(client_dates[i])) _, _ = get_batch_details_start_if_needed(req) # 1. The client sends us all its PKs. So "delete" anything not in that # list. flag_deleted_where_clientpk_not(req, table, clientpk_name, clientpk_values) # 2. See which ones are new or updates. pks_needed = [] client_pks_on_server_to_spk = client_pks_that_exist( req, table, clientpk_name, clientpk_values) for clientpkval in clientpk_values: if clientpkval not in client_pks_on_server_to_spk: pks_needed.append(clientpkval) else: serverpk = client_pks_on_server_to_spk[clientpkval] when = client_pk_to_datetime[clientpkval] if not record_identical_by_date(req, table, serverpk, when): pks_needed.append(clientpkval) # Success pk_csv_list = ",".join([str(x) for x in pks_needed if x is not None]) # log.info("which_keys_to_send successful: table {}", table.name) return pk_csv_list
# ============================================================================= # Action maps # ============================================================================= class Operations: CHECK_DEVICE_REGISTERED = "check_device_registered" CHECK_UPLOAD_USER_DEVICE = "check_upload_user_and_device" DELETE_WHERE_KEY_NOT = "delete_where_key_not" END_UPLOAD = "end_upload" GET_EXTRA_STRINGS = "get_extra_strings" GET_ID_INFO = "get_id_info" GET_ALLOWED_TABLES = "get_allowed_tables" # v2.2.0 REGISTER = "register" START_PRESERVATION = "start_preservation" START_UPLOAD = "start_upload" UPLOAD_TABLE = "upload_table" UPLOAD_RECORD = "upload_record" UPLOAD_EMPTY_TABLES = "upload_empty_tables" WHICH_KEYS_TO_SEND = "which_keys_to_send" OPERATIONS_ANYONE = { Operations.CHECK_DEVICE_REGISTERED: check_device_registered, } OPERATIONS_REGISTRATION = { Operations.REGISTER: register, Operations.GET_EXTRA_STRINGS: get_extra_strings, Operations.GET_ALLOWED_TABLES: get_allowed_tables, # v2.2.0 # noqa } OPERATIONS_UPLOAD = { Operations.CHECK_UPLOAD_USER_DEVICE: check_upload_user_and_device, Operations.GET_ID_INFO: get_id_info, Operations.START_UPLOAD: start_upload, Operations.END_UPLOAD: end_upload, Operations.UPLOAD_TABLE: upload_table, Operations.UPLOAD_RECORD: upload_record, Operations.UPLOAD_EMPTY_TABLES: upload_empty_tables, Operations.START_PRESERVATION: start_preservation, Operations.DELETE_WHERE_KEY_NOT: delete_where_key_not, Operations.WHICH_KEYS_TO_SEND: which_keys_to_send, } # ============================================================================= # Client API main functions # =============================================================================
[docs]def main_client_api(req: CamcopsRequest) -> Dict[str, str]: """ Main HTTP processor. For success, returns a dictionary to send (will use status '200 OK') For failure, raises an exception. """ # log.info("CamCOPS database script starting at {}", # format_datetime(req.now, DateFormat.ISO8601)) ts = req.tabletsession fn = None if ts.operation in OPERATIONS_ANYONE: fn = OPERATIONS_ANYONE.get(ts.operation) if ts.operation in OPERATIONS_REGISTRATION: ts.ensure_valid_user_for_device_registration() fn = OPERATIONS_REGISTRATION.get(ts.operation) if ts.operation in OPERATIONS_UPLOAD: ts.ensure_valid_device_and_user_for_uploading() fn = OPERATIONS_UPLOAD.get(ts.operation) if not fn: fail_unsupported_operation(ts.operation) result = fn(req) if result is None: result = {TabletParam.RESULT: ts.operation} elif not isinstance(result, dict): result = {TabletParam.RESULT: result} return result
[docs]@view_config(route_name=Routes.CLIENT_API, permission=NO_PERMISSION_REQUIRED) def client_api(req: CamcopsRequest) -> Response: """ View for client API. All tablet interaction comes through here. Wraps main_client_api(). """ # log.critical("{!r}", req.environ) # log.critical("{!r}", req.params) t0 = time.time() # in seconds try: resultdict = main_client_api(req) resultdict[TabletParam.SUCCESS] = SUCCESS_CODE status = '200 OK' except IgnoringAntiqueTableException as e: log.warning(IGNORING_ANTIQUE_TABLE_MESSAGE) resultdict = { TabletParam.RESULT: escape_newlines(str(e)), TabletParam.SUCCESS: SUCCESS_CODE, } status = '200 OK' except UserErrorException as e: log.warning("CLIENT-SIDE SCRIPT ERROR: {}", e) resultdict = { TabletParam.SUCCESS: FAILURE_CODE, TabletParam.ERROR: escape_newlines(str(e)) } status = '200 OK' except ServerErrorException as e: log.error("SERVER-SIDE SCRIPT ERROR: {}", e) # rollback? Not sure resultdict = { TabletParam.SUCCESS: FAILURE_CODE, TabletParam.ERROR: escape_newlines(str(e)) } status = "503 Database Unavailable: " + str(e) except Exception as e: # All other exceptions. May include database write failures. # Let's return with status '200 OK'; though this seems dumb, it means # the tablet user will at least see the message. log.exception("Unhandled exception") # + traceback.format_exc() resultdict = { TabletParam.SUCCESS: FAILURE_CODE, TabletParam.ERROR: escape_newlines(exception_description(e)) } status = '200 OK' # Add session token information ts = req.tabletsession resultdict[TabletParam.SESSION_ID] = ts.session_id resultdict[TabletParam.SESSION_TOKEN] = ts.session_token # Convert dictionary to text in name-value pair format txt = "".join("{}:{}\n".format(k, v) for k, v in resultdict.items()) t1 = time.time() log.debug("Time in script (s): {t}", t=t1 - t0) return TextResponse(txt, status=status)
# ============================================================================= # Unit tests # ============================================================================= def get_reply_dict_from_response(response: Response) -> Dict[str, str]: txt = str(response) d = {} # type: Dict[str, str] # Format is: "200 OK\r\n<other headers>\r\n\r\n<content>" # There's a blank line between the heads and the body. http_gap = "\r\n\r\n" camcops_linesplit = "\n" camcops_k_v_sep = ":" try: start_of_content = txt.index(http_gap) + len(http_gap) txt = txt[start_of_content:] for line in txt.split(camcops_linesplit): if not line: continue colon_pos = line.index(camcops_k_v_sep) key = line[:colon_pos] value = line[colon_pos + len(camcops_k_v_sep):] key = key.strip() value = value.strip() d[key] = value return d except ValueError: return {}
[docs]class ClientApiTests(DemoDatabaseTestCase): def test_client_api_basics(self) -> None: self.announce("test_client_api_basics") with self.assertRaises(UserErrorException): fail_user_error("testmsg") with self.assertRaises(ServerErrorException): fail_server_error("testmsg") with self.assertRaises(UserErrorException): fail_unsupported_operation("duffop") # Encoding/decoding tests # data = bytearray("hello") data = b"hello" enc_b64data = base64_64format_encode(data) enc_hexdata = hex_xformat_encode(data) not_enc_1 = "X'012345'" not_enc_2 = "64'aGVsbG8='" teststring = """one, two, 3, 4.5, NULL, 'hello "hi with linebreak"', 'NULL', 'quote''s here', {b}, {h}, {s1}, {s2}""" sql_csv_testdict = { teststring.format( b=enc_b64data, h=enc_hexdata, s1=sql_quote_string(not_enc_1), s2=sql_quote_string(not_enc_2), ): [ "one", "two", 3, 4.5, None, 'hello "hi\n with linebreak"', "NULL", "quote's here", data, data, not_enc_1, not_enc_2, ], "": [], } for k, v in sql_csv_testdict.items(): r = decode_values(k) self.assertEqual(r, v, "Mismatch! Result: {r!s}\n" "Should have been: {v!s}\n" "Key was: {k!s}".format(r=r, v=v, k=k)) # Newline encoding/decodine ts2 = "slash \\ newline \n ctrl_r \r special \\n other special \\r " \ "quote ' doublequote \" " self.assertEqual(unescape_newlines(escape_newlines(ts2)), ts2, "Bug in escape_newlines() or unescape_newlines()") # TODO: more tests here... ? def test_client_api_antique_support_1(self): self.announce("test_client_api_antique_support_1") self.req.fake_request_post_from_dict({ TabletParam.CAMCOPS_VERSION: MINIMUM_TABLET_VERSION, TabletParam.DEVICE: self.other_device.name, TabletParam.OPERATION: Operations.WHICH_KEYS_TO_SEND, TabletParam.TABLE: DEVICE_STORED_VAR_TABLENAME_DEFUNCT, }) response = client_api(self.req) d = get_reply_dict_from_response(response) assert d[TabletParam.SUCCESS] == SUCCESS_CODE def test_client_api_antique_support_2(self): self.announce("test_client_api_antique_support_2") self.req.fake_request_post_from_dict({ TabletParam.CAMCOPS_VERSION: MINIMUM_TABLET_VERSION, TabletParam.DEVICE: self.other_device.name, TabletParam.OPERATION: Operations.WHICH_KEYS_TO_SEND, TabletParam.TABLE: "nonexistent_table", }) response = client_api(self.req) d = get_reply_dict_from_response(response) assert d[TabletParam.SUCCESS] == FAILURE_CODE def test_client_api_antique_support_3(self): self.announce("test_client_api_antique_support_3") self.req.fake_request_post_from_dict({ TabletParam.CAMCOPS_VERSION: MINIMUM_TABLET_VERSION, TabletParam.DEVICE: self.other_device.name, TabletParam.OPERATION: Operations.UPLOAD_TABLE, TabletParam.TABLE: DEVICE_STORED_VAR_TABLENAME_DEFUNCT, }) response = client_api(self.req) d = get_reply_dict_from_response(response) assert d[TabletParam.SUCCESS] == SUCCESS_CODE
# ============================================================================= # main # ============================================================================= # run with "python -m camcops_server.cc_modules.client_api -v" to be verbose if __name__ == "__main__": main_only_quicksetup_rootlogger(level=logging.DEBUG) unittest.main()