Source code for camcops_server.cc_modules.cc_db

#!/usr/bin/env python
# camcops_server/cc_modules/cc_db.py

"""
===============================================================================

    Copyright (C) 2012-2018 Rudolf Cardinal (rudolf@pobox.com).

    This file is part of CamCOPS.

    CamCOPS is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    CamCOPS is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with CamCOPS. If not, see <http://www.gnu.org/licenses/>.

===============================================================================
"""

from collections import OrderedDict
import logging
from typing import (Any, Dict, Generator, List, Optional, Set, Tuple, Type,
                    TYPE_CHECKING, TypeVar, Union)

from cardinal_pythonlib.logs import BraceStyleAdapter
from cardinal_pythonlib.sqlalchemy.orm_inspect import gen_columns
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.orm import relationship
from sqlalchemy.orm.relationships import RelationshipProperty
from sqlalchemy.orm import Session as SqlASession
from sqlalchemy.sql.schema import Column, ForeignKey
from sqlalchemy.sql.sqltypes import Boolean, DateTime, Integer

from .cc_constants import ERA_NOW
from .cc_sqla_coltypes import (
    CamcopsColumn,
    EraColType,
    gen_ancillary_relationships,
    gen_camcops_blob_columns,
    PendulumDateTimeAsIsoTextColType,
    PermittedValueChecker,
    RelationshipInfo,
    SemanticVersionColType,
)
from .cc_summaryelement import SummaryElement
from .cc_tsv import TsvPage
from .cc_version import CAMCOPS_SERVER_VERSION
from .cc_xml import (
    make_xml_branches_from_blobs,
    make_xml_branches_from_columns,
    make_xml_branches_from_summaries,
    XML_COMMENT_STORED,
    XML_COMMENT_CALCULATED,
    XmlElement,
)

if TYPE_CHECKING:
    from .cc_blob import Blob
    from .cc_request import CamcopsRequest

log = BraceStyleAdapter(logging.getLogger(__name__))

T = TypeVar('T')


# =============================================================================
# Base classes implementing common fields
# =============================================================================

# noinspection PyAttributeOutsideInit
[docs]class GenericTabletRecordMixin(object): """ From the server's perspective, _pk is unique. However, records are defined also in their tablet context, for which an individual tablet (defined by the combination of _device_id and _era) sees its own PK, "id". """ __tablename__ = None # type: str # sorts out some mixin type checking # ------------------------------------------------------------------------- # On the server side: # ------------------------------------------------------------------------- # Plain columns # noinspection PyMethodParameters @declared_attr def _pk(cls) -> Column: return Column( "_pk", Integer, primary_key=True, autoincrement=True, index=True, comment="(SERVER) Primary key (on the server)" ) # noinspection PyMethodParameters @declared_attr def _device_id(cls) -> Column: return Column( "_device_id", Integer, ForeignKey("_security_devices.id"), nullable=False, index=True, comment="(SERVER) ID of the source tablet device" ) # noinspection PyMethodParameters @declared_attr def _era(cls) -> Column: return Column( "_era", EraColType, nullable=False, index=True, comment="(SERVER) 'NOW', or when this row was preserved and " "removed from the source device (UTC ISO 8601)", ) # ... note that _era is textual so that plain comparison # with "=" always works, i.e. no NULLs -- for USER comparison too, not # just in CamCOPS code # noinspection PyMethodParameters @declared_attr def _current(cls) -> Column: return Column( "_current", Boolean, nullable=False, index=True, comment="(SERVER) Is the row current (1) or not (0)?" ) # noinspection PyMethodParameters @declared_attr def _when_added_exact(cls) -> Column: return Column( "_when_added_exact", PendulumDateTimeAsIsoTextColType, comment="(SERVER) Date/time this row was added (ISO 8601)" ) # noinspection PyMethodParameters @declared_attr def _when_added_batch_utc(cls) -> Column: return Column( "_when_added_batch_utc", DateTime, comment="(SERVER) Date/time of the upload batch that added this " "row (DATETIME in UTC)" ) # noinspection PyMethodParameters @declared_attr def _adding_user_id(cls) -> Column: return Column( "_adding_user_id", Integer, ForeignKey("_security_users.id"), comment="(SERVER) ID of user that added this row", ) # noinspection PyMethodParameters @declared_attr def _when_removed_exact(cls) -> Column: return Column( "_when_removed_exact", PendulumDateTimeAsIsoTextColType, comment="(SERVER) Date/time this row was removed, i.e. made " "not current (ISO 8601)" ) # noinspection PyMethodParameters @declared_attr def _when_removed_batch_utc(cls) -> Column: return Column( "_when_removed_batch_utc", DateTime, comment="(SERVER) Date/time of the upload batch that removed " "this row (DATETIME in UTC)" ) # noinspection PyMethodParameters @declared_attr def _removing_user_id(cls) -> Column: return Column( "_removing_user_id", Integer, ForeignKey("_security_users.id"), comment="(SERVER) ID of user that removed this row" ) # noinspection PyMethodParameters @declared_attr def _preserving_user_id(cls) -> Column: return Column( "_preserving_user_id", Integer, ForeignKey("_security_users.id"), comment="(SERVER) ID of user that preserved this row" ) # noinspection PyMethodParameters @declared_attr def _forcibly_preserved(cls) -> Column: return Column( "_forcibly_preserved", Boolean, default=False, comment="(SERVER) Forcibly preserved by superuser (rather than " "normally preserved by tablet)?" ) # noinspection PyMethodParameters @declared_attr def _predecessor_pk(cls) -> Column: return Column( "_predecessor_pk", Integer, comment="(SERVER) PK of predecessor record, prior to modification" ) # noinspection PyMethodParameters @declared_attr def _successor_pk(cls) -> Column: return Column( "_successor_pk", Integer, comment="(SERVER) PK of successor record (after modification) " "or NULL (whilst live, or after deletion)" ) # noinspection PyMethodParameters @declared_attr def _manually_erased(cls) -> Column: return Column( "_manually_erased", Boolean, default=False, comment="(SERVER) Record manually erased (content destroyed)?" ) # noinspection PyMethodParameters @declared_attr def _manually_erased_at(cls) -> Column: return Column( "_manually_erased_at", PendulumDateTimeAsIsoTextColType, comment="(SERVER) Date/time of manual erasure (ISO 8601)" ) # noinspection PyMethodParameters @declared_attr def _manually_erasing_user_id(cls) -> Column: return Column( "_manually_erasing_user_id", Integer, ForeignKey("_security_users.id"), comment="(SERVER) ID of user that erased this row manually" ) # noinspection PyMethodParameters @declared_attr def _camcops_version(cls) -> Column: return Column( "_camcops_version", SemanticVersionColType, default=CAMCOPS_SERVER_VERSION, comment="(SERVER) CamCOPS version number of the uploading device" ) # noinspection PyMethodParameters @declared_attr def _addition_pending(cls) -> Column: return Column( "_addition_pending", Boolean, nullable=False, default=False, comment="(SERVER) Addition pending?" ) # noinspection PyMethodParameters @declared_attr def _removal_pending(cls) -> Column: return Column( "_removal_pending", Boolean, default=False, comment="(SERVER) Removal pending?" ) # noinspection PyMethodParameters @declared_attr def _group_id(cls) -> Column: return Column( "_group_id", Integer, ForeignKey("_security_groups.id"), nullable=False, index=True, comment="(SERVER) ID of group to which this record belongs" ) RESERVED_FIELDS = [ # fields that tablets can't upload "_pk", "_device_id", "_era", "_current", "_when_added_exact", "_when_added_batch_utc", "_adding_user_id", "_when_removed_exact", "_when_removed_batch_utc", "_removing_user_id", "_preserving_user_id", "_forcibly_preserved", "_predecessor_pk", "_successor_pk", "_manually_erased", "_manually_erased_at", "_manually_erasing_user_id", "_camcops_version", "_addition_pending", "_removal_pending", "_group_id", ] # but more generally: they start with "_"... # ------------------------------------------------------------------------- # Fields that *all* client tables have: # ------------------------------------------------------------------------- # noinspection PyMethodParameters @declared_attr def id(cls) -> Column: return Column( "id", Integer, nullable=False, index=True, comment="(TASK) Primary key (task ID) on the tablet device" ) # noinspection PyMethodParameters @declared_attr def when_last_modified(cls) -> Column: return Column( "when_last_modified", PendulumDateTimeAsIsoTextColType, index=True, # ... as used by database upload script comment="(STANDARD) Date/time this row was last modified on the " "source tablet device (ISO 8601)" ) # noinspection PyMethodParameters @declared_attr def _move_off_tablet(cls) -> Column: return Column( "_move_off_tablet", Boolean, default=False, comment="(SERVER/TABLET) Record-specific preservation pending?" ) # ------------------------------------------------------------------------- # Relationships # ------------------------------------------------------------------------- # noinspection PyMethodParameters @declared_attr def _device(cls) -> RelationshipProperty: return relationship("Device") # noinspection PyMethodParameters @declared_attr def _adding_user(cls) -> RelationshipProperty: return relationship("User", foreign_keys=[cls._adding_user_id]) # noinspection PyMethodParameters @declared_attr def _removing_user(cls) -> RelationshipProperty: return relationship("User", foreign_keys=[cls._removing_user_id]) # noinspection PyMethodParameters @declared_attr def _preserving_user(cls) -> RelationshipProperty: return relationship("User", foreign_keys=[cls._preserving_user_id]) # noinspection PyMethodParameters @declared_attr def _manually_erasing_user(cls) -> RelationshipProperty: return relationship("User", foreign_keys=[cls._manually_erasing_user_id]) # noinspection PyMethodParameters @declared_attr def _group(cls) -> RelationshipProperty: return relationship("Group", foreign_keys=[cls._group_id]) # ------------------------------------------------------------------------- # Fetching attributes # ------------------------------------------------------------------------- def get_pk(self) -> Optional[int]: return self._pk def get_era(self) -> Optional[str]: return self._era def get_device_id(self) -> Optional[int]: return self._device_id def get_group_id(self) -> Optional[int]: return self._group_id # ------------------------------------------------------------------------- # Autoscanning objects and their relationships # ------------------------------------------------------------------------- def _get_xml_root(self, req: "CamcopsRequest", skip_attrs: List[str] = None, include_plain_columns: bool=True, include_blobs: bool = True, include_calculated: bool = True, sort_by_attr: bool = True) -> XmlElement: """ Called to create an XML root object for records ancillary to Task objects. Tasks themselves use a more complex mechanism. """ skip_attrs = skip_attrs or [] # type: List[str] # "__tablename__" will make the type checker complain, as we're # defining a function for a mixin that assumes it's mixed in to a # SQLAlchemy Base-derived class # noinspection PyUnresolvedReferences return XmlElement( name=self.__tablename__, value=self._get_xml_branches( req=req, skip_attrs=skip_attrs, include_plain_columns=include_plain_columns, include_blobs=include_blobs, include_calculated=include_calculated, sort_by_attr=sort_by_attr ) ) def _get_xml_branches(self, req: "CamcopsRequest", skip_attrs: List[str], include_plain_columns: bool = True, include_blobs: bool = True, include_calculated: bool = True, sort_by_attr: bool = True) -> List[XmlElement]: """ Gets the values of SQLAlchemy columns as XmlElement objects. Optionally, find any SQLAlchemy relationships that are relationships to Blob objects, and include them too. Used by _get_xml_root above, but also by Tasks themselves. """ # log.critical("_get_xml_branches for {!r}", self) skip_attrs = skip_attrs or [] # type: List[str] stored_branches = [] # type: List[XmlElement] if include_plain_columns: stored_branches += make_xml_branches_from_columns( self, skip_fields=skip_attrs) if include_blobs: stored_branches += make_xml_branches_from_blobs( req, self, skip_fields=skip_attrs) if sort_by_attr: stored_branches.sort(key=lambda el: el.name) branches = [XML_COMMENT_STORED] + stored_branches # Calculated if include_calculated: branches.append(XML_COMMENT_CALCULATED) branches.extend(make_xml_branches_from_summaries( self.get_summaries(req), skip_fields=skip_attrs, sort_by_name=sort_by_attr )) # log.critical("... branches for {!r}: {!r}", self, branches) return branches def _get_core_tsv_page(self, req: "CamcopsRequest", heading_prefix: str = "") -> TsvPage: row = OrderedDict() for attrname, column in gen_columns(self): row[heading_prefix + attrname] = getattr(self, attrname) for s in self.get_summaries(req): row[heading_prefix + s.name] = s.value return TsvPage(name=self.__tablename__, rows=[row]) # ------------------------------------------------------------------------- # Erasing (overwriting data, not deleting the database records) # -------------------------------------------------------------------------
[docs] def manually_erase_with_dependants(self, req: "CamcopsRequest") -> None: """ Manually erases a standard record and marks it so erased. The object remains _current (if it was), as a placeholder, but its contents are wiped. WRITES TO DATABASE. """ if self._manually_erased or self._pk is None or self._era == ERA_NOW: # ... _manually_erased: don't do it twice # ... _pk: basic sanity check # ... _era: don't erase things that are current on the tablet return # 1. "Erase my dependants" for ancillary in self.gen_ancillary_instances_even_noncurrent(): ancillary.manually_erase_with_dependants(req) for blob in self.gen_blobs_even_noncurrent(): blob.manually_erase_with_dependants(req) # 2. "Erase me" erasure_attrs = [] # type: List[str] for attrname, column in gen_columns(self): if attrname.startswith("_"): # system field continue if not column.nullable: # this should cover FKs continue if column.foreign_keys: # ... but to be sure... continue erasure_attrs.append(attrname) for attrname in erasure_attrs: setattr(self, attrname, None) self._current = False self._manually_erased = True self._manually_erased_at = req.now self._manually_erasing_user_id = req.user_id
def delete_with_dependants(self, req: "CamcopsRequest") -> None: if self._pk is None: return # 1. "Delete my dependants" for ancillary in self.gen_ancillary_instances_even_noncurrent(): ancillary.delete_with_dependants(req) for blob in self.gen_blobs_even_noncurrent(): blob.delete_with_dependants(req) # 2. "Delete me" dbsession = SqlASession.object_session(self) dbsession.delete(self) def gen_attrname_ancillary_pairs(self) \ -> Generator[Tuple[str, "GenericTabletRecordMixin"], None, None]: for attrname, rel_prop, rel_cls in gen_ancillary_relationships(self): if rel_prop.uselist: ancillaries = getattr(self, attrname) # type: List[GenericTabletRecordMixin] # noqa else: ancillaries = [getattr(self, attrname)] # type: List[GenericTabletRecordMixin] # noqa for ancillary in ancillaries: if ancillary is None: continue yield attrname, ancillary def gen_ancillary_instances(self) -> Generator["GenericTabletRecordMixin", None, None]: for attrname, ancillary in self.gen_attrname_ancillary_pairs(): yield ancillary def gen_ancillary_instances_even_noncurrent(self) \ -> Generator["GenericTabletRecordMixin", None, None]: seen = set() # type: Set[GenericTabletRecordMixin] for ancillary in self.gen_ancillary_instances(): for lineage_member in ancillary.get_lineage(): if lineage_member in seen: continue seen.add(lineage_member) yield lineage_member def gen_blobs(self) -> Generator["Blob", None, None]: for id_attrname, column in gen_camcops_blob_columns(self): relationship_attr = column.blob_relationship_attr_name blob = getattr(self, relationship_attr) if blob is None: continue yield blob def gen_blobs_even_noncurrent(self) -> Generator["Blob", None, None]: seen = set() # type: Set["Blob"] for blob in self.gen_blobs(): if blob is None: continue for lineage_member in blob.get_lineage(): if lineage_member in seen: continue seen.add(lineage_member) yield lineage_member
[docs] def get_lineage(self) -> List["GenericTabletRecordMixin"]: """ Returns all records that are part of the same "lineage", that is: matching on id/device_id/era, but including both current and any historical non-current versions. """ dbsession = SqlASession.object_session(self) cls = self.__class__ q = dbsession.query(cls)\ .filter(cls.id == self.id)\ .filter(cls._device_id == self._device_id)\ .filter(cls._era == self._era) return list(q)
# ------------------------------------------------------------------------- # History functions for server-side editing # -------------------------------------------------------------------------
[docs] def set_predecessor(self, req: "CamcopsRequest", predecessor: "GenericTabletRecordMixin") -> None: """ Used for some unusual server-side manipulations (e.g. editing patient details). The "self" object replaces the predecessor, so "self" becomes current and refers back to "predecessor", while "predecessor" becomes non-current and refers forward to "self". """ assert predecessor._current # We become new and current, and refer to our predecessor self._device_id = predecessor._device_id self._era = predecessor._era self._current = True self._when_added_exact = req.now self._when_added_batch_utc = req.now_utc self._adding_user_id = req.user_id if self._era != ERA_NOW: self._preserving_user_id = req.user_id self._forcibly_preserved = True self._predecessor_pk = predecessor._pk self._camcops_version = predecessor._camcops_version self._group_id = predecessor._group_id # Make our predecessor refer to us if self._pk is None: req.dbsession.add(self) # ensure we have a PK, part 1 req.dbsession.flush() # ensure we have a PK, part 2 predecessor._set_successor(req, self)
def _set_successor(self, req: "CamcopsRequest", successor: "GenericTabletRecordMixin") -> None: """ See set_predecessor() above. """ assert successor._pk is not None self._current = False self._when_removed_exact = req.now self._when_removed_batch_utc = req.now_utc self._removing_user_id = req.user_id self._successor_pk = successor._pk
[docs] def mark_as_deleted(self, req: "CamcopsRequest") -> None: """ Ends the history chain and marks this record as non-current. """ if self._current: self._when_removed_exact = req.now self._when_removed_batch_utc = req.now_utc self._removing_user_id = req.user_id self._current = False
[docs] def create_fresh(self, req: "CamcopsRequest", device_id: int, era: str, group_id: int) -> None: """ Used to create a record from scratch. """ self._device_id = device_id self._era = era self._group_id = group_id self._current = True self._when_added_exact = req.now self._when_added_batch_utc = req.now_utc self._adding_user_id = req.user_id
# ------------------------------------------------------------------------- # Override this if you provide summaries # ------------------------------------------------------------------------- # noinspection PyMethodMayBeStatic
[docs] def get_summaries(self, req: "CamcopsRequest") -> List[SummaryElement]: """ Return a list of summary value objects, for this database object (not any dependent classes/tables). """ return [] # type: List[SummaryElement]
[docs] def get_summary_names(self, req: "CamcopsRequest") -> List[str]: """ Returns a list of summary field names. """ return [x.name for x in self.get_summaries(req)]
# ============================================================================= # Relationships # =============================================================================
[docs]def ancillary_relationship( parent_class_name: str, ancillary_class_name: str, ancillary_fk_to_parent_attr_name: str, ancillary_order_by_attr_name: str = None, read_only: bool = True) -> RelationshipProperty: """ Implements a one-to-many relationship, i.e. one parent to many ancillaries. """ parent_pk_attr_name = "id" # always return relationship( ancillary_class_name, primaryjoin=( "and_(" " remote({a}.{fk}) == foreign({p}.{pk}), " " remote({a}._device_id) == foreign({p}._device_id), " " remote({a}._era) == foreign({p}._era), " " remote({a}._current) == True " ")".format( a=ancillary_class_name, fk=ancillary_fk_to_parent_attr_name, p=parent_class_name, pk=parent_pk_attr_name, ) ), uselist=True, order_by="{a}.{f}".format(a=ancillary_class_name, f=ancillary_order_by_attr_name), viewonly=read_only, info={ RelationshipInfo.IS_ANCILLARY: True, }, # ... "info" is a user-defined dictionary; see # http://docs.sqlalchemy.org/en/latest/orm/relationship_api.html#sqlalchemy.orm.relationship.params.info # noqa # http://docs.sqlalchemy.org/en/latest/orm/internals.html#MapperProperty.info # noqa )
# ============================================================================= # Field creation assistance # ============================================================================= # TypeEngineBase = TypeVar('TypeEngineBase', bound=TypeEngine)
[docs]def add_multiple_columns( cls: Type, prefix: str, start: int, end: int, coltype=Integer, # this type fails: Union[Type[TypeEngineBase], TypeEngine] # ... https://stackoverflow.com/questions/38106227 # ... https://github.com/python/typing/issues/266 colkwargs: Dict[str, Any] = None, comment_fmt: str = None, comment_strings: List[str] = None, minimum: Union[int, float] = None, maximum: Union[int, float] = None, pv: List[Any] = None) -> None: """ Add a sequence of SQLAlchemy columns to a class. Called from a metaclass. :param cls: class to which to add columns :param prefix: Fieldname will be prefix + str(n), where n defined as below. :param start: Start of range. :param end: End of range. Thus: i will range from 0 to (end - start) inclusive; n will range from start to end inclusive. :param coltype: SQLAlchemy column type, in either of these formats: (a) ``Integer`` (of general type ``Type[TypeEngine]``?); (b) ``Integer()`` (of general type ``TypeEngine``). :param colkwargs: SQLAlchemy column arguments, as in ``Column(name, coltype, **colkwargs)`` :param comment_fmt: Format string defining field comments. Substitutable values are: ``{n}``: field number (from range). ``{s}``: comment_strings[i], or "" if out of range. :param comment_strings: see comment_fmt :param minimum: minimum permitted value, or None :param maximum: maximum permitted value, or None :param pv: list of permitted values, or None """ colkwargs = {} if colkwargs is None else colkwargs # type: Dict[str, Any] comment_strings = comment_strings or [] for n in range(start, end + 1): nstr = str(n) i = n - start colname = prefix + nstr if comment_fmt: s = "" if 0 <= i < len(comment_strings): s = comment_strings[i] or "" colkwargs["comment"] = comment_fmt.format(n=n, s=s) if minimum is not None or maximum is not None or pv is not None: colkwargs["permitted_value_checker"] = PermittedValueChecker( minimum=minimum, maximum=maximum, permitted_values=pv ) setattr(cls, colname, CamcopsColumn(colname, coltype, **colkwargs)) else: setattr(cls, colname, Column(colname, coltype, **colkwargs))