#!/usr/bin/env python
# camcops_server/cc_modules/cc_hl7.py
"""
===============================================================================
Copyright (C) 2012-2018 Rudolf Cardinal (rudolf@pobox.com).
This file is part of CamCOPS.
CamCOPS is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
CamCOPS is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with CamCOPS. If not, see <http://www.gnu.org/licenses/>.
===============================================================================
"""
import errno
import codecs
import hl7
import lockfile
import logging
import os
import socket
import subprocess
import sys
from typing import List, Optional, TextIO, Tuple, TYPE_CHECKING, Union
from cardinal_pythonlib.datetimefunc import (
format_datetime,
get_now_utc_datetime,
)
from cardinal_pythonlib.logs import BraceStyleAdapter
from cardinal_pythonlib.network import ping
from sqlalchemy.orm import reconstructor, relationship
from sqlalchemy.sql.expression import or_, not_
from sqlalchemy.sql.schema import Column, ForeignKey
from sqlalchemy.sql.sqltypes import (
BigInteger,
Boolean,
DateTime,
Integer,
Text,
UnicodeText,
)
from .cc_constants import DateFormat, HL7MESSAGE_TABLENAME, ERA_NOW
from .cc_filename import change_filename_ext, FileType
from .cc_hl7core import (
make_msh_segment,
msg_is_successful_ack,
SEGMENT_SEPARATOR,
)
from .cc_config import CamcopsConfig
from .cc_recipdef import RecipientDefinition
from .cc_request import CamcopsRequest
from .cc_sqla_coltypes import (
HostnameColType,
LongText,
SendingFormatColType,
TableNameColType,
)
from .cc_sqlalchemy import Base
if TYPE_CHECKING:
from .cc_task import Task
log = BraceStyleAdapter(logging.getLogger(__name__))
# =============================================================================
# General HL7 sources
# =============================================================================
# http://python-hl7.readthedocs.org/en/latest/
# http://www.interfaceware.com/manual/v3gen_python_library_details.html
# http://www.interfaceware.com/hl7_video_vault.html#how
# http://www.interfaceware.com/hl7-standard/hl7-segments.html
# http://www.hl7.org/special/committees/vocab/v26_appendix_a.pdf
# http://www.ncbi.nlm.nih.gov/pmc/articles/PMC130066/
# =============================================================================
# HL7 design
# =============================================================================
# WHICH RECORDS TO SEND?
# Most powerful mechanism is not to have a sending queue (which would then
# require careful multi-instance locking), but to have a "sent" log. That way:
# - A record needs sending if it's not in the sent log (for an appropriate
# server).
# - You can add a new server and the system will know about the (new) backlog
# automatically.
# - You can specify criteria, e.g. don't upload records before 1/1/2014, and
# modify that later, and it would catch up with the backlog.
# - Successes and failures are logged in the same table.
# - Multiple recipients are handled with ease.
# - No need to alter database.pl code that receives from tablets.
# - Can run with a simple cron job.
# LOCKING
# - Don't use database locking:
# https://blog.engineyard.com/2011/5-subtle-ways-youre-using-mysql-as-a-queue-and-why-itll-bite-you # noqa
# - Locking via UNIX lockfiles:
# https://pypi.python.org/pypi/lockfile
# http://pythonhosted.org/lockfile/
# ... which also works on Windows.
# CALLING THE HL7 PROCESSOR
# - Use "camcops -7 ..." or "camcops --hl7 ..."
# - Call it via a cron job, e.g. every 5 minutes.
# CONFIG FILE
# q.v.
# TO CONSIDER
# - batched messages (HL7 batching protocol)
# http://docs.oracle.com/cd/E23943_01/user.1111/e23486/app_hl7batching.htm
# - note: DG1 segment = diagnosis
# BASIC MESSAGE STRUCTURE
# - package into HL7 2.X message as encapsulated PDF
# http://www.hl7standards.com/blog/2007/11/27/pdf-attachment-in-hl7-message/
# - message ORU^R01
# http://www.corepointhealth.com/resource-center/hl7-resources/hl7-messages
# MESSAGES: http://www.interfaceware.com/hl7-standard/hl7-messages.html
# - OBX segment = observation/result segment
# http://www.corepointhealth.com/resource-center/hl7-resources/hl7-obx-segment # noqa
# http://www.interfaceware.com/hl7-standard/hl7-segment-OBX.html
# - SEGMENTS:
# http://www.corepointhealth.com/resource-center/hl7-resources/hl7-segments
# - ED field (= encapsulated data)
# http://www.interfaceware.com/hl7-standard/hl7-fields.html
# - base-64 encoding
# - Option for structure (XML), HTML, PDF export.
# =============================================================================
# HL7Run class
# =============================================================================
[docs]class HL7Run(Base):
"""
Class representing an HL7/file run for a specific recipient.
May be associated with multiple HL7/file messages.
"""
__tablename__ = "_hl7_run_log"
run_id = Column(
"run_id", BigInteger,
primary_key=True, autoincrement=True,
comment="Arbitrary primary key"
)
start_at_utc = Column(
"start_at_utc", DateTime,
nullable=False, index=True,
comment="Time run was started (UTC)"
)
finish_at_utc = Column(
"finish_at_utc", DateTime,
comment="Time run was finished (UTC)"
)
recipient = Column(
"recipient", HostnameColType,
index=True,
comment="Recipient definition name (determines uniqueness)"
)
# Common to all ways of sending:
type = Column(
"type", SendingFormatColType,
nullable=False,
comment="Recipient type (e.g. hl7, file)"
)
group_id = Column(
"group_id", Integer, ForeignKey("_security_groups.id"),
nullable=False, index=True,
comment="ID of CamCOPS group to export data from"
)
primary_idnum = Column(
"primary_idnum", Integer,
nullable=False,
comment="Which ID number was used as the primary ID?"
)
require_idnum_mandatory = Column(
"require_idnum_mandatory", Boolean,
comment="Must the primary ID number be mandatory in the relevant "
"policy?"
)
start_date = Column(
"start_date", DateTime,
comment="Start date for tasks (UTC)"
)
end_date = Column(
"end_date", DateTime,
comment="End date for tasks (UTC)"
)
finalized_only = Column(
"finalized_only", Boolean,
comment="Send only finalized tasks"
)
task_format = Column(
"task_format", SendingFormatColType,
comment="Format that task information was sent in (e.g. PDF)"
)
xml_field_comments = Column(
"xml_field_comments", Boolean,
comment="Whether to include field comments in XML output"
)
# For HL7 method:
host = Column(
"host", HostnameColType,
comment="(HL7) Destination host name/IP address"
)
port = Column(
"port", Integer,
comment="(HL7) Destination port number"
)
divert_to_file = Column(
"divert_to_file", Text,
comment="(HL7) Divert to file with this name"
)
treat_diverted_as_sent = Column(
"treat_diverted_as_sent", Boolean,
comment="(HL7) Treat messages diverted to file as sent"
)
# For file method:
include_anonymous = Column(
"include_anonymous", Boolean,
comment="(FILE) Include anonymous tasks"
)
overwrite_files = Column(
"overwrite_files", Boolean,
comment="(FILE) Overwrite existing files"
)
rio_metadata = Column(
"rio_metadata", Boolean,
comment="(FILE) Export RiO metadata file along with main file?"
)
rio_idnum = Column(
"rio_idnum", Integer,
comment="(FILE) RiO metadata: which ID number is the RiO ID?"
)
rio_uploading_user = Column(
"rio_uploading_user", Text,
comment="(FILE) RiO metadata: name of automatic upload user"
)
rio_document_type = Column(
"rio_document_type", Text,
comment="(FILE) RiO metadata: document type for RiO"
)
script_after_file_export = Column(
"script_after_file_export", Text,
comment="(FILE) Command/script to run after file export"
)
# More, beyond the recipient definition:
script_retcode = Column(
"script_retcode", Integer,
comment="Return code from the script_after_file_export script"
)
script_stdout = Column(
"script_stdout", UnicodeText,
comment="stdout from the script_after_file_export script"
)
script_stderr = Column(
"script_stderr", UnicodeText,
comment="stderr from the script_after_file_export script"
)
def __init__(self, recipdef: RecipientDefinition = None,
*args, **kwargs) -> None:
"""
Initialize from a RecipientDefinition, copying its fields.
(However, we must also support a no-parameter constructor, not least
for our merge_db() function.)
"""
super().__init__(*args, **kwargs)
if recipdef:
# Copy:
# ... common
self.recipient = recipdef.recipient
self.type = recipdef.type
self.group_id = recipdef.group_id
self.primary_idnum = recipdef.primary_idnum
self.require_idnum_mandatory = recipdef.require_idnum_mandatory
self.start_date = recipdef.start_date
self.end_date = recipdef.end_date
self.finalized_only = recipdef.finalized_only
self.task_format = recipdef.task_format
self.xml_field_comments = recipdef.xml_field_comments
# ... HL7
self.host = recipdef.host
self.port = recipdef.port
self.divert_to_file = recipdef.divert_to_file
self.treat_diverted_as_sent = recipdef.treat_diverted_as_sent
# ... File
self.include_anonymous = recipdef.include_anonymous
self.overwrite_files = recipdef.overwrite_files
self.rio_metadata = recipdef.rio_metadata
self.rio_idnum = recipdef.rio_idnum
self.rio_uploading_user = recipdef.rio_uploading_user
self.rio_document_type = recipdef.rio_document_type
self.script_after_file_export = recipdef.script_after_file_export
# New things:
self.start_at_utc = get_now_utc_datetime()
self.finish_at_utc = None
def call_script(self, files_exported: Optional[List[str]]) -> None:
if not self.script_after_file_export:
# No script to call
return
if not files_exported:
# Didn't export any files; nothing to do.
self.script_after_file_export = None # wasn't called
return
args = [self.script_after_file_export] + files_exported
try:
encoding = sys.getdefaultencoding()
p = subprocess.Popen(args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = p.communicate()
self.script_stdout = out.decode(encoding)
self.script_stderr = err.decode(encoding)
self.script_retcode = p.returncode
except Exception as e:
self.script_stdout = "Failed to run script"
self.script_stderr = str(e)
def finish(self) -> None:
self.finish_at_utc = get_now_utc_datetime()
# =============================================================================
# HL7Message class
# =============================================================================
[docs]class HL7Message(Base):
__tablename__ = HL7MESSAGE_TABLENAME # indirected to resolve circular dependency # noqa
msg_id = Column(
"msg_id", Integer,
primary_key=True, autoincrement=True,
comment="Arbitrary primary key"
)
run_id = Column(
"run_id", BigInteger, ForeignKey("_hl7_run_log.run_id"),
comment="FK to _hl7_run_log.run_id"
)
hl7run = relationship("HL7Run")
basetable = Column(
"basetable", TableNameColType,
index=True,
comment="Base table of task concerned"
)
serverpk = Column(
"serverpk", Integer,
index=True,
comment="Server PK of task in basetable (_pk field)"
)
sent_at_utc = Column(
"sent_at_utc", DateTime,
comment="Time message was sent at (UTC)"
)
reply_at_utc = Column(
"reply_at_utc", DateTime,
comment="(HL7) Time message was replied to (UTC)"
)
success = Column(
"success", Boolean,
comment="Message sent successfully (and, for HL7, acknowledged)"
)
failure_reason = Column(
"failure_reason", Text,
comment="Reason for failure"
)
message = Column(
"message", LongText,
comment="(HL7) Message body, if kept"
)
reply = Column(
"reply", Text,
comment="(HL7) Server's reply, if kept"
)
filename = Column(
"filename", Text,
comment="(FILE) Destination filename"
)
rio_metadata_filename = Column(
"rio_metadata_filename", Text,
comment="(FILE) RiO metadata filename, if used"
)
cancelled = Column(
"cancelled", Boolean,
comment="Message subsequently invalidated (may trigger resend)"
)
cancelled_at_utc = Column(
"cancelled_at_utc", DateTime,
comment="Time message was cancelled at (UTC)"
)
def __init__(self,
task: "Task" = None,
recipient_def: RecipientDefinition = None,
hl7run: HL7Run = None,
show_queue_only: bool = False,
*args, **kwargs) -> None:
"""
Must support parameter-free construction, not least for merge_db().
"""
super().__init__(*args, **kwargs)
# Internal attributes
self._host = None # type: str
self._port = None # type: int
self._msg = None # type: str
self._recipient_def = recipient_def
self._show_queue_only = show_queue_only
self._task = task # type: Task
# Columns
self.hl7run = hl7run
if task:
self.basetable = task.__tablename__
self.serverpk = task.get_pk()
else:
self.basetable = None
self.serverpk = None
@reconstructor
def init_on_load(self) -> None:
self._host = None # type: str
self._port = None # type: int
self._msg = None # type: str
self._recipient_def = None # type: RecipientDefinition
self._show_queue_only = True
self._task = None # type: Task
[docs] def valid(self, req: CamcopsRequest) -> bool:
"""Checks for internal validity; returns Boolean."""
if not self._recipient_def or not self._recipient_def.valid(req):
return False
if not self.basetable or self.serverpk is None:
return False
if not self._task:
return False
anonymous_ok = (self._recipient_def.using_file() and
self._recipient_def.include_anonymous)
task_is_anonymous = self._task.is_anonymous
if task_is_anonymous and not anonymous_ok:
return False
# After this point, all anonymous tasks must be OK. So:
task_has_primary_id = self._task.get_patient_idnum_value(
self._recipient_def.primary_idnum) is not None
if not task_is_anonymous and not task_has_primary_id:
return False
return True
[docs] def divert_to_file(self, f: TextIO) -> None:
"""Write an HL7 message to a file."""
infomsg = (
"OUTBOUND MESSAGE DIVERTED FROM RECIPIENT {} AT {}\n".format(
self._recipient_def.recipient,
format_datetime(self.sent_at_utc, DateFormat.ISO8601)
)
)
print(infomsg, file=f)
print(str(self._msg), file=f)
print("\n", file=f)
log.debug(infomsg)
self._host = self._recipient_def.divert_to_file
if self._recipient_def.treat_diverted_as_sent:
self.success = True
[docs] def send(self,
req: CamcopsRequest,
queue_file: TextIO = None,
divert_file: TextIO = None) -> Tuple[bool, bool]:
"""Send an outbound HL7/file message, by the appropriate method."""
# returns: tried, succeeded
if not self.valid(req):
return False, False
if self._show_queue_only:
print("{},{},{},{},{}".format(
self._recipient_def.recipient,
self._recipient_def.type,
self.basetable,
self.serverpk,
self._task.when_created
), file=queue_file)
return False, True
if not self.hl7run:
return True, False
if self.msg_id is None:
# The "self" object should be in the request's dbsession, so:
req.dbsession.flush()
assert self.msg_id is not None
self.sent_at_utc = req.now_utc
if self._recipient_def.using_hl7():
self.make_hl7_message(req) # will write its own error msg/flags
if self._recipient_def.divert_to_file:
self.divert_to_file(divert_file)
else:
self.transmit_hl7()
elif self._recipient_def.using_file():
self.send_to_filestore(req)
else:
raise AssertionError("HL7Message.send: invalid recipient_def.type")
self.save()
log.debug("HL7Message.send: recipient={}, basetable={}, serverpk={}",
self._recipient_def.recipient,
self.basetable,
self.serverpk)
return True, self.success
[docs] def send_to_filestore(self, req: CamcopsRequest) -> None:
"""Send a file to a filestore."""
self.filename = self._recipient_def.get_filename(
req=req,
is_anonymous=self._task.is_anonymous,
surname=self._task.get_patient_surname(),
forename=self._task.get_patient_forename(),
dob=self._task.get_patient_dob(),
sex=self._task.get_patient_sex(),
idnum_objects=self._task.get_patient_idnum_objects(),
creation_datetime=self._task.get_creation_datetime(),
basetable=self.basetable,
serverpk=self.serverpk,
)
filename = self.filename
directory = os.path.dirname(filename)
task = self._task
task_format = self._recipient_def.task_format
allow_overwrite = self._recipient_def.overwrite_files
if task_format == FileType.PDF:
binary_data = task.get_pdf(req)
string_data = None
elif task_format == FileType.HTML:
binary_data = None
string_data = task.get_html(req)
elif task_format == FileType.XML:
binary_data = None
string_data = task.get_xml(req)
else:
raise AssertionError("write_to_filestore_file: bug")
if not allow_overwrite and os.path.isfile(filename):
self.failure_reason = "File already exists"
return
if self._recipient_def.make_directory:
try:
make_sure_path_exists(directory)
except Exception as e:
self.failure_reason = "Couldn't make directory {} ({})".format(
directory, e)
return
try:
if task_format == FileType.PDF:
# binary for PDF
with open(filename, mode="wb") as f:
f.write(binary_data)
else:
# UTF-8 for HTML, XML
with codecs.open(filename, mode="w", encoding="utf8") as f:
f.write(string_data)
except Exception as e:
self.failure_reason = "Failed to open or write file: {}".format(e)
return
# RiO metadata too?
if self._recipient_def.rio_metadata:
# No spaces in filename
self.rio_metadata_filename = change_filename_ext(
self.filename, ".metadata").replace(" ", "")
self.rio_metadata_filename = self.rio_metadata_filename
metadata = task.get_rio_metadata(
self._recipient_def.rio_idnum,
self._recipient_def.rio_uploading_user,
self._recipient_def.rio_document_type
)
try:
dos_newline = "\r\n"
# ... Servelec say CR = "\r", but DOS is \r\n.
with codecs.open(self.rio_metadata_filename, mode="w",
encoding="ascii") as f:
# codecs.open() means that file writing is in binary mode,
# so newline conversion has to be manual:
f.write(metadata.replace("\n", dos_newline))
# UTF-8 is NOT supported by RiO for metadata.
except Exception as e:
self.failure_reason = ("Failed to open or write RiO metadata "
"file: {}".format(e))
return
self.success = True
[docs] def make_hl7_message(self, req: CamcopsRequest) -> None:
"""
Stores HL7 message in self.msg.
May also store it in self.message (which is saved to the database), if
we're saving HL7 messages.
"""
# http://python-hl7.readthedocs.org/en/latest/index.html
msh_segment = make_msh_segment(
message_datetime=req.now,
message_control_id=str(self.msg_id)
)
pid_segment = self._task.get_patient_hl7_pid_segment(
req, self._recipient_def)
other_segments = self._task.get_hl7_data_segments(
req, self._recipient_def)
# ---------------------------------------------------------------------
# Whole message
# ---------------------------------------------------------------------
segments = [msh_segment, pid_segment] + other_segments
self._msg = hl7.Message(SEGMENT_SEPARATOR, segments)
if self._recipient_def.keep_message:
self.message = str(self._msg)
[docs] def transmit_hl7(self) -> None:
"""Sends HL7 message over TCP/IP."""
# Default MLLP/HL7 port is 2575
# ... MLLP = minimum lower layer protocol
# ... http://www.cleo.com/support/byproduct/lexicom/usersguide/mllp_configuration.htm # noqa
# ... http://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml?search=hl7 # noqa
# Essentially just a TCP socket with a minimal wrapper:
# http://stackoverflow.com/questions/11126918
self._host = self._recipient_def.host
self._port = self._recipient_def.port
self.success = False
# http://python-hl7.readthedocs.org/en/latest/api.html
# ... but we've modified that
try:
with MLLPTimeoutClient(self._recipient_def.host,
self._recipient_def.port,
self._recipient_def.network_timeout_ms) \
as client:
server_replied, reply = client.send_message(self._msg)
except socket.timeout:
self.failure_reason = "Failed to send message via MLLP: timeout"
return
except Exception as e:
self.failure_reason = "Failed to send message via MLLP: {}".format(
str(e))
return
if not server_replied:
self.failure_reason = "No response from server"
return
self.reply_at_utc = get_now_utc_datetime()
if self._recipient_def.keep_reply:
self.reply = reply
try:
replymsg = hl7.parse(reply)
except Exception as e:
self.failure_reason = "Malformed reply: {}".format(e)
return
self.success, self.failure_reason = msg_is_successful_ack(replymsg)
# =============================================================================
# MLLPTimeoutClient
# =============================================================================
# Modification of MLLPClient from python-hl7, to allow timeouts and failure.
SB = '\x0b' # <SB>, vertical tab
EB = '\x1c' # <EB>, file separator
CR = '\x0d' # <CR>, \r
FF = '\x0c' # <FF>, new page form feed
RECV_BUFFER = 4096
[docs]class MLLPTimeoutClient(object):
"""Class for MLLP TCP/IP transmission that implements timeouts."""
def __init__(self, host: str, port: int, timeout_ms: int = None) -> None:
"""Creates MLLP client and opens socket."""
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
timeout_s = float(timeout_ms) / float(1000) \
if timeout_ms is not None else None
self.socket.settimeout(timeout_s)
self.socket.connect((host, port))
def __enter__(self):
"""For use with "with" statement."""
return self
# noinspection PyUnusedLocal
def __exit__(self, exc_type, exc_val, traceback):
"""For use with "with" statement."""
self.close()
[docs] def close(self):
"""Release the socket connection"""
self.socket.close()
[docs] def send_message(self, message: Union[str, hl7.Message]) \
-> Tuple[bool, Optional[str]]:
"""Wraps a str, unicode, or :py:class:`hl7.Message` in a MLLP container
and send the message to the server
"""
if isinstance(message, hl7.Message):
message = str(message)
# wrap in MLLP message container
data = SB + message + CR + EB + CR
# ... the CR immediately after the message is my addition, because
# HL7 Inspector otherwise says: "Warning: last segment have no segment
# termination char 0x0d !" (sic).
return self.send(data.encode('utf-8'))
[docs] def send(self, data: bytes) -> Tuple[bool, Optional[str]]:
"""Low-level, direct access to the socket.send (data must be already
wrapped in an MLLP container). Blocks until the server returns.
"""
# upload the data
self.socket.send(data)
# wait for the ACK/NACK
try:
ack_msg = self.socket.recv(RECV_BUFFER)
return True, ack_msg
except socket.timeout:
return False, None
# =============================================================================
# Main functions
# =============================================================================
[docs]def send_all_pending_hl7_messages(cfg: CamcopsConfig,
show_queue_only: bool = False) -> None:
"""Sends all pending HL7 or file messages.
Obtains a file lock, then iterates through all recipients.
"""
queue_stdout = sys.stdout
if not cfg.hl7_lockfile:
log.error("send_all_pending_hl7_messages: No HL7_LOCKFILE specified"
" in config; can't proceed")
return
# On UNIX, lockfile uses LinkLockFile
# https://github.com/smontanaro/pylockfile/blob/master/lockfile/
# linklockfile.py
lock = lockfile.FileLock(cfg.hl7_lockfile)
if lock.is_locked():
log.warning("send_all_pending_hl7_messages: locked by another"
" process; aborting")
return
with lock: # calls lock.__enter__() and, later, lock.__exit__()
with cfg.get_dbsession_context() as dbsession:
if show_queue_only:
print("recipient,basetable,_pk,when_created", file=queue_stdout)
for recipient_def in cfg.hl7_recipient_defs:
send_pending_hl7_messages(dbsession, recipient_def,
show_queue_only, queue_stdout)
[docs]def send_pending_hl7_messages(req: CamcopsRequest,
recipient_def: RecipientDefinition,
show_queue_only: bool,
queue_stdout: TextIO) -> None:
"""Pings recipient if necessary, opens any files required, creates an
HL7Run, then sends all pending HL7/file messages to a specific
recipient."""
# Called once per recipient.
log.debug("send_pending_hl7_messages: " + str(recipient_def))
use_ping = (recipient_def.using_hl7() and
not recipient_def.divert_to_file and
recipient_def.ping_first)
if use_ping:
# No HL7 PING method yet. Proposal is:
# http://hl7tsc.org/wiki/index.php?title=FTSD-ConCalls-20081028
# So use TCP/IP ping.
try:
timeout_s = min(recipient_def.network_timeout_ms // 1000, 1)
if not ping(hostname=recipient_def.host,
timeout_s=timeout_s):
log.error("Failed to ping {}", recipient_def.host)
return
except socket.error:
log.error("Socket error trying to ping {}; likely need to "
"run as root", recipient_def.host)
return
if show_queue_only:
hl7run = None
else:
hl7run = HL7Run(recipient_def)
# Do things, but with safe file closure if anything goes wrong
use_divert = (recipient_def.using_hl7() and recipient_def.divert_to_file)
if use_divert:
try:
with codecs.open(recipient_def.divert_to_file, "a", "utf8") as f:
send_pending_hl7_messages_2(req, recipient_def,
show_queue_only,
queue_stdout, hl7run, f)
except Exception as e:
log.error("Couldn't open file {} for appending: {}",
recipient_def.divert_to_file, e)
return
else:
send_pending_hl7_messages_2(req, recipient_def, show_queue_only,
queue_stdout, hl7run, None)
[docs]def send_pending_hl7_messages_2(
req: CamcopsRequest,
recipient_def: RecipientDefinition,
show_queue_only: bool,
queue_stdout: TextIO,
hl7run: HL7Run,
divert_file: Optional[TextIO]) -> None:
"""
Sends all pending HL7/file messages to a specific recipient.
Also called once per recipient, but after diversion files safely
opened and recipient pinged successfully (if desired).
"""
dbsession = req.dbsession
n_hl7_sent = 0
n_hl7_successful = 0
n_file_sent = 0
n_file_successful = 0
files_exported = []
for cls in Task.all_subclasses_by_tablename():
if cls.is_anonymous and not recipient_def.include_anonymous:
continue
basetable = cls.__tablename__
# FETCH TASKS TO SEND.
# Records from the correct group...
# Current records...
# noinspection PyProtectedMember, PyPep8
q = (
dbsession.query(cls)
.filter(cls._group_id == recipient_def.group_id)
.filter(cls._current == True)
)
# Having an appropriate date...
# Best to use when_created, or _when_added_batch_utc?
# The former. Because nobody would want a system that would miss
# amendments to records, and records are defined (date-wise) by
# when_created.
if recipient_def.start_date:
q = q.filter(cls.when_created >= recipient_def.start_date)
if recipient_def.end_date:
q = q.filter(cls.when_created <= recipient_def.end_date)
# That haven't already had a successful HL7 message sent to this
# server..
subquery = (
dbsession.query(HL7Message.serverpk)
.join(HL7Run) # automatic: HL7Run.run_id == HL7Message.run_id
.filter(HL7Message.basetable == basetable)
.filter(HL7Run.recipient == recipient_def.recipient)
.filter(HL7Message.success == True)
.filter(or_(not_(HL7Message.cancelled),
HL7Message.cancelled.is_(None)))
) # nopep8
# noinspection PyProtectedMember
q = q.filter(cls._pk.notin_(subquery))
# http://explainextended.com/2009/09/18/not-in-vs-not-exists-vs-left-join-is-null-mysql/ # noqa
# That are finalized (i.e. aren't still on the tablet and potentially
# subject to modification)?
if recipient_def.finalized_only:
# noinspection PyProtectedMember
q = q.filter(cls._era != ERA_NOW)
# OK. Fetch PKs and send information.
for task in q:
msg = HL7Message(task=task,
hl7run=hl7run,
recipient_def=recipient_def,
show_queue_only=show_queue_only)
dbsession.add(msg)
tried, succeeded = msg.send(req, queue_stdout, divert_file)
if not tried:
continue
if recipient_def.using_hl7():
n_hl7_sent += 1
n_hl7_successful += 1 if succeeded else 0
else:
n_file_sent += 1
n_file_successful += 1 if succeeded else 0
if succeeded:
files_exported.append(msg.filename)
if msg.rio_metadata_filename:
files_exported.append(msg.rio_metadata_filename)
if hl7run:
hl7run.call_script(files_exported)
hl7run.finish()
log.info("{} HL7 messages sent, {} successful, {} failed",
n_hl7_sent, n_hl7_successful, n_hl7_sent - n_hl7_successful)
log.info("{} files sent, {} successful, {} failed",
n_file_sent, n_file_successful, n_file_sent - n_file_successful)
# =============================================================================
# File-handling functions
# =============================================================================
[docs]def make_sure_path_exists(path: str) -> None:
"""Creates a directory/directories if the path doesn't already exist."""
# http://stackoverflow.com/questions/273192
try:
os.makedirs(path)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise