Coverage for jsanctions/ofac.py : 95%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import logging
2import re
3from datetime import date
4from time import strptime
5from typing import Dict, Any, Tuple, Optional
6from django.core.exceptions import ValidationError
7from django.db import transaction
8from django.utils import translation
9from django.utils.timezone import now
10from django.utils.translation import gettext as _
11from jutil.admin import admin_log
12from jutil.format import choices_label
13from jutil.xml import xml_to_dict
14from jsanctions.models import SanctionsListFile, SanctionEntity, NameAlias, SubjectType, BirthDate, Address, \
15 Identification, Remark
17logger = logging.getLogger(__name__)
19OFAC_LIST_TYPE = "OFAC"
21OFAC_XML_ARRAY_TAGS = ['sdnEntry', 'program', 'aka', 'dateOfBirthItem', 'placeOfBirthItem', 'address', 'id']
24def load_ofac_sanction_list_as_dict(filename: str) -> Dict[str, Any]:
25 with open(filename, "rb") as fp:
26 data: Dict[str, Any] = xml_to_dict(fp.read(), array_tags=OFAC_XML_ARRAY_TAGS)
27 return data
30def parse_ofac_date(v: str) -> date:
31 st = strptime(v, '%m/%d/%Y')
32 if not st:
33 raise ValidationError(_('Invalid date value {}').format(v))
34 return date(st.tm_year, st.tm_mon, st.tm_mday)
37def parse_ofac_dob(v: str) -> Tuple[Optional[int], Optional[int], Optional[int]]:
38 if re.fullmatch(r'\d{4}', v):
39 return int(v), None, None
40 if re.fullmatch(r'\d{1,2}/\d{1,2}/\d{4}', v):
41 st = strptime(v, '%m/%d/%Y')
42 return st.tm_year, st.tm_mon, st.tm_mday
43 if re.fullmatch(r'\d{1,2} \w{3} \d{4}', v):
44 with translation.override('en_US'):
45 st = strptime(v, '%d %b %Y')
46 return st.tm_year, st.tm_mon, st.tm_mday
47 return None, None, None
50def get_opt_ofac_str(data: Dict[str, Any], key: str) -> str:
51 return data.get(key, '') or ''
54def get_ofac_subject_type(data: Dict[str, Any]) -> SubjectType:
55 sdn_type = data["sdnType"]
56 if sdn_type == 'Entity':
57 obj, created = SubjectType.objects.get_or_create(classification_code=SubjectType.ENTERPRISE)
58 elif sdn_type == 'Individual':
59 obj, created = SubjectType.objects.get_or_create(classification_code=SubjectType.PERSON)
60 elif sdn_type == 'Vessel':
61 obj, created = SubjectType.objects.get_or_create(classification_code=SubjectType.VESSEL)
62 elif sdn_type == 'Aircraft':
63 obj, created = SubjectType.objects.get_or_create(classification_code=SubjectType.AIRCRAFT)
64 else:
65 logger.warning('Unknown sdnType: %s', sdn_type)
66 obj, created = SubjectType.objects.get_or_create(classification_code=sdn_type, code=sdn_type)
67 assert isinstance(obj, SubjectType)
68 if created:
69 obj.code = choices_label(SubjectType.CLASSIFICATION_CODES, obj.classification_code)
70 obj.save(update_fields=['code'])
71 return obj
74def parse_ofac_uid(data: Dict[str, Any]) -> int:
75 uid = data.get('uid')
76 if uid is None:
77 raise ValidationError(_('UID missing'))
78 return int(uid)
81def create_ofac_alias(se: SanctionEntity, **kwargs) -> NameAlias:
82 first_name = kwargs.get('firstName') or ''
83 last_name = kwargs.get('lastName') or ''
84 uid = parse_ofac_uid(kwargs)
85 whole_name = (first_name + ' ' + last_name).strip()
86 alias = NameAlias(sanction=se, first_name=first_name, last_name=last_name, whole_name=whole_name, logical_id=uid)
87 alias.full_clean()
88 alias.save()
89 return alias
92def create_ofac_dob(se: SanctionEntity, **kwargs) -> BirthDate:
93 dob = BirthDate(sanction=se)
94 dob.logical_id = parse_ofac_uid(kwargs)
95 dob.birth_date_description = kwargs.get("dateOfBirth") or ''
96 year, month_of_year, day_of_month = parse_ofac_dob(dob.birth_date_description)
97 dob.year, dob.month_of_year, dob.day_of_month = year, month_of_year, day_of_month
98 if year and month_of_year and day_of_month:
99 dob.birth_date = date(year, month_of_year, day_of_month)
100 dob.full_clean()
101 dob.save()
102 return dob
105def create_ofac_place_of_birth(se: SanctionEntity, **kwargs) -> BirthDate:
106 dob = BirthDate.objects.all().filter(sanction=se, place='').order_by('id').first()
107 if dob is None:
108 dob = BirthDate(sanction=se, logical_id=parse_ofac_uid(kwargs))
109 dob.place = get_opt_ofac_str(kwargs, 'placeOfBirth')
110 dob.full_clean()
111 dob.save()
112 return dob
115def create_ofac_address(se: SanctionEntity, **kwargs) -> Address:
116 address = Address(sanction=se)
117 address.logical_id = parse_ofac_uid(kwargs)
118 address.region = get_opt_ofac_str(kwargs, 'stateOrProvince')
119 address.city = get_opt_ofac_str(kwargs, 'city')
120 address.zip_code = get_opt_ofac_str(kwargs, 'postalCode')
121 address.country_description = get_opt_ofac_str(kwargs, 'country')
122 street = ''
123 for n in range(1, 5):
124 k = 'address{}'.format(n)
125 if k in kwargs:
126 street = street + '\n' + kwargs.get(k)
127 else:
128 break
129 address.street = street.strip()
130 address.full_clean()
131 address.save()
132 return address
135def create_ofac_id(se: SanctionEntity, **kwargs) -> Identification:
136 id_obj = Identification(sanction=se)
137 id_obj.logical_id = parse_ofac_uid(kwargs)
138 id_obj.number = kwargs.get('idNumber') or ''
139 id_obj.identification_type_description = kwargs.get('idType') or ''
140 id_obj.country_description = kwargs.get('idCountry') or ''
141 id_obj.full_clean()
142 id_obj.save()
143 return id_obj
146def set_ofac_members( # noqa
147 se: SanctionEntity, data: Dict[str, Any], verbose: bool = False, padding: int = 0,
148):
149 # uid
150 se.logical_id = parse_ofac_uid(data)
152 # firstName, lastName
153 first_name, last_name = get_opt_ofac_str(data, 'firstName'), get_opt_ofac_str(data, 'lastName')
154 if first_name or last_name:
155 create_ofac_alias(se, **data)
157 # sdnType
158 se.subject_type = get_ofac_subject_type(data)
160 # remarks
161 remarks = data.get('remarks') or ''
162 if remarks:
163 remark_obj = Remark(container=se, text=remarks)
164 remark_obj.full_clean()
165 remark_obj.save()
167 # programList
168 for program in data.get('programList', {}).get('program', []) or []:
169 if program:
170 remark_obj = Remark(container=se, text='program={}'.format(program))
171 remark_obj.full_clean()
172 remark_obj.save()
174 # akaList
175 for e_data in data.get('akaList', {}).get('aka', []) or []:
176 create_ofac_alias(se, **e_data)
178 # dateOfBirthList
179 for e_data in data.get('dateOfBirthList', {}).get('dateOfBirthItem', []) or []:
180 create_ofac_dob(se, **e_data)
182 # placeOfBirthList
183 for e_data in data.get('placeOfBirthList', {}).get('placeOfBirthItem', []) or []:
184 create_ofac_place_of_birth(se, **e_data)
186 # addressList
187 for e_data in data.get('addressList', {}).get('address', []) or []:
188 create_ofac_address(se, **e_data)
190 # idList
191 for e_data in data.get('idList', {}).get('id', []) or []:
192 create_ofac_id(se, **e_data)
194 se.full_clean()
195 se.save()
196 if verbose:
197 logger.info("%sSaved %s", padding * ' ', se)
200def import_ofac_sanctions(source: SanctionsListFile, verbose: bool = False):
201 data = load_ofac_sanction_list_as_dict(source.full_path)
202 source.generation_date = parse_ofac_date(data['publshInformation']['Publish_Date'])
204 t0 = now()
205 entities_list = data.get("sdnEntry", [])
206 for se_data in entities_list:
207 assert isinstance(se_data, dict)
208 if verbose:
209 logger.info(" sdnEntry uid %s", se_data.get('uid'))
210 with transaction.atomic():
211 se = SanctionEntity.objects.create(source=source, data=se_data)
212 set_ofac_members(se, se_data, verbose=verbose, padding=4)
214 source.imported = now()
215 source.save()
216 msg = "Imported {} sanction entities from {} in {}".format(
217 len(entities_list), source.full_path, source.imported - t0
218 )
219 logger.info(msg)
220 admin_log([source], msg)