Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import logging 

2from typing import Any, Dict, List, Optional 

3from django.core.exceptions import ValidationError 

4from django.db import transaction 

5from django.utils.dateparse import parse_date 

6from django.utils.timezone import now 

7from django.utils.translation import gettext as _ 

8from jutil.admin import admin_log 

9from jutil.format import choices_label 

10from jutil.parse import parse_datetime 

11from jutil.xml import xml_to_dict 

12 

13from jsanctions.helpers import get_country_iso2_code 

14from jsanctions.models import SanctionsListFile, SanctionEntity, NameAlias, Remark, Address, Identification, \ 

15 SanctionListObject, SubjectType 

16 

17logger = logging.getLogger(__name__) 

18 

19UN_LIST_TYPE = "UN" 

20 

21UN_XML_ARRAY_TAGS = [ 

22 'VALUE', 

23 'INDIVIDUAL', 

24 'INDIVIDUAL_ALIAS', 

25 'INDIVIDUAL_ADDRESS', 

26 'INDIVIDUAL_DATE_OF_BIRTH', 

27 'INDIVIDUAL_PLACE_OF_BIRTH', 

28 'INDIVIDUAL_DOCUMENT', 

29 'ENTITY', 

30 'ENTITY_ALIAS', 

31 'ENTITY_ADDRESS', 

32] 

33 

34UN_NAME_FIELDS = ['FIRST_NAME', 'SECOND_NAME', 'THIRD_NAME', 'FOURTH_NAME', 'FIFTH_NAME', 'SIXTH_NAME'] 

35 

36 

37def load_un_sanction_list_as_dict(filename: str) -> Dict[str, Any]: 

38 with open(filename, "rb") as fp: 

39 data: Dict[str, Any] = xml_to_dict(fp.read(), array_tags=UN_XML_ARRAY_TAGS) 

40 return data 

41 

42 

43def parse_un_data_id(data: Dict[str, Any]) -> int: 

44 uid = data.get('DATAID') 

45 if uid is None: 

46 raise ValidationError(_('DATAID missing')) 

47 return int(uid) 

48 

49 

50def create_un_alias(se: SanctionEntity, **kwargs) -> Optional[NameAlias]: 

51 names = [] 

52 for k in UN_NAME_FIELDS: 

53 if k in kwargs and kwargs[k]: 

54 names.append(kwargs[k]) 

55 if not names: 

56 logger.warning('No names: %s', kwargs) 

57 return None 

58 

59 alias = NameAlias(sanction=se, logical_id=parse_un_data_id(kwargs)) 

60 alias.title = kwargs.get('TITLE') or '' 

61 alias.last_name = names.pop() or '' 

62 alias.first_name = ' '.join(names).strip() 

63 alias.full_clean() 

64 alias.save() 

65 return alias 

66 

67 

68def create_un_comments(se: SanctionEntity, **kwargs) -> List[Remark]: 

69 out: List[Remark] = [] 

70 for n in range(1, 10): 

71 k = 'COMMENTS{}'.format(n) 

72 if k in kwargs and kwargs[k]: 

73 obj_out = Remark(container=se, text=kwargs.get(k) or '') # type: ignore 

74 obj_out.full_clean() 

75 obj_out.save() 

76 out.append(obj_out) 

77 else: 

78 break 

79 return out 

80 

81 

82def create_un_note(obj: SanctionListObject, note: Any): 

83 if note: 

84 remark = Remark(container=obj, text=str(note)) 

85 remark.full_clean() 

86 remark.save() 

87 

88 

89def create_un_address(se: SanctionEntity, **kwargs) -> Address: 

90 # {'STATE_PROVINCE', 'NOTE', 'COUNTRY', 'STREET', 'CITY', 'ZIP_CODE'} 

91 address = Address(sanction=se) 

92 address.region = kwargs.get('STATE_PROVINCE') or '' 

93 address.city = kwargs.get('CITY') or '' 

94 address.zip_code = kwargs.get('ZIP_CODE') or '' 

95 address.country_description = kwargs.get('COUNTRY') or '' 

96 address.street = kwargs.get('STREET') or '' 

97 for k, v in kwargs.items(): 

98 if hasattr(address, k): 

99 setattr(address, k, v) 

100 address.full_clean() 

101 address.save() 

102 create_un_note(address, kwargs.get('NOTE')) 

103 return address 

104 

105 

106def create_un_document(se: SanctionEntity, **kwargs) -> Identification: 

107 # {'DATE_OF_ISSUE', 'NUMBER', 'NOTE', 'ISSUING_COUNTRY', 'CITY_OF_ISSUE', 'COUNTRY_OF_ISSUE', 

108 # 'TYPE_OF_DOCUMENT', 'TYPE_OF_DOCUMENT2'} 

109 id_obj = Identification(sanction=se) 

110 id_obj.identification_type_description = kwargs.get('TYPE_OF_DOCUMENT') or kwargs.get('TYPE_OF_DOCUMENT2') or '' 

111 id_obj.issue_date = parse_date(str(kwargs.get('DATE_OF_ISSUE'))) if kwargs.get('DATE_OF_ISSUE') else None # type: ignore 

112 id_obj.latin_number = kwargs.get('NUMBER') or '' 

113 id_obj.issued_by = '{} {} {}'.format(kwargs.get('CITY_OF_ISSUE') or '', kwargs.get('COUNTRY_OF_ISSUE') or '', kwargs.get('ISSUING_COUNTRY') or '').strip() 

114 id_obj.country_description = kwargs.get('COUNTRY_OF_ISSUE') or kwargs.get('ISSUING_COUNTRY') or '' 

115 id_obj.full_clean() 

116 id_obj.save() 

117 create_un_note(id_obj, kwargs.get('NOTE')) 

118 return id_obj 

119 

120 

121def set_un_members( # noqa 

122 se: SanctionEntity, data: Dict[str, Any], verbose: bool = False, padding: int = 0, 

123): 

124 # DATAID 

125 se.logical_id = parse_un_data_id(data) 

126 

127 # FIRST_NAME, ... 

128 create_un_alias(se, **data) 

129 

130 # COMMENTSx 

131 create_un_comments(se, **data) 

132 

133 # INVIDUAL_ADDRESS / ENTITY_ADDRESS 

134 address_list = data.get("INVIDUAL_ADDRESS", []) or data.get('ENTITY_ADDRESS', []) 

135 addresses: List[Address] = [] 

136 if address_list: 

137 for e_data in address_list: 

138 if e_data: 

139 addresses.append(create_un_address(se, **e_data)) 

140 

141 # try to fill address information from UN list name 

142 if not addresses: 

143 un_list_type = data.get("UN_LIST_TYPE") 

144 if un_list_type: 

145 country_code = get_country_iso2_code(un_list_type) 

146 if country_code: 

147 create_un_address(se, country_description=un_list_type, country_code=country_code) 

148 

149 # INDIVIDUAL_DOCUMENT 

150 docs = data.get('INDIVIDUAL_DOCUMENT') 

151 if docs: 

152 for e_data in docs: 

153 if e_data: 

154 create_un_document(se, **e_data) 

155 

156 se.full_clean() 

157 se.save() 

158 if verbose: 

159 logger.info("%sSaved %s", padding * ' ', se) 

160 

161 

162def import_un_sanctions(source: SanctionsListFile, verbose: bool = False): 

163 data = load_un_sanction_list_as_dict(source.full_path) 

164 source.generation_date = parse_datetime(data["@dateGenerated"]).date() 

165 

166 enterprise, created = SubjectType.objects.get_or_create(classification_code=SubjectType.ENTERPRISE) 

167 assert isinstance(enterprise, SubjectType) 

168 if created or not enterprise.code: 

169 enterprise.code = choices_label(SubjectType.CLASSIFICATION_CODES, enterprise.classification_code) 

170 enterprise.save() 

171 person, created = SubjectType.objects.get_or_create(classification_code=SubjectType.PERSON) 

172 assert isinstance(person, SubjectType) 

173 if created or not person.code: 

174 person.code = choices_label(SubjectType.CLASSIFICATION_CODES, person.classification_code) 

175 person.save() 

176 

177 t0 = now() 

178 individuals_list = data.get("INDIVIDUALS", {}).get('INDIVIDUAL') 

179 for se_data in individuals_list: 

180 assert isinstance(se_data, dict) 

181 if verbose: 

182 logger.info(" sdnEntry uid %s", se_data.get('uid')) 

183 with transaction.atomic(): 

184 se = SanctionEntity.objects.create(source=source, data=se_data, subject_type=person) 

185 set_un_members(se, se_data, verbose=verbose, padding=4) 

186 

187 entities_list = data.get("ENTITIES", {}).get('ENTITY') 

188 for se_data in entities_list: 

189 assert isinstance(se_data, dict) 

190 if verbose: 

191 logger.info(" sdnEntry uid %s", se_data.get('uid')) 

192 with transaction.atomic(): 

193 se = SanctionEntity.objects.create(source=source, data=se_data, subject_type=enterprise) 

194 set_un_members(se, se_data, verbose=verbose, padding=4) 

195 

196 source.imported = now() 

197 source.save() 

198 msg = "Imported {} sanction entities and {} individuals from {} in {}".format( 

199 len(entities_list), len(individuals_list), source.full_path, source.imported - t0 

200 ) 

201 logger.info(msg) 

202 admin_log([source], msg)