Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/local/bin/python 

2# encoding: utf-8 

3""" 

4*Render a python list of dictionaries in various list and markup formats* 

5 

6:Author: 

7 David Young 

8 

9:Date Created: 

10 September 14, 2016 

11""" 

12################# GLOBAL IMPORTS #################### 

13from builtins import str 

14from builtins import range 

15from builtins import object 

16import sys 

17import os 

18import io 

19import re 

20import unicodecsv as csv 

21import codecs 

22import copy 

23import json 

24import yaml 

25from decimal import Decimal 

26from datetime import datetime 

27os.environ['TERM'] = 'vt100' 

28from fundamentals import tools 

29from fundamentals.mysql import convert_dictionary_to_mysql_table 

30 

31 

32class list_of_dictionaries(object): 

33 """ 

34 *The dataset object is a list of python dictionaries. Using this class, the data can be rendered as various list and markup formats* 

35 

36 **Key Arguments:** 

37 - ``log`` -- logger 

38 - ``listOfDictionaries`` -- the list of dictionaries to render 

39 - ``reDatetime`` -- a pre-compiled datetime regex. Default *False*fss  

40 

41 **Usage:** 

42 

43 To initialise the dataset object: 

44 

45 .. code-block:: python 

46 

47 dataList = [ 

48 { 

49 "owner": "daisy", 

50 "pet": "dog", 

51 "address": "belfast, uk" 

52 }, 

53 { 

54 "owner": "john", 

55 "pet": "snake", 

56 "address": "the moon" 

57 }, 

58 { 

59 "owner": "susan", 

60 "pet": "crocodile", 

61 "address": "larne" 

62 } 

63 

64 ] 

65 

66 from fundamentals.renderer import list_of_dictionaries 

67 dataSet = list_of_dictionaries( 

68 log=log, 

69 listOfDictionaries=dataList 

70 ) 

71 """ 

72 

73 def __init__( 

74 self, 

75 log, 

76 listOfDictionaries, 

77 reDatetime=False 

78 ): 

79 self.log = log 

80 self.log.debug("instansiating a new 'list_of_dictionaries' object") 

81 self.listOfDictionaries = listOfDictionaries 

82 self.reDatetime = reDatetime 

83 

84 return None 

85 

86 @property 

87 def list( 

88 self): 

89 """*Returns the original list of dictionaries* 

90 

91 **Usage:** 

92 

93 dataSet.list 

94 """ 

95 return self.listOfDictionaries 

96 

97 def csv( 

98 self, 

99 filepath=None 

100 ): 

101 """*Render the data in CSV format* 

102 

103 **Key Arguments:** 

104 - ``filepath`` -- path to the file to write the csv content to. Default *None* 

105 

106 **Return:** 

107 - ``renderedData`` -- the data rendered in csv format 

108 

109 **Usage:** 

110 

111 To render the data set as csv: 

112 

113 .. code-block:: python 

114 

115 print(dataSet.csv()) 

116 

117 .. code-block:: text 

118 

119 owner,pet,address 

120 daisy,dog,"belfast, uk" 

121 john,snake,the moon 

122 susan,crocodile,larne 

123 

124 and to save the csv rendering to file: 

125 

126 .. code-block:: python 

127 

128 dataSet.csv("/path/to/myfile.csv") 

129 """ 

130 self.log.debug('starting the ``csv`` method') 

131 

132 renderedData = self._list_of_dictionaries_to_csv("machine") 

133 

134 if filepath and renderedData != "NO MATCH": 

135 

136 # RECURSIVELY CREATE MISSING DIRECTORIES 

137 if not os.path.exists(os.path.dirname(filepath)): 

138 os.makedirs(os.path.dirname(filepath)) 

139 

140 writeFile = codecs.open(filepath, encoding='utf-8', mode='w') 

141 writeFile.write(renderedData) 

142 writeFile.close() 

143 

144 self.log.debug('completed the ``csv`` method') 

145 return renderedData 

146 

147 def table( 

148 self, 

149 filepath=None 

150 ): 

151 """*Render the data as a plain text table* 

152 

153 **Key Arguments:** 

154 - ``filepath`` -- path to the file to write the table to. Default *None* 

155 

156 **Return:** 

157 - ``renderedData`` -- the data rendered as a plain text table 

158 

159 **Usage:** 

160 

161 To render the data set as a plain text table: 

162 

163 .. code-block:: python 

164 

165 print(dataSet.table()) 

166 

167 .. code-block:: text 

168 

169 +--------+------------+--------------+ 

170 | owner | pet | address | 

171 +========+============+==============+ 

172 | daisy | dog | belfast, uk | 

173 | john | snake | the moon | 

174 | susan | crocodile | larne | 

175 +--------+------------+--------------+ 

176 

177 and to save the table rendering to file: 

178 

179 .. code-block:: python 

180 

181 dataSet.table("/path/to/myfile.ascii") 

182 """ 

183 self.log.debug('starting the ``table`` method') 

184 

185 self.filepath = filepath 

186 renderedData = self._list_of_dictionaries_to_csv("human") 

187 

188 if filepath and len(self.listOfDictionaries): 

189 

190 # RECURSIVELY CREATE MISSING DIRECTORIES 

191 if not os.path.exists(os.path.dirname(filepath)): 

192 os.makedirs(os.path.dirname(filepath)) 

193 

194 writeFile = codecs.open(filepath, encoding='utf-8', mode='w') 

195 writeFile.write(renderedData) 

196 writeFile.close() 

197 

198 self.log.debug('completed the ``table`` method') 

199 return renderedData 

200 

201 def reST( 

202 self, 

203 filepath=None 

204 ): 

205 """*Render the data as a resturcturedText table* 

206 

207 **Key Arguments:** 

208 - ``filepath`` -- path to the file to write the table to. Default *None* 

209 

210 **Return:** 

211 - ``renderedData`` -- the data rendered as a resturcturedText table 

212 

213 **Usage:** 

214 

215 To render the data set as a resturcturedText table: 

216 

217 .. code-block:: python 

218 

219 print(dataSet.reST()) 

220 

221 .. code-block:: text 

222 

223 +--------+------------+--------------+ 

224 | owner | pet | address | 

225 +========+============+==============+ 

226 | daisy | dog | belfast, uk | 

227 +--------+------------+--------------+ 

228 | john | snake | the moon | 

229 +--------+------------+--------------+ 

230 | susan | crocodile | larne | 

231 +--------+------------+--------------+ 

232 

233 and to save the table rendering to file: 

234 

235 .. code-block:: python 

236 

237 dataSet.reST("/path/to/myfile.rst") 

238 """ 

239 self.log.debug('starting the ``table`` method') 

240 

241 self.filepath = filepath 

242 renderedData = self._list_of_dictionaries_to_csv("reST") 

243 

244 if filepath and len(self.listOfDictionaries): 

245 

246 # RECURSIVELY CREATE MISSING DIRECTORIES 

247 if not os.path.exists(os.path.dirname(filepath)): 

248 os.makedirs(os.path.dirname(filepath)) 

249 

250 writeFile = codecs.open(filepath, encoding='utf-8', mode='w') 

251 writeFile.write(renderedData) 

252 writeFile.close() 

253 

254 self.log.debug('completed the ``table`` method') 

255 return renderedData 

256 

257 def markdown( 

258 self, 

259 filepath=None 

260 ): 

261 """*Render the data as a markdown table* 

262 

263 **Key Arguments:** 

264 - ``filepath`` -- path to the file to write the markdown to. Default *None* 

265 

266 **Return:** 

267 - ``renderedData`` -- the data rendered as a markdown table 

268 

269 **Usage:** 

270 

271 To render the data set as a markdown table: 

272 

273 .. code-block:: python 

274 

275 print(dataSet.markdown()) 

276 

277 .. code-block:: markdown 

278 

279 | owner | pet | address | 

280 |:-------|:-----------|:-------------| 

281 | daisy | dog | belfast, uk | 

282 | john | snake | the moon | 

283 | susan | crocodile | larne | 

284 

285 and to save the markdown table rendering to file: 

286 

287 .. code-block:: python 

288 

289 dataSet.table("/path/to/myfile.md") 

290 """ 

291 self.log.debug('starting the ``markdown`` method') 

292 

293 self.filepath = filepath 

294 renderedData = self._list_of_dictionaries_to_csv("markdown") 

295 

296 if filepath and len(self.listOfDictionaries): 

297 

298 # RECURSIVELY CREATE MISSING DIRECTORIES 

299 if not os.path.exists(os.path.dirname(filepath)): 

300 os.makedirs(os.path.dirname(filepath)) 

301 

302 writeFile = codecs.open(filepath, encoding='utf-8', mode='w') 

303 writeFile.write(renderedData) 

304 writeFile.close() 

305 

306 self.log.debug('completed the ``markdown`` method') 

307 return renderedData 

308 

309 def json( 

310 self, 

311 filepath=None 

312 ): 

313 """*Render the data in json format* 

314 

315 **Key Arguments:** 

316 - ``filepath`` -- path to the file to write the json content to. Default *None* 

317 

318 **Return:** 

319 - ``renderedData`` -- the data rendered as json 

320 

321 **Usage:** 

322 

323 To render the data set as json: 

324 

325 .. code-block:: python 

326 

327 print(dataSet.json()) 

328 

329 .. code-block:: json 

330 

331 [ 

332 { 

333 "address": "belfast, uk", 

334 "owner": "daisy", 

335 "pet": "dog" 

336 }, 

337 { 

338 "address": "the moon", 

339 "owner": "john", 

340 "pet": "snake" 

341 }, 

342 { 

343 "address": "larne", 

344 "owner": "susan", 

345 "pet": "crocodile" 

346 } 

347 ] 

348 

349 and to save the json rendering to file: 

350 

351 .. code-block:: python 

352 

353 dataSet.json("/path/to/myfile.json") 

354 """ 

355 self.log.debug('starting the ``json`` method') 

356 

357 dataCopy = copy.deepcopy(self.listOfDictionaries) 

358 for d in dataCopy: 

359 for k, v in list(d.items()): 

360 if isinstance(v, datetime): 

361 d[k] = v.strftime("%Y%m%dt%H%M%S") 

362 

363 renderedData = json.dumps( 

364 dataCopy, 

365 separators=(',', ': '), 

366 sort_keys=True, 

367 indent=4 

368 ) 

369 

370 if filepath and len(self.listOfDictionaries): 

371 

372 # RECURSIVELY CREATE MISSING DIRECTORIES 

373 if not os.path.exists(os.path.dirname(filepath)): 

374 os.makedirs(os.path.dirname(filepath)) 

375 

376 writeFile = codecs.open(filepath, encoding='utf-8', mode='w') 

377 writeFile.write(renderedData) 

378 writeFile.close() 

379 

380 self.log.debug('completed the ``json`` method') 

381 return renderedData 

382 

383 def yaml( 

384 self, 

385 filepath=None 

386 ): 

387 """*Render the data in yaml format* 

388 

389 **Key Arguments:** 

390 - ``filepath`` -- path to the file to write the yaml content to. Default *None* 

391 

392 **Return:** 

393 - ``renderedData`` -- the data rendered as yaml 

394 

395 **Usage:** 

396 

397 To render the data set as yaml: 

398 

399 .. code-block:: python 

400 

401 print(dataSet.yaml()) 

402 

403 .. code-block:: yaml 

404 

405 - address: belfast, uk 

406 owner: daisy 

407 pet: dog 

408 - address: the moon 

409 owner: john 

410 pet: snake 

411 - address: larne 

412 owner: susan 

413 pet: crocodile 

414 

415 and to save the yaml rendering to file: 

416 

417 .. code-block:: python 

418 

419 dataSet.json("/path/to/myfile.yaml") 

420 """ 

421 self.log.debug('starting the ``yaml`` method') 

422 

423 dataCopy = [] 

424 dataCopy[:] = [dict(l) for l in self.listOfDictionaries] 

425 renderedData = yaml.dump(dataCopy, default_flow_style=False) 

426 

427 if filepath and len(self.listOfDictionaries): 

428 

429 # RECURSIVELY CREATE MISSING DIRECTORIES 

430 if not os.path.exists(os.path.dirname(filepath)): 

431 os.makedirs(os.path.dirname(filepath)) 

432 

433 stream = open(filepath, 'w') 

434 yaml.dump(dataCopy, stream, default_flow_style=False) 

435 stream.close() 

436 

437 self.log.debug('completed the ``yaml`` method') 

438 return renderedData 

439 

440 def mysql( 

441 self, 

442 tableName, 

443 filepath=None, 

444 createStatement=None 

445 ): 

446 """*Render the dataset as a series of mysql insert statements* 

447 

448 **Key Arguments:** 

449 - ``tableName`` -- the name of the mysql db table to assign the insert statements to. 

450 - ``filepath`` -- path to the file to write the mysql inserts content to. Default *None* 

451 createStatement 

452 

453 **Return:** 

454 - ``renderedData`` -- the data rendered mysql insert statements (string format) 

455 

456 **Usage:** 

457 

458 .. code-block:: python 

459 

460 print(dataSet.mysql("testing_table")) 

461 

462 this output the following: 

463 

464 .. code-block:: plain 

465 

466 INSERT INTO `testing_table` (address,dateCreated,owner,pet) VALUES ("belfast, uk" ,"2016-09-14T16:21:36" ,"daisy" ,"dog") ON DUPLICATE KEY UPDATE address="belfast, uk", dateCreated="2016-09-14T16:21:36", owner="daisy", pet="dog" ; 

467 INSERT INTO `testing_table` (address,dateCreated,owner,pet) VALUES ("the moon" ,"2016-09-14T16:21:36" ,"john" ,"snake") ON DUPLICATE KEY UPDATE address="the moon", dateCreated="2016-09-14T16:21:36", owner="john", pet="snake" ; 

468 INSERT INTO `testing_table` (address,dateCreated,owner,pet) VALUES ("larne" ,"2016-09-14T16:21:36" ,"susan" ,"crocodile") ON DUPLICATE KEY UPDATE address="larne", dateCreated="2016-09-14T16:21:36", owner="susan", pet="crocodile" ; 

469 

470 To save this rendering to file use: 

471 

472 .. code-block:: python 

473 

474 dataSet.mysql("testing_table", "/path/to/myfile.sql") 

475 

476 """ 

477 self.log.debug('starting the ``mysql`` method') 

478 

479 import re 

480 if createStatement and "create table if not exists" not in createStatement.lower(): 

481 regex = re.compile(r'^\s*CREATE TABLE ', re.I | re.S) 

482 createStatement = regex.sub( 

483 "CREATE TABLE IF NOT EXISTS ", createStatement) 

484 

485 renderedData = self._list_of_dictionaries_to_mysql_inserts( 

486 tableName=tableName, 

487 createStatement=createStatement 

488 ) 

489 

490 if filepath and len(self.listOfDictionaries): 

491 

492 # RECURSIVELY CREATE MISSING DIRECTORIES 

493 if not os.path.exists(os.path.dirname(filepath)): 

494 os.makedirs(os.path.dirname(filepath)) 

495 

496 writeFile = open(filepath, mode='w') 

497 writeFile.write(renderedData) 

498 writeFile.close() 

499 

500 self.log.debug('completed the ``mysql`` method') 

501 return renderedData 

502 

503 def _list_of_dictionaries_to_csv( 

504 self, 

505 csvType="human"): 

506 """Convert a python list of dictionaries to pretty csv output 

507 

508 **Key Arguments:** 

509 - ``csvType`` -- human, machine or reST 

510 

511 **Return:** 

512 - ``output`` -- the contents of a CSV file 

513 """ 

514 self.log.debug( 

515 'starting the ``_list_of_dictionaries_to_csv`` function') 

516 

517 if not len(self.listOfDictionaries): 

518 return "NO MATCH" 

519 

520 dataCopy = copy.deepcopy(self.listOfDictionaries) 

521 

522 tableColumnNames = list(dataCopy[0].keys()) 

523 columnWidths = [] 

524 columnWidths[:] = [len(tableColumnNames[i]) 

525 for i in range(len(tableColumnNames))] 

526 

527 output = io.BytesIO() 

528 # setup csv styles 

529 if csvType == "machine": 

530 delimiter = "," 

531 elif csvType in ["human", "markdown"]: 

532 delimiter = "|" 

533 elif csvType in ["reST"]: 

534 delimiter = "|" 

535 if csvType in ["markdown"]: 

536 writer = csv.writer(output, delimiter=delimiter, 

537 quoting=csv.QUOTE_NONE, doublequote=False, quotechar='"', escapechar="\\", lineterminator="\n") 

538 else: 

539 writer = csv.writer(output, dialect='excel', delimiter=delimiter, 

540 quotechar='"', quoting=csv.QUOTE_MINIMAL, lineterminator="\n") 

541 

542 if csvType in ["markdown"]: 

543 dividerWriter = csv.writer( 

544 output, delimiter="|", quoting=csv.QUOTE_NONE, doublequote=False, quotechar='"', escapechar="\\", lineterminator="\n") 

545 else: 

546 dividerWriter = csv.writer(output, dialect='excel', delimiter="+", 

547 quotechar='"', quoting=csv.QUOTE_MINIMAL, lineterminator="\n") 

548 # add column names to csv 

549 header = [] 

550 divider = [] 

551 rstDivider = [] 

552 allRows = [] 

553 

554 # clean up data 

555 for row in dataCopy: 

556 for c in tableColumnNames: 

557 if isinstance(row[c], float) or isinstance(row[c], Decimal): 

558 row[c] = "%0.9g" % row[c] 

559 elif isinstance(row[c], datetime): 

560 thisDate = str(row[c])[:10] 

561 row[c] = "%(thisDate)s" % locals() 

562 

563 # set the column widths 

564 for row in dataCopy: 

565 for i, c in enumerate(tableColumnNames): 

566 if len(str(row[c])) > columnWidths[i]: 

567 columnWidths[i] = len(str(row[c])) 

568 

569 # table borders for human readable 

570 if csvType in ["human", "markdown", "reST"]: 

571 header.append("") 

572 divider.append("") 

573 rstDivider.append("") 

574 

575 for i, c in enumerate(tableColumnNames): 

576 if csvType == "machine": 

577 header.append(c) 

578 elif csvType in ["human", "markdown", "reST"]: 

579 header.append( 

580 c.ljust(columnWidths[i] + 2).rjust(columnWidths[i] + 3)) 

581 divider.append('-' * (columnWidths[i] + 3)) 

582 rstDivider.append('=' * (columnWidths[i] + 3)) 

583 

584 # table border for human readable 

585 if csvType in ["human", "markdown", "reST"]: 

586 header.append("") 

587 divider.append("") 

588 rstDivider.append("") 

589 

590 # fill in the data 

591 for row in dataCopy: 

592 thisRow = [] 

593 # table border for human readable 

594 if csvType in ["human", "markdown", "reST"]: 

595 thisRow.append("") 

596 

597 for i, c in enumerate(tableColumnNames): 

598 if csvType in ["human", "markdown", "reST"]: 

599 if row[c] == None: 

600 row[c] = "" 

601 row[c] = str(str(row[c]).ljust(columnWidths[i] + 2) 

602 .rjust(columnWidths[i] + 3)) 

603 thisRow.append(row[c]) 

604 # table border for human readable 

605 if csvType in ["human", "markdown", "reST"]: 

606 thisRow.append("") 

607 allRows.append(thisRow) 

608 if csvType in ["reST"]: 

609 allRows.append(divider) 

610 

611 if csvType == "machine": 

612 writer.writerow(header) 

613 if csvType in ["reST"]: 

614 dividerWriter.writerow(divider) 

615 writer.writerow(header) 

616 dividerWriter.writerow(rstDivider) 

617 if csvType in ["human"]: 

618 dividerWriter.writerow(divider) 

619 writer.writerow(header) 

620 dividerWriter.writerow(divider) 

621 elif csvType in ["markdown"]: 

622 writer.writerow(header) 

623 dividerWriter.writerow(divider) 

624 

625 # write out the data 

626 writer.writerows(allRows) 

627 # table border for human readable 

628 if csvType in ["human"]: 

629 dividerWriter.writerow(divider) 

630 

631 output = output.getvalue() 

632 output = output.strip() 

633 output = str(output) 

634 

635 if csvType in ["markdown"]: 

636 output = output.replace("|--", "|:-") 

637 if csvType in ["reST"]: 

638 output = output.replace("|--", "+--").replace("--|", "--+") 

639 

640 self.log.debug( 

641 'completed the ``_list_of_dictionaries_to_csv`` function') 

642 

643 return output 

644 

645 def _list_of_dictionaries_to_mysql_inserts( 

646 self, 

647 tableName, 

648 createStatement=None): 

649 """Convert a python list of dictionaries to pretty csv output 

650 

651 **Key Arguments:** 

652 - ``tableName`` -- the name of the table to create the insert statements for 

653 - ``createStatement`` -- add this create statement to the top of the file. Will only be executed if no table of that name exists in database. Default *None* 

654 

655 **Return:** 

656 - ``output`` -- the mysql insert statements (as a string) 

657 """ 

658 self.log.debug( 

659 'completed the ````_list_of_dictionaries_to_mysql_inserts`` function') 

660 

661 if not len(self.listOfDictionaries): 

662 return "NO MATCH" 

663 

664 dataCopy = copy.deepcopy(self.listOfDictionaries) 

665 

666 if createStatement: 

667 output = createStatement + "\n" 

668 else: 

669 output = "" 

670 

671 inserts = [] 

672 

673 inserts = [] 

674 inserts[:] = [convert_dictionary_to_mysql_table(log=self.log, dictionary=d, dbTableName=tableName, uniqueKeyList=[ 

675 ], dateModified=False, returnInsertOnly=True, replace=True, batchInserts=False, reDatetime=self.reDatetime) for d in dataCopy] 

676 output += ";\n".join(inserts) + ";" 

677 

678 self.log.debug( 

679 'completed the ``_list_of_dictionaries_to_mysql_inserts`` function') 

680 return output