Coverage for /Users/Dave/git_repos/_packages_/python/fundamentals/fundamentals/mysql/yaml_to_database.py : 20%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/local/bin/python
2# encoding: utf-8
3"""
4*Take key-values from a yaml file including a tablename(s) and add them to a mysql database table*
6Usage:
7 yaml2db [-d] -s <pathToSettingsFile> <pathToYaml>
8 yaml2db [-d] --host=<host> --user=<user> --passwd=<passwd> --dbName=<dbName> <pathToYaml>
10Options:
12 pathToYaml path to a single yaml file or directory of yaml files
13 pathToSettingsFile path to a settings file with logging and database information (yaml file)
14 --host=<host> the database host
15 --user=<user> database user
16 --passwd=<passwd> database user password
17 --dbName=<dbName> name of the database to add the table content to
19 -d, --delete delete yaml open(s) once added to datbase
20 -h, --help show this help message
21 -v, --version show version
22 -s, --settings the settings file
23"""
24from __future__ import print_function
26from builtins import object
27import sys
28import os
29import yaml
30import requests
31import re
32import glob
33import docopt
34from fundamentals import tools, times
35from fundamentals.mysql import convert_dictionary_to_mysql_table
36import requests.packages.urllib3
37requests.packages.urllib3.disable_warnings()
40def main(arguments=None):
41 """
42 The main function used when ``yaml_to_database.py`` when installed as a cl tool
43 """
45 # setup the command-line util settings
46 su = tools(
47 arguments=arguments,
48 docString=__doc__,
49 logLevel="WARNING",
50 options_first=False,
51 projectName=False
52 )
53 arguments, settings, log, dbConn = su.setup()
55 # unpack remaining cl arguments using `exec` to setup the variable names
56 # automatically
57 for arg, val in list(arguments.items()):
58 if arg[0] == "-":
59 varname = arg.replace("-", "") + "Flag"
60 else:
61 varname = arg.replace("<", "").replace(">", "")
62 if isinstance(val, str):
63 exec(varname + " = '%s'" % (val,))
64 else:
65 exec(varname + " = %s" % (val,))
66 if arg == "--dbConn":
67 dbConn = val
68 log.debug('%s = %s' % (varname, val,))
70 if os.path.isfile(pathToYaml):
71 from fundamentals.mysql import yaml_to_database
72 # PARSE YAML FILE CONTENTS AND ADD TO DATABASE
73 yaml2db = yaml_to_database(
74 log=log,
75 settings=settings,
76 dbConn=dbConn
77 )
78 yaml2db.add_yaml_file_content_to_database(
79 filepath=pathToYaml,
80 deleteFile=deleteFlag
81 )
82 basename = os.path.basename(pathToYaml)
83 print("Content of %(basename)s added to database" % locals())
85 else:
86 from fundamentals.mysql import yaml_to_database
87 yaml2db = yaml_to_database(
88 log=log,
89 settings=settings,
90 dbConn=dbConn,
91 pathToInputDir=pathToYaml,
92 deleteFiles=deleteFlag
93 )
94 yaml2db.ingest()
95 print("Content of %(pathToYaml)s directory added to database" % locals())
97 return
100class yaml_to_database(object):
101 """
102 *Take key-values from yaml files including a tablename(s) and add them to a mysql database table*
104 **Key Arguments:**
105 - ``log`` -- logger
106 - ``settings`` -- the settings dictionary
107 - ``pathToInputDir`` -- path to the directory containing the yaml files that will be added to the database table(s). Default *False*
108 - ``dbConn`` -- connection to database to add the content to
109 - ``deleteFiles`` - - delete the yamls files once their content has been added to the database. Default * False*
111 **Usage:**
113 To setup your logger, settings and database connections, please use the ``fundamentals`` package (`see tutorial here <http://fundamentals.readthedocs.io/en/latest/#tutorial>`_).
115 To initiate a ``yaml2db`` object, use the following:
117 .. code-block:: python
119 from fundamentals.mysql import yaml_to_database
120 yaml2db = yaml_to_database(
121 log=log,
122 settings=settings,
123 dbConn=dbConn,
124 pathToInputDir="/path/to/yaml/directory",
125 deleteFiles=False
126 )
128 And here's an example of the content in a yaml file that this ``yaml2db`` object can parse:
130 .. code-block:: yaml
132 title: Why you should do most of your text editing in : Sublime Text | Sublime Text Tips
133 url: http://sublimetexttips.com/why-you-should-do-most-of-your-text-editing-in-sublime-text/?utm_source=drip&utm_medium=email&utm_campaign=editor-proliferation
134 kind: webpage
135 subtype: article
136 table: web_articles,podcasts
137 """
138 # Initialisation
140 def __init__(
141 self,
142 log,
143 dbConn,
144 pathToInputDir=False,
145 settings=False,
146 deleteFiles=False
148 ):
149 self.log = log
150 log.debug("instansiating a new 'yaml_to_database' object")
151 self.settings = settings
152 self.pathToInputDir = pathToInputDir
153 self.dbConn = dbConn
154 self.deleteFiles = deleteFiles
155 # xt-self-arg-tmpx
157 return None
159 def ingest(self):
160 """
161 *ingest the contents of the directory of yaml files into a database*
163 **Return:**
164 - None
166 **Usage:**
168 To import an entire directory of yaml files into a database, use the following:
170 .. code-block:: python
172 from fundamentals.mysql import yaml_to_database
173 yaml2db = yaml_to_database(
174 log=log,
175 settings=settings,
176 dbConn=dbConn,
177 pathToInputDir="/path/to/yaml/directory",
178 deleteFiles=False
179 )
180 yaml2db.ingest()
181 """
182 self.log.debug('starting the ``ingest`` method')
184 for d in os.listdir(self.pathToInputDir):
185 if os.path.isfile(os.path.join(self.pathToInputDir, d)) and "yaml" in d.lower():
186 self.add_yaml_file_content_to_database(
187 filepath=os.path.join(self.pathToInputDir, d),
188 deleteFile=self.deleteFiles
189 )
191 self.log.debug('completed the ``ingest`` method')
192 return None
194 def add_yaml_file_content_to_database(
195 self,
196 filepath,
197 deleteFile=False
198 ):
199 """*given a file to a yaml file, add yaml file content to database*
201 **Key Arguments:**
202 - ``filepath`` -- the path to the yaml file
203 - ``deleteFile`` -- delete the yaml file when its content has been added to the database. Default *False*
205 **Return:**
206 - None
208 **Usage:**
210 To parse and import the contents of a single yaml file into the database, use the following:
212 .. code-block:: python
214 from fundamentals.mysql import yaml_to_database
215 # PARSE YAML FILE CONTENTS AND ADD TO DATABASE
216 yaml2db = yaml_to_database(
217 log=log,
218 settings=settings,
219 dbConn=dbConn
220 )
221 yaml2db.add_yaml_file_content_to_database(
222 filepath=${1:"/path/to/file.yaml"},
223 deleteFile=True
224 )
225 """
226 self.log.debug(
227 'completed the ````add_yaml_file_content_to_database`` method')
229 import codecs
230 try:
231 self.log.debug("attempting to open the file %s" % (filepath,))
232 readFile = codecs.open(filepath, encoding='utf-8', mode='r')
233 thisData = readFile.read()
234 readFile.close()
235 except IOError as e:
236 message = 'could not open the file %s' % (filepath,)
237 self.log.critical(message)
238 raise IOError(message)
239 readFile.close()
241 matchObject = re.finditer(
242 r'(^|\n)(?P<key>[^\:]*)\:\s(?P<value>.*?)(\n|$)',
243 thisData,
244 flags=re.M | re.S # re.S
245 )
247 yamlContent = {}
248 for match in matchObject:
249 if match.group("value")[0] == '"' and match.group("value")[-1] == '"':
250 v = match.group("value")[1:-1]
251 elif match.group("value")[0] == "'" and match.group("value")[-1] == "'":
252 v = match.group("value")[1:-1]
253 else:
254 v = match.group("value")
255 yamlContent[match.group("key")] = v
257 if "table" not in yamlContent:
258 self.log.warning(
259 'A table value is need in the yaml content to indicate which database table to add the content to: %(filepath)s' % locals())
260 return None
262 # NOTE THERE MAY BE MORE THAN ONE DATABASE TABLE
263 dbTablesTmp = yamlContent["table"].split(",")
264 del yamlContent["table"]
265 dbTables = []
266 dbTables[:] = [d.strip() for d in dbTablesTmp]
268 # UNSHORTEN URL
269 try:
270 r = requests.head(yamlContent["url"], allow_redirects=True)
271 yamlContent["url"] = r.url
272 except:
273 pass
275 yamlContent["original_yaml_path"] = filepath
277 if "url" in yamlContent:
278 uniqueKeyList = ["url"]
279 else:
280 uniqueKeyList = []
282 for t in dbTables:
283 convert_dictionary_to_mysql_table(
284 dbConn=self.dbConn,
285 log=self.log,
286 dictionary=yamlContent,
287 dbTableName=t,
288 uniqueKeyList=uniqueKeyList,
289 dateModified=True,
290 returnInsertOnly=False,
291 replace=True
292 )
293 if deleteFile:
294 os.remove(filepath)
296 self.log.debug(
297 'completed the ``add_yaml_file_content_to_database`` method')
298 return None
300 # use the tab-trigger below for new method
301 # xt-class-method
304if __name__ == '__main__':
305 main()