Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/local/bin/python 

2# encoding: utf-8 

3""" 

4*Given a directory of MySQL scripts, execute the scripts and process the script files according to their success or failure* 

5 

6Run the following code once to set a login-path for your mysql server: 

7 

8 mysql_config_editor set --login-path=<uniqueLoginName> --host=localhost --user=<myUsername> --password 

9 

10This store's your credentials in an encrypted file located at '~/.mylogin.cnf'.  

11Use `mysql_config_editor print --all` to see all of the login-paths set. 

12 

13 

14Usage: 

15 mysqlSucker <pathToDirectory> <loginPath> <databaseName> [-s successRule -f failureRule] 

16 

17 pathToDirectory path to the directory containing the sql scripts to run (scripts must have `.sql` extension) 

18 loginPath the local-path as set with `mysql_config_editor` (`mysqlSucker -h` for more details) 

19 databaseName the name of the database to execute the scripts within  

20  

21Options: 

22 -h, --help show this help message 

23 -s successRule, --success successRule what to do if script succeeds. Default *None* [None|delete|subFolderName] 

24 -f failureRule, --failure failureRule what to do if script fails. Default *None* [None|delete|subFolderName]  

25 

26:Examples: 

27 

28 To simply execute the scripts in a directory you can run: 

29 

30 mysqlSucker /path/to/scriptdir myLoginPath myDatabaseName 

31 

32 To delete script after thay have executed successfully: 

33 

34 mysqlSucker /path/to/scriptdir myLoginPath myDatabaseName -s delete 

35 

36 To move successful script to a `passed` sub-directory of `/path/to/scriptdir` and failed scripts to a `failed` sub-directory 

37 

38 mysqlSucker /path/to/scriptdir myLoginPath myDatabaseName -s pass -f failed 

39 

40:Author: 

41 David Young 

42 

43:Date Created: 

44 September 22, 2016 

45""" 

46################# GLOBAL IMPORTS #################### 

47from builtins import str 

48import sys 

49import os 

50import time 

51import collections 

52import datetime 

53from subprocess import Popen, PIPE, STDOUT 

54os.environ['TERM'] = 'vt100' 

55from fundamentals import tools 

56 

57 

58def directory_script_runner( 

59 log, 

60 pathToScriptDirectory, 

61 databaseName, 

62 loginPath, 

63 force=True, 

64 waitForResult=True, 

65 successRule=None, 

66 failureRule=None): 

67 """A function to run all of the mysql scripts in a given directory (in a modified date order, oldest first) and then act on the script file in accordance with the succcess or failure of its execution 

68 

69 As it's insecure to pass in mysql database credentials via the command-line, run the following command from the terminal  

70 

71 .. code-block:: bash 

72 

73 mysql_config_editor set --login-path=<uniqueLoginName> --host=localhost --user=<myUsername> --password 

74 > Enter password: 

75 

76 This will store your credentials in an encrypted file located at '~/.mylogin.cnf'. This function takes advantage of the `--login-path` so as not to compromise the user's credentials. Use `mysql_config_editor print --all` to see all of the login-paths set. 

77 

78 **Key Arguments:** 

79 - ``log`` -- logger 

80 - ``pathToScriptDirectory`` -- the path to the directory containing the sql script to be run 

81 - ``databaseName`` -- the name of the database  

82 - ``force`` -- force the script to run, skipping over lines with errors, Default *True* 

83 - ``loginPath`` -- the local-path as set with `mysql_config_editor` 

84 - ``waitForResult`` -- wait for the mysql script to finish execution? If 'False' the MySQL script will run in background (do not wait for completion), or if 'delete' the script will run then delete regardless of success status. Default *True*. [True|False|delete] 

85 - ``successRule`` -- what to do if script succeeds. Default *None* [None|delete|subFolderName] 

86 - ``failureRule`` -- what to do if script fails. Default *None* [None|delete|subFolderName] 

87 

88 **Return:** 

89 - None 

90 

91 **Usage:** 

92 

93 To run the scripts in the directroy and not act on the script file use something similar to: 

94 

95 .. code-block:: python  

96 

97 from fundamentals.mysql import directory_script_runner 

98 directory_script_runner( 

99 log=log, 

100 pathToScriptDirectory="/path/to/mysql_scripts", 

101 databaseName="imports", 

102 loginPath="myLoginDetails" 

103 ) 

104 

105 To delete successful scripts and archive failed scripts for later inspection: 

106 

107 .. code-block:: python  

108 

109 from fundamentals.mysql import directory_script_runner 

110 directory_script_runner( 

111 log=log, 

112 pathToScriptDirectory="/path/to/mysql_scripts", 

113 databaseName="imports", 

114 loginPath="myLoginDetails", 

115 successRule="delete", 

116 failureRule="failed" 

117 ) 

118 

119 This creates a folder at `/path/to/mysql_scripts/failed` and moves the failed scripts into that folder. 

120 

121 Finally to execute the scripts within a directory but not wait for the results to return (much fast but you lose error checking in the MySQL scripts): 

122 

123 .. code-block:: python  

124 

125 from fundamentals.mysql import directory_script_runner 

126 directory_script_runner( 

127 log=log, 

128 pathToScriptDirectory="/path/to/mysql_scripts", 

129 databaseName="imports", 

130 loginPath="myLoginDetails", 

131 waitForResults=False 

132 ) 

133 

134 Setting ``waitForResults`` = 'delete' will trash the script once it has run (or failed ... be very careful!) 

135 """ 

136 log.debug('starting the ``directory_script_runner`` function') 

137 

138 # COMPILE A DICTIONARY OF SCRIPTS / MODIFIED TIMES 

139 scriptList = {} 

140 for d in os.listdir(pathToScriptDirectory): 

141 filePath = os.path.join(pathToScriptDirectory, d) 

142 filename = os.path.basename(filePath) 

143 extension = filePath.split(".")[-1] 

144 if os.path.isfile(filePath) and extension == "sql": 

145 modified = datetime.datetime.strptime(time.ctime( 

146 os.path.getmtime(filePath)), "%a %b %d %H:%M:%S %Y") 

147 scriptList[str(modified) + filename] = filePath 

148 

149 # FORCE MYSQL SCRIPT? 

150 if force: 

151 force = "--force" 

152 else: 

153 force = "" 

154 

155 # ORDER THE DICTIONARY BY MODIFIED TIME - OLDEST FIRST 

156 scriptList = collections.OrderedDict(sorted(scriptList.items())) 

157 for k, v in list(scriptList.items()): 

158 scriptname = os.path.basename(v) 

159 if waitForResult == True: 

160 cmd = """mysql --login-path=%(loginPath)s %(force)s %(databaseName)s < "%(v)s" """ % locals( 

161 ) 

162 p = Popen(cmd, stdout=PIPE, stderr=PIPE, close_fds=True, 

163 env={'PATH': os.getenv('PATH') + ":/usr/local/bin:/usr/bin:/usr/bin:/usr/local/mysql/bin", "MYSQL_TEST_LOGIN_FILE": os.getenv('HOME') + "/.mylogin.cnf"}, shell=True) 

164 stdout, stderr = p.communicate() 

165 

166 if len(stderr): 

167 log.error( 

168 "MySQL Script `%(scriptname)s` Failed: '%(stderr)s'" % locals()) 

169 if failureRule == None or failureRule == False: 

170 pass 

171 elif failureRule == "delete": 

172 os.remove(v) 

173 elif "/" not in failureRule: 

174 moveTo = pathToScriptDirectory + "/" + failureRule 

175 # Recursively create missing directories 

176 if not os.path.exists(moveTo): 

177 os.makedirs(moveTo) 

178 moveTo = moveTo + "/" + scriptname 

179 try: 

180 log.debug("attempting to rename file %s to %s" % 

181 (v, moveTo)) 

182 os.rename(v, moveTo) 

183 except Exception as e: 

184 log.error( 

185 "could not rename file %s to %s - failed with this error: %s " % (v, moveTo, str(e),)) 

186 else: 

187 if successRule == None or successRule == False: 

188 pass 

189 elif successRule == "delete": 

190 os.remove(v) 

191 elif "/" not in successRule: 

192 moveTo = pathToScriptDirectory + "/" + successRule 

193 # Recursively create missing directories 

194 if not os.path.exists(moveTo): 

195 os.makedirs(moveTo) 

196 moveTo = moveTo + "/" + scriptname 

197 try: 

198 log.debug("attempting to rename file %s to %s" % 

199 (v, moveTo)) 

200 os.rename(v, moveTo) 

201 except Exception as e: 

202 log.error( 

203 "could not rename file %s to %s - failed with this error: %s " % (v, moveTo, str(e),)) 

204 else: 

205 if waitForResult == "delete": 

206 cmd = """mysql --login-path=%(loginPath)s %(force)s %(databaseName)s < "%(v)s" > /dev/null 2>&1 & rm "%(v)s" """ % locals() 

207 else: 

208 cmd = """mysql --login-path=%(loginPath)s %(force)s %(databaseName)s < "%(v)s" > /dev/null 2>&1 """ % locals() 

209 p = Popen(cmd, close_fds=True, 

210 env={'PATH': os.getenv('PATH') + ":/usr/local/bin:/usr/bin:", "MYSQL_TEST_LOGIN_FILE": os.getenv('HOME') + "/.mylogin.cnf"}, shell=True, stdin=None, stdout=None, stderr=None) 

211 

212 log.debug('completed the ``directory_script_runner`` function') 

213 return None 

214 

215 

216def main(arguments=None): 

217 """ 

218 The main function used when ``directory_script_runner.py`` is run as a single script from the cl, or when installed as a cl command 

219 """ 

220 

221 # setup the command-line util settings 

222 su = tools( 

223 arguments=arguments, 

224 docString=__doc__, 

225 logLevel="WARNING", 

226 options_first=False, 

227 projectName="fundmentals" 

228 ) 

229 arguments, settings, log, dbConn = su.setup() 

230 

231 # UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES 

232 # AUTOMATICALLY 

233 for arg, val in list(arguments.items()): 

234 if arg[0] == "-": 

235 varname = arg.replace("-", "") + "Flag" 

236 else: 

237 varname = arg.replace("<", "").replace(">", "") 

238 if isinstance(val, ("".__class__, u"".__class__)): 

239 exec(varname + " = '%s'" % (val,)) 

240 else: 

241 exec(varname + " = %s" % (val,)) 

242 if arg == "--dbConn": 

243 dbConn = val 

244 log.debug('%s = %s' % (varname, val,)) 

245 

246 if successFlag and successFlag.lower() == "none": 

247 successFlag = None 

248 if failureFlag and failureFlag.lower() == "none": 

249 failureFlag = None 

250 

251 directory_script_runner( 

252 log=log, 

253 pathToScriptDirectory=pathToDirectory, 

254 databaseName=databaseName, 

255 loginPath=loginPath, 

256 successRule=successFlag, 

257 failureRule=failureFlag 

258 ) 

259 

260 return 

261 

262if __name__ == '__main__': 

263 main()