Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/local/bin/python 

2# encoding: utf-8 

3""" 

4*Execute a MySQL write query on a database table* 

5 

6:Author: 

7 David Young 

8 

9:Date Created: 

10 June 21, 2016 

11""" 

12################# GLOBAL IMPORTS #################### 

13from builtins import str 

14import sys 

15import os 

16os.environ['TERM'] = 'vt100' 

17from fundamentals import tools 

18import time 

19 

20 

21def writequery( 

22 log, 

23 sqlQuery, 

24 dbConn, 

25 Force=False, 

26 manyValueList=False 

27): 

28 """*Execute a MySQL write command given a sql query* 

29 

30 **Key Arguments:** 

31 - ``sqlQuery`` -- the MySQL command to execute 

32 - ``dbConn`` -- the db connection 

33 - ``Force`` -- do not exit code if error occurs, move onto the next command 

34 - ``manyValueList`` -- a list of value tuples if executing more than one insert 

35 

36 **Return:** 

37 - ``message`` -- error/warning message 

38 

39 **Usage:** 

40 

41 Here's an example of how to create a table using the database connection passed to the function: 

42 

43 .. code-block:: python  

44 

45 from fundamentals.mysql import writequery 

46 sqlQuery = "CREATE TABLE `testing_table` (`id` INT NOT NULL, PRIMARY KEY (`id`))" 

47 message = writequery( 

48 log=log, 

49 sqlQuery=sqlQuery, 

50 dbConn=dbConn, 

51 Force=False, 

52 manyValueList=False 

53 ) 

54 

55 Here's a many value insert example: 

56 

57 .. code-block:: python  

58 

59 from fundamentals.mysql import writequery 

60 sqlQuery = "INSERT INTO testing_table (id) values (%s)" 

61 message = writequery( 

62 log=log, 

63 sqlQuery=sqlQuery, 

64 dbConn=dbConn, 

65 Force=False, 

66 manyValueList=[(1,), (2,), (3,), (4,), (5,), (6,), (7,), 

67 (8,), (9,), (10,), (11,), (12,), ] 

68 ) 

69 

70 """ 

71 log.debug('starting the ``writequery`` function') 

72 import pymysql 

73 import warnings 

74 warnings.filterwarnings('error', category=pymysql.Warning) 

75 message = "" 

76 try: 

77 cursor = dbConn.cursor(pymysql.cursors.DictCursor) 

78 except Exception as e: 

79 log.error('could not create the database cursor.') 

80 # EXECUTE THE SQL COMMAND 

81 

82 try: 

83 if manyValueList == False: 

84 cursor.execute(sqlQuery) 

85 

86 else: 

87 # cursor.executemany(sqlQuery, manyValueList) 

88 # INSET LARGE LISTS IN BATCHES TO STOP MYSQL SERVER BARFING 

89 batch = 100000 

90 offset = 0 

91 stop = 0 

92 

93 while stop == 0: 

94 thisList = manyValueList[offset:offset + batch] 

95 offset += batch 

96 a = len(thisList) 

97 cursor.executemany(sqlQuery, thisList) 

98 dbConn.commit() 

99 if len(thisList) < batch: 

100 stop = 1 

101 except pymysql.err.ProgrammingError as e: 

102 message = 'MySQL write command not executed for this query: << %s >>\nThe error was: %s \n' % (sqlQuery, 

103 str(e)) 

104 if Force == False: 

105 log.error(message) 

106 raise 

107 else: 

108 log.warning(message) 

109 except pymysql.Error as e: 

110 

111 try: 

112 e = e.args 

113 except: 

114 pass 

115 

116 if e[0] == 1050 and 'already exists' in e[1]: 

117 log.info(str(e) + '\n') 

118 elif e[0] == 1062: 

119 # Duplicate Key error 

120 log.debug('Duplicate Key error: %s\n' % (str(e), )) 

121 message = "duplicate key error" 

122 elif e[0] == 1061: 

123 # Duplicate Key error 

124 log.debug('index already exists: %s\n' % (str(e), )) 

125 message = "index already exists" 

126 elif "Duplicate entry" in str(e): 

127 log.debug('Duplicate Key error: %s\n' % (str(e), )) 

128 message = "duplicate key error" 

129 elif "Deadlock" in str(e): 

130 i = 0 

131 while i < 10: 

132 time.sleep(1) 

133 i += 1 

134 try: 

135 if manyValueList == False: 

136 cursor.execute(sqlQuery) 

137 else: 

138 # cursor.executemany(sqlQuery, manyValueList) 

139 # INSET LARGE LISTS IN BATCHES TO STOP MYSQL SERVER 

140 # BARFING 

141 batch = 100000 

142 offset = 0 

143 stop = 0 

144 

145 while stop == 0: 

146 thisList = manyValueList[offset:offset + batch] 

147 offset += batch 

148 a = len(thisList) 

149 cursor.executemany(sqlQuery, thisList) 

150 dbConn.commit() 

151 if len(thisList) < batch: 

152 stop = 1 

153 i = 20 

154 except: 

155 pass 

156 if i == 10: 

157 log.error('Deadlock: %s\n' % (str(e), )) 

158 message = "Deadlock error" 

159 raise 

160 

161 else: 

162 message = 'MySQL write command not executed for this query: << %s >>\nThe error was: %s \n' % (sqlQuery, 

163 str(e)) 

164 if Force == False: 

165 log.error(message) 

166 raise 

167 else: 

168 log.warning(message) 

169 

170 except pymysql.Warning as e: 

171 log.info(str(e)) 

172 except Exception as e: 

173 if "truncated" in str(e): 

174 log.error('%s\n Here is the sqlquery:\n%s\n' % (str(e), sqlQuery)) 

175 if manyValueList: 

176 log.error('... and the values:\n%s\n' % (thisList, )) 

177 elif "Duplicate entry" in str(e): 

178 log.warning('Duplicate Key error: %s\n' % (str(e), )) 

179 message = "duplicate key error" 

180 else: 

181 log.error( 

182 'MySQL write command not executed for this query: << %s >>\nThe error was: %s \n' % 

183 (sqlQuery, str(e))) 

184 if Force == False: 

185 sys.exit(0) 

186 cursor.close() 

187 return -1 

188 dbConn.commit() 

189 # CLOSE THE CURSOR 

190 cOpen = True 

191 count = 0 

192 while cOpen: 

193 try: 

194 cursor.close() 

195 cOpen = False 

196 except Exception as e: 

197 time.sleep(1) 

198 count += 1 

199 if count == 10: 

200 log.warning('could not close the db cursor ' + str(e) + '\n') 

201 raise e 

202 count = 0 

203 

204 log.debug('completed the ``writequery`` function') 

205 return message