code-record--mysql deep connect
实现了Mysql短线重连而不是为确保安全每次写mysql都要连接一次。
用递归的方式实现Mysql连接函数失败时自动重连,失败次数太大时发出提示。
尽在不言中,贴之
import bs4
import MySQLdb
from bs4 import BeautifulSoup
import time
import sys
import datetime
__OUT__ = sys.stdout
#try to connect mysql, it will alert you when connect fail more than five times
def _connect_mysql(deep):
if deep > 5:
__out__ = sys.stdout
sys.stdout = __OUT__
print 'Maybe mysql server can not work'
sys.stdout = __out__
time.sleep(10)
try:
conn = MySQLdb.connect(host='',user='',passwd='',db='',port=)
except:
time.sleep(0.5)
return _connect_mysql(deep+1)
else:
return conn
#try to write infoomation to mysql, it will return True if it write success,else return False
def _write_mysql(con, _problem_id, _title, _description, _input, _output, _sample_input, _sample_output, _author, _recommend, _info_temp):
try:
cur = con.cursor()
cur.execute('set names utf8')
except:
return False
else:
sql = ('INSERT INTO `hdu_problem`'
'(`problem_id`, `title`, `description`, `input`, `output`, `sample_input`,'
' `sample_output`, `author`, `recommend`, `info_temp`, `get_time`)'
' VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)',
(_problem_id, _title.encode('utf-8'), _description.encode('utf-8'),
_input.encode('utf-8'), _output.encode('utf-8'), _sample_input.encode('utf-8'),
_sample_output.encode('utf-8'), _author.encode('utf-8'), _recommend.encode('utf-8'),
_info_temp.encode('utf-8'), datetime.datetime.now()))
cur.execute(*sql)
con.commit()
return True
number = 1000
con = _connect_mysql(0)
while True:
print number
content = open(str(number) + '.html')
soup = BeautifulSoup(content, 'html.parser')
_title = soup.h1.text
if _title == 'System Message':
print _title
else:
_info_temp = soup.find_all(style='font-family:Arial;font-size:12px;font-weight:bold;color:green')[0].text
content = soup.find_all(class_='panel_title')
for i in content:
item = i.find_next_sibling()
if i.text == 'Problem Description':
del(item['class'])
img = item.find_all('img')
if img!=[]:
for one_img in img:
if one_img.attrs['src'][0] == '.':
img_url_head = 'http://acm.hdu.edu.cn/'
else:
img_url_head = 'http://acm.hdu.edu.cn'
one_img.attrs['src'] = img_url_head + one_img.attrs['src']
_description = item
elif i.text == 'Input':
del(item['class'])
_input= item
elif i.text == 'Output':
del(item['class'])
_output= item
elif i.text == 'Sample Input':
del(item['class'])
_sample_input= item
elif i.text == 'Sample Output':
del(item['class'])
_sample_output= item
elif i.text == 'Author':
del(item['class'])
_author= item
elif i.text == 'Recommend':
del(item['class'])
_recommend= item
flag = False
_times = 1
while True:
print 'I will write to mysql, try' , _times
if _write_mysql(con, number, _title, _description, _input, _output, _sample_input, _sample_output, _author, _recommend, _info_temp):
print 'write success'
break
else:
#reconnect tomysql when lost connection
_times += 1
con = _connect_mysql(0)
number += 1