Python指令碼：將Redis資料轉存到Mysql列表中
阿新 • • 發佈：2018-12-10
目錄
一、思路
連線指定的redis和mysql資料庫，從redis中取出資料，然後存到mysql中,中間會遇到幾個問題，在下面的程式碼片段中指出
二、程式碼實現
# coding=utf-8 import json import time import math import redis import pymysql # 忽略警告 import warnings warnings.filterwarnings("ignore") def main(): # 指定redis資料庫資訊 redis_cli = redis.StrictRedis(host='127.0.0.1', port=6379, db=0) # 指定mysql資料庫資訊 mysql_cli = pymysql.connect(host='localhost', port=3306, user="root", password="mysql", db="test", charset="utf8") # 新增表唯一索引 mysql_cli.cursor().execute("ALTER TABLE `tb_stop_record` ADD unique(`unique_id`)") page_start = 0 # 每頁取值數量 step = 1000 page_end = page_start+step-1 # 當前頁碼 page_num = 0 # 新增資料量 add_num = 0 # 真實插入資料量 total_num = 0 # 計算總頁數,向上取整 sum_page = math.ceil(redis_cli.llen('test3')/(step)) for i in xrange(int(sum_page)): # 從redis中提取值的列表 data_list = redis_cli.lrange('test3', start=page_start, end=page_end) if data_list: page_start = page_end+1 page_end = page_start+step page_num = i+1 params = [] for data in data_list: # data裡的資料不一定都是json格式,所以為了防止程式中斷,考慮到 # 容錯性,這裡捕獲異常 try: dict_data = json.loads(data) except Exception as e: print '資料型別錯誤',e # 對要新增的資料進行篩選,過濾 if (dict_data.get('session_id') and dict_data.get('member_name') and dict_data.get('stop_time') and dict_data.get('nas_ip') and dict_data.get('game_id')): # 如果滿足條件則新增整條資料 add_num += 1 params.append(([dict_data.get('session_id'), dict_data.get('unique_id'), dict_data.get('member_name'), dict_data.get('mobile'), "", "",dict_data.get('nas_ip'), dict_data.get('nas_port_id'), dict_data.get('nas_port_type'),dict_data.get('start_time'), dict_data.get('update_time'), dict_data.get('stop_time'), 0, 0, dict_data.get('authentic'), "", "", dict_data.get('input_octets'), dict_data.get('output_octets'), dict_data.get('called_station_id'), dict_data.get('calling_station_id'), dict_data.get('terminate_cause'), dict_data.get('service_type'), dict_data.get('framed_protocol'), dict_data.get('framed_ip'),0, 0, "", "", dict_data.get('game_id'), dict_data.get('sn'), dict_data.get('dev_type')])) else: print '資料缺失' continue try: # 獲取遊標 cursor = mysql_cli.cursor() # insert ignore into在執行插入的時候,防止重複新增 sql = "insert ignore into tb_stop_record(session_id,unique_id,member_name," \ "mobile,group_name,realm,nas_ip,nas_port_id,nas_port_type,start_time,update_time,stop_time,"\ "acct_interval,acctsession_time,authentic,connectinfo_start,connectinfo_stop,input_octets,output_octets," \ "called_station_id,calling_station_id,terminate_cause,service_type,framed_protocol,framed_ip," \ "acctstartdelay,acctstopdelay,xascendsessionsvrkey,userrequestoffline,game_id,sn,dev_type) " \ "VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" # 執行批量插入資料,返回每次插入的數量 insert_num = cursor.executemany(sql, params) # 實際上轉存到Mysql的資料量 total_num += insert_num mysql_cli.commit() # 關閉遊標 cursor.close() except pymysql.Error as e: mysql_cli.rollback() print("插入資料錯誤", e) print '當前頁是第{}頁'.format(page_num) else: break print '總計新增資料{}條'.format(add_num) print '實際插入量{}條'.format(total_num) if __name__ == '__main__': start_time = time.clock() main() end_time = time.clock() print "耗時{}s".format(end_time-start_time)