优化消耗数据拉取

dgp/get_data_hourly.py

@@ -5,7 +5,7 @@ Created on Thu Jun  4 15:06:05 2020
 
 @author: chencong
 """
-
+import sys
 import requests
 import hashlib
 import time
@@ -15,10 +15,11 @@ import random
 import token_list as tl
 import smtplib
 from email.mime.text import MIMEText
-from datetime import datetime
-import importlib
-importlib.reload(tl)
+from datetime import datetime,timedelta
 from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime
+
+
 
 
 def md5value(s):
@@ -42,8 +43,8 @@ def daily_reports_get(access_token, account_id, level, fields):
 		"level": level,
 		"date_range":
 			{
-				"start_date": time.strftime("%Y-%m-%d", time.localtime()),
-				"end_date": time.strftime("%Y-%m-%d", time.localtime())
+				"start_date": my_time,
+				"end_date": my_time
 			},
 		"page": 1,
 		"page_size": 1000,
@@ -85,20 +86,23 @@ def get_v_data(y,li):
 def get_daily_vx():
 	token_list_v = tl.token_list_vx
 	time1 = time.time()
-	executor = ThreadPoolExecutor(max_workers=30)
+	executor = ThreadPoolExecutor(max_workers=max_workers)
 	li=[]
 	for y in token_list_v:
+
 		executor.submit(get_v_data,y,li)
 	executor.shutdown()
 	print('get_daily_vx:', len(li), 'cost:', int(time.time()-time1))
+
 	return li
 
 def get_daily_qq():
 	token_list_q = tl.token_list_qq
 	time1 = time.time()
 	li=[]
-	executor = ThreadPoolExecutor(max_workers=30)
+	executor = ThreadPoolExecutor(max_workers=max_workers)
 	for x in token_list_q:
+
 		executor.submit(get_q_data,x,li)
 	executor.shutdown()
 
@@ -126,14 +130,14 @@ def mysql_insert_daily_vx(data):
 	db = pymysql.connect('rm-bp1c9cj79872tx3aaro.mysql.rds.aliyuncs.com', 'superc', 'Cc719199895', 'quchen_text')
 	cursor = db.cursor()
 	time1 = time.time()
-	sql1 = 'delete from daily_vx where date = %s'
-	sql2 = 'insert ignore into daily_vx (date,cost,view_count,valid_click_count,ctr,official_account_follow_rate,order_amount,order_roi,order_count,order_rate,order_unit_price,web_order_cost,first_day_order_amount,first_day_order_count,account_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
-	try:
-		cursor.execute(sql1, str(time.strftime("%Y-%m-%d", time.localtime())))
-		db.commit()
-		print('clear_daily_vx access')
-	except:
-		print('clear_daily_vx defeat')
+	# sql1 = 'delete from daily_vx where date = %s'
+	sql2 = 'replace into daily_vx (date,cost,view_count,valid_click_count,ctr,official_account_follow_rate,order_amount,order_roi,order_count,order_rate,order_unit_price,web_order_cost,first_day_order_amount,first_day_order_count,account_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
+	# try:
+	# 	cursor.execute(sql1, str(time.strftime("%Y-%m-%d", time.localtime())))
+	# 	db.commit()
+	# 	print('clear_daily_vx access')
+	# except:
+	# 	print('clear_daily_vx defeat')
 	try:
 		cursor.executemany(sql2, data)
 		db.commit()
@@ -148,14 +152,14 @@ def mysql_insert_daily_qq(data):
 	db = pymysql.connect('rm-bp1c9cj79872tx3aaro.mysql.rds.aliyuncs.com', 'superc', 'Cc719199895', 'quchen_text')
 	cursor = db.cursor()
 	time1 = time.time()
-	sql1 = 'delete from daily_qq where date = %s'
-	sql2 = 'insert ignore into daily_qq (date,view_count,valid_click_count,ctr,cpc,cost,web_order_count,web_order_rate,web_order_cost,follow_count,order_amount,order_roi,platform_page_view_count,web_commodity_page_view_count,from_follow_uv,account_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
-	try:
-		cursor.execute(sql1, str(time.strftime("%Y-%m-%d", time.localtime())))
-		db.commit()
-		print('clear_daily_qq access')
-	except:
-		print('clear_daily_qq defeat')
+	# sql1 = 'delete from daily_qq where date = %s'
+	sql2 = 'replace into daily_qq (date,view_count,valid_click_count,ctr,cpc,cost,web_order_count,web_order_rate,web_order_cost,follow_count,order_amount,order_roi,platform_page_view_count,web_commodity_page_view_count,from_follow_uv,account_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
+	# try:
+	# 	cursor.execute(sql1, str(time.strftime("%Y-%m-%d", time.localtime())))
+	# 	db.commit()
+	# 	print('clear_daily_qq access')
+	# except:
+	# 	print('clear_daily_qq defeat')
 	try:
 		cursor.executemany(sql2, data)
 		db.commit()
@@ -189,20 +193,19 @@ def send_waring_email():
 
 
 def start_cost_job():
-	st_unix_time = time.time()
-	st_unix = int((st_unix_time + 8 * 3600) // 3600 * 3600 - 8 * 3600 - 3600)
-	et_unix = int((st_unix_time + 8 * 3600) // 3600 * 3600 - 8 * 3600)
-	# et_unix = et_unix - 1
-	st_dt = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(st_unix)) + '+08:00'
-	et_dt = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(et_unix)) + '+08:00'
-	print(st_dt, et_dt)
 	mysql_insert_daily_vx(get_daily_vx())
 	mysql_insert_daily_qq(get_daily_qq())
 
 
 
 if __name__ == '__main__':
-	print(datetime.today())
+	print("start_at ===================="+str(datetime.today())+"===================")
+	max_workers = 5
+	my_time = datetime.today().strftime('%Y-%m-%d')
+	if sys.argv.__len__() ==2:
+		yestoday=(datetime.today()-timedelta(days=1)).strftime('%Y-%m-%d')
+		my_time=yestoday
+		print("跑昨天["+yestoday+"]数据")
 	start_cost_job()