get_data_hourly.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """
  4. Created on Thu Jun 4 15:06:05 2020
  5. @author: chencong
  6. """
  7. import requests
  8. import hashlib
  9. import time
  10. from urllib import parse
  11. import datetime
  12. import json
  13. import math
  14. import pymysql
  15. import random
  16. from apscheduler.schedulers.blocking import BlockingScheduler
  17. import account_list as al
  18. import token_list as tl
  19. import smtplib
  20. from email.mime.text import MIMEText
  21. def md5value(s):
  22. md5 = hashlib.md5()
  23. md5.update(s.encode("utf-8"))
  24. return md5.hexdigest()
  25. def get_yuewen_order(st, et):
  26. t = ()
  27. time1 = time.time()
  28. url = 'https://open.yuewen.com/cpapi/wxRecharge/querychargelog'
  29. version = 1
  30. secert_list = al.yuewen_account_list
  31. for secert in secert_list:
  32. start_time = st
  33. end_time = et
  34. email = secert[0]
  35. appsecert = secert[1]
  36. for i in range((et - st) // 86400 + 1):
  37. timestamp = int(time.time())
  38. s = ''
  39. page = 1
  40. order_status = 2
  41. data = {
  42. 'email': email,
  43. 'version': version,
  44. 'timestamp': timestamp,
  45. 'start_time': start_time,
  46. 'end_time': end_time,
  47. 'page': page,
  48. 'order_status': order_status
  49. }
  50. # 'last_min_id':last_min_id,
  51. # 'last_max_id':last_max_id,
  52. # 'total_count':total_count,
  53. # 'last_page':last_page}
  54. sorted_data = sorted(data.items())
  55. for k, v in sorted_data:
  56. s = s + str(k) + str(v)
  57. sign = md5value(appsecert + s).upper()
  58. data1 = {
  59. 'email': email,
  60. 'version': version,
  61. 'timestamp': timestamp,
  62. 'start_time': start_time,
  63. 'end_time': end_time,
  64. 'page': page,
  65. 'order_status': order_status,
  66. 'sign': sign
  67. }
  68. list1 = requests.get(url=url, params=data1)
  69. total_count = list1.json()['data']['total_count']
  70. last_min_id = list1.json()['data']['min_id']
  71. last_max_id = list1.json()['data']['max_id']
  72. last_page = list1.json()['data']['page']
  73. if total_count > 0:
  74. for x in list1.json()['data']['list']:
  75. y = {}
  76. dtime = datetime.datetime.strptime(x['order_time'], "%Y-%m-%d %H:%M:%S")
  77. y['date'] = ((int(time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
  78. y['platform'] = '阅文'
  79. y['channel'] = x['app_name']
  80. y['from_novel'] = x['book_name']
  81. y['user_id'] = x['openid']
  82. y['stage'] = ''
  83. y['channel_id'] = 0
  84. y['order_time'] = x['order_time']
  85. y['amount'] = x['amount']
  86. y['reg_time'] = x['reg_time']
  87. y['order_id'] = x['order_id']
  88. """
  89. del x['app_name']
  90. del x['order_status']
  91. del x['order_type']
  92. del x['openid']
  93. del x['user_name']
  94. del x['sex']
  95. del x['channel_name']
  96. del x['book_id']
  97. del x['book_name']
  98. del x['report_status']
  99. if y['order_time'] != time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(et)):
  100. """
  101. y = sorted(y.items(), key=lambda item: item[0])
  102. y = dict(y)
  103. y = tuple(y.values())
  104. t = t + ((y),)
  105. if total_count > 100:
  106. for page in range(2, math.ceil(total_count / 100) + 1):
  107. data = {
  108. 'email': email,
  109. 'version': version,
  110. 'timestamp': timestamp,
  111. 'start_time': start_time,
  112. 'end_time': end_time,
  113. 'page': page,
  114. 'last_min_id': last_min_id,
  115. 'last_max_id': last_max_id,
  116. 'total_count': total_count,
  117. 'last_page': last_page,
  118. 'order_status': order_status
  119. }
  120. sorted_data = sorted(data.items())
  121. s1 = ''
  122. for k, v in sorted_data:
  123. s1 = s1 + str(k) + str(v)
  124. sign = md5value(appsecert + s1).upper()
  125. data2 = {
  126. 'email': email,
  127. 'version': version,
  128. 'timestamp': timestamp,
  129. 'start_time': start_time,
  130. 'end_time': end_time,
  131. 'page': page,
  132. 'last_min_id': last_min_id,
  133. 'last_max_id': last_max_id,
  134. 'total_count': total_count,
  135. 'last_page': last_page,
  136. 'order_status': order_status,
  137. 'sign': sign
  138. }
  139. list2 = requests.get(url=url, params=data2)
  140. for x in list2.json()['data']['list']:
  141. y = {}
  142. dtime = datetime.datetime.strptime(x['order_time'], "%Y-%m-%d %H:%M:%S")
  143. y['date'] = ((int(time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
  144. y['platform'] = '阅文'
  145. y['channel'] = x['app_name']
  146. y['from_novel'] = x['book_name']
  147. y['user_id'] = x['openid']
  148. y['stage'] = ''
  149. y['channel_id'] = 0
  150. y['order_time'] = x['order_time']
  151. y['amount'] = x['amount']
  152. y['reg_time'] = x['reg_time']
  153. y['order_id'] = x['order_id']
  154. """
  155. del x['report_status']
  156. del x['app_name']
  157. del x['order_status']
  158. del x['order_type']
  159. del x['openid']
  160. del x['user_name']
  161. del x['sex']
  162. del x['channel_name']
  163. del x['book_id']
  164. del x['book_name']
  165. if y['order_time'] != time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(et)):
  166. """
  167. y = sorted(y.items(), key=lambda item: item[0])
  168. y = dict(y)
  169. y = tuple(y.values())
  170. t = t + ((y),)
  171. total_count = list2.json()['data']['total_count']
  172. last_min_id = list2.json()['data']['min_id']
  173. last_max_id = list2.json()['data']['max_id']
  174. last_page = list2.json()['data']['page']
  175. cost_time = round((time.time() - time1) / 60, 1)
  176. print('get_yuewen_order:', len(t), 'cost_minutes:', cost_time)
  177. return t
  178. def get_zhangdu_order(st, et):
  179. time1 = time.time()
  180. secert_list = al.zhangdu_account_list
  181. t = ()
  182. url = 'https://api.zhangdu520.com/channel/getorder'
  183. for item in secert_list: # 分渠道
  184. uid = item[0]
  185. appsecert = item[1]
  186. channel = item[2]
  187. timestamp = int(time.time())
  188. sign = md5value(str(uid) + '&' + appsecert + '&' + str(timestamp))
  189. page = 1
  190. starttime = st
  191. endtime = et
  192. for x in range((et - st) // 86400 + 1): # 分时段
  193. Params = {'uid': uid,
  194. 'timestamp': timestamp,
  195. 'sign': sign,
  196. 'starttime': starttime,
  197. 'endtime': endtime
  198. }
  199. list1 = requests.get(url=url, params=Params)
  200. pageCount = list1.json()['data']['pageCount']
  201. if pageCount > 0:
  202. for a in range(1, pageCount + 1): # 分页
  203. page = a
  204. Params = {
  205. 'uid': uid,
  206. 'timestamp': timestamp,
  207. 'sign': sign,
  208. 'starttime': starttime,
  209. 'endtime': endtime,
  210. 'page': page
  211. }
  212. list2 = requests.get(url=url, params=Params).json()['data']['list']
  213. for b in list2:
  214. c = {}
  215. c['channel_id'] = uid
  216. c['order_id'] = str(b['orderno'])
  217. c['order_time'] = b['ctime']
  218. c['user_id'] = b['openid']
  219. c['platform'] = '掌读'
  220. c['channel'] = channel
  221. c['stage'] = ''
  222. c['from_novel'] = ''
  223. c['reg_time'] = b['regtime']
  224. c['date'] = ((int(b['ctime']) + 8 * 3600) // 86400) * 86400 - 8 * 3600
  225. c['amount'] = b['amount']
  226. """
  227. del b['openid']
  228. del b['regtime']
  229. del b['ip']
  230. del b['ua']
  231. del b['id']
  232. del b['ctime']
  233. del b['userid']
  234. del b['orderno']
  235. del b['source']
  236. del b['sourceid']
  237. """
  238. if b['status'] == '1' and int(c['order_time']) < et:
  239. del b['status']
  240. x = sorted(c.items(), key=lambda item: item[0])
  241. x = dict(x)
  242. x = tuple(x.values())
  243. t = t + ((x),)
  244. cost_time = round((time.time() - time1) / 60, 1)
  245. print('get_zhangdu_order:', len(t), 'cost_minutes:', cost_time)
  246. return t
  247. def get_huasheng_order(st, et):
  248. time1 = time.time()
  249. apikey_list = al.huasheng_account_list
  250. url = 'https://vip.rlcps.cn/api/getMerchants'
  251. t = ()
  252. for key in apikey_list: # 获取每个vip账号下的channel_id
  253. apiKEY = key[0]
  254. apiSecurity = key[1]
  255. stage = key[2]
  256. timestamp = str(int(time.time()))
  257. sign = md5value(apiKEY + timestamp + apiSecurity).upper()
  258. data = {'apiKey': apiKEY,
  259. 'apiSecurity': apiSecurity,
  260. 'timestamp': timestamp,
  261. 'sign': sign
  262. }
  263. list0 = requests.post(url, data).json()
  264. for merchant in list0['data']:
  265. merchant_id = merchant['merchant_id']
  266. merchant_name = merchant['merchant_name']
  267. url1 = 'https://vip.rlcps.cn/api/orderList'
  268. start_time = st
  269. for i in range((et - st) // 86400 + 1):
  270. date = time.strftime("%Y-%m-%d", time.localtime(start_time))
  271. sign = md5value(apiKEY + date + str(merchant_id) + timestamp + apiSecurity).upper()
  272. page = 1
  273. data1 = {
  274. 'apiKey': apiKEY,
  275. 'apiSecurity': apiSecurity,
  276. 'timestamp': timestamp,
  277. 'date': date,
  278. 'merchant_id': merchant_id,
  279. 'sign': sign,
  280. 'page': page
  281. }
  282. list1 = requests.post(url1, data1).json()
  283. if 'data' in list1.keys() and len(list1['data']) > 0:
  284. for i in range(int(math.ceil(list1['count'] / 500))):
  285. data2 = {
  286. 'apiKey': apiKEY,
  287. 'apiSecurity': apiSecurity,
  288. 'timestamp': timestamp,
  289. 'date': date,
  290. 'merchant_id': merchant_id,
  291. 'sign': sign,
  292. 'page': page
  293. }
  294. list2 = requests.post(url1, data2).json()
  295. for x in list2['data']:
  296. if x['order_status'] == 1:
  297. t1 = time.strptime(x['pay_at'], "%Y-%m-%d %H:%M:%S")
  298. ts = int(time.mktime(t1))
  299. if ts >= st and ts < et:
  300. y = {}
  301. # dtime = datetime.datetime.strptime(x['pay_at'],"%Y-%m-%d")
  302. # y['date']= ((int(time.mktime(dtime.timetuple()))+8*3600)//86400)*86400-8*3600
  303. y['order_id'] = x['trans_id']
  304. y['order_time'] = x['pay_at']
  305. y['reg_time'] = x['join_at']
  306. y['date'] = (start_time + 8 * 3600) // 86400 * 86400 - 8 * 3600
  307. y['channel'] = merchant_name
  308. y['channel_id'] = merchant_id
  309. y['from_novel'] = x['book_name']
  310. y['platform'] = '花生'
  311. y['stage'] = stage
  312. y['user_id'] = x['openid']
  313. y['amount'] = x['amount']
  314. """
  315. del x['order_num']
  316. del x['book_name']
  317. del x['trans_id']
  318. del x['pay_at']
  319. del x['join_at']
  320. del x['subscribe_at']
  321. del x['openid']
  322. del x['charge_count']
  323. del x['book_id']
  324. del x['order_status']
  325. del x['user_name']
  326. del x['spread_name']
  327. del x['request_at']
  328. if y['order_time'] != time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(et)):
  329. """
  330. y = sorted(y.items(), key=lambda item: item[0])
  331. y = dict(y)
  332. y = tuple(y.values())
  333. t = t + ((y),)
  334. page = page + 1
  335. cost_time = round((time.time() - time1) / 60, 1)
  336. print('get_huasheng_order:', len(t), 'cost_minutes:', cost_time)
  337. return t
  338. def get_zzy_order(st, et):
  339. time1 = time.time()
  340. API_list = al.zzy_account_list
  341. r = ()
  342. for x in API_list:
  343. my_key = x[0]
  344. secert = x[1]
  345. stage = x[2]
  346. my_sign = md5value(secert + 'key=' + my_key)
  347. #url = 'https://openapi.818tu.com/partners/channel/channels/list?'
  348. url = 'https://inovel.818tu.com/partners/channel/channels/list?'
  349. parameter = 'key=' + my_key + '&sign=' + my_sign
  350. channel_list = requests.get(url + parameter) # 获取子渠道列表
  351. if 'data' in channel_list.json().keys():
  352. items = channel_list.json()['data']['items']
  353. elif len(x) > 3:
  354. my_key = x[3]
  355. secert = x[4]
  356. my_sign = md5value(secert + 'key=' + my_key)
  357. #url = 'https://openapi.818tu.com/partners/channel/channels/list?'
  358. url = 'https://inovel.818tu.com/partners/channel/channels/list?'
  359. parameter = 'key=' + my_key + '&sign=' + my_sign
  360. channel_list = requests.get(url + parameter)
  361. items = channel_list.json()['data']['items']
  362. else:
  363. print(channel_list.json())
  364. items = []
  365. if items != []:
  366. for item in items: # 获取channel_id 后逐个拉取历史orders
  367. channel_id = item['id']
  368. channel = item['nickname']
  369. status = str(1)
  370. per_page = str(1000)
  371. limit_time = et
  372. get_time = st
  373. lt = parse.urlencode({'created_at[lt]': limit_time})
  374. gt = parse.urlencode({'created_at[gte]': get_time})
  375. #url_1 = 'https://inovel.818tu.com/partners/channel/orders/list?'
  376. url_1 = 'https://openapi.818tu.com/partners/channel/orders/list?'
  377. my_sign_1 = md5value(secert + 'channel_id=' + str(
  378. channel_id) + '&created_at[gte]=' + get_time + '&created_at[lt]=' + limit_time + '&key=' + my_key + '&per_page=' + per_page + '&status=' + status)
  379. parameter_1 = 'channel_id=' + str(
  380. channel_id) + '&' + gt + '&' + lt + '&per_page=' + per_page + '&status=' + status + '&key=' + my_key + '&sign=' + my_sign_1
  381. orders = requests.get(url_1 + parameter_1)
  382. t = orders.json()['data']['count'] // int(per_page) + 1
  383. for page in range(1, t + 1):
  384. my_sign_2 = md5value(secert + 'channel_id=' + str(
  385. channel_id) + '&created_at[gte]=' + get_time + '&created_at[lt]=' + limit_time + '&key=' + my_key + '&page=' + str(
  386. page) + '&per_page=' + per_page + '&status=' + status)
  387. parameter_2 = 'channel_id=' + str(channel_id) + '&' + gt + '&' + lt + '&page=' + str(
  388. page) + '&per_page=' + per_page + '&status=' + status + '&key=' + my_key + '&sign=' + my_sign_2
  389. orders_1 = requests.get(url_1 + parameter_2)
  390. b = orders_1.json()['data']['items']
  391. for a in b:
  392. c = {}
  393. c['user_id'] = str(a['member']['openid'])
  394. c['channel'] = channel
  395. c['reg_time'] = a['member']['created_at']
  396. c['channel_id'] = channel_id
  397. c['amount'] = round(a['price'] / 100, 2)
  398. c['order_id'] = str(a['id'])
  399. c['order_time'] = a['created_at']
  400. c['platform'] = '掌中云'
  401. dtime = datetime.datetime.strptime(a['created_at'][0:10], "%Y-%m-%d")
  402. c['date'] = ((int(time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
  403. c['stage'] = stage
  404. if str(a['from_novel_id']) != 'None':
  405. c['from_novel'] = a['from_novel']['title']
  406. else:
  407. c['from_novel'] = 'None'
  408. """
  409. del[a['member']]
  410. del[a['referral_link_id']]
  411. del[a['id']]
  412. del[a['created_at']]
  413. del[a['paid_at']]
  414. del a['border_id']
  415. del a['from_novel_id']
  416. del a['status']
  417. del a['price']
  418. del a['agent_uid']
  419. """
  420. x = sorted(c.items(), key=lambda item: item[0])
  421. x = dict(x)
  422. x = tuple(x.values())
  423. r = r + ((x),)
  424. cost_time = round((time.time() - time1) / 60, 1)
  425. print('get_zzy_order:', len(r), 'cost_minutes:', cost_time)
  426. return r
  427. def get_ysg_order(st, et):
  428. time1 = time.time()
  429. key_list = al.ysg_account_list
  430. url = 'https://novel.youshuge.com/v2/open/orders'
  431. o = ()
  432. for key in key_list:
  433. host_name = key[0]
  434. channel_id = key[1]
  435. secert_key = key[2]
  436. channel = key[3]
  437. stage = key[4]
  438. timestamp = int(time.time())
  439. start_date = time.strftime("%Y-%m-%d", time.localtime(st))
  440. end_date = time.strftime("%Y-%m-%d", time.localtime(et + 86400))
  441. page = 1
  442. str1 = 'channel_id=' + str(channel_id) + '&end_date=' + end_date + '&host_name=' + host_name + '&page=' + str(
  443. page) + '&pay_status=1' + '&start_date=' + start_date + '&time=' + str(timestamp) + '&key=' + secert_key
  444. sign = md5value(str1).upper()
  445. data = {
  446. 'sign': sign,
  447. 'host_name': host_name,
  448. 'time': timestamp,
  449. 'channel_id': channel_id,
  450. 'page': page,
  451. 'pay_status': 1,
  452. 'start_date': start_date,
  453. 'end_date': end_date
  454. }
  455. r = requests.post(url, data).json()
  456. if 'data' in r.keys():
  457. if len(r['data']) > 0:
  458. for i in range((r['data'][0]['count'] - 1) // 100 + 1):
  459. timestamp = int(time.time())
  460. str1 = 'channel_id=' + str(
  461. channel_id) + '&end_date=' + end_date + '&host_name=' + host_name + '&page=' + str(
  462. page) + '&pay_status=1' + '&start_date=' + start_date + '&time=' + str(
  463. timestamp) + '&key=' + secert_key
  464. sign = md5value(str1).upper()
  465. data2 = {
  466. 'sign': sign,
  467. 'host_name': host_name,
  468. 'time': timestamp,
  469. 'channel_id': channel_id,
  470. 'page': page,
  471. 'pay_status': 1,
  472. 'start_date': start_date,
  473. 'end_date': end_date
  474. }
  475. r2 = requests.post(url, data2).json()
  476. if 'data' in r2.keys():
  477. if len(r2['data']) > 0:
  478. for x in r2['data']:
  479. t1 = time.strptime(x['create_time'], "%Y-%m-%d %H:%M:%S")
  480. ts = int(time.mktime(t1))
  481. if ts >= st and ts < et:
  482. y = {}
  483. y['order_id'] = x['order_num']
  484. y['amount'] = round(int(x['price']) / 100, 2)
  485. y['order_time'] = x['create_time']
  486. y['channel'] = channel
  487. y['from_novel'] = x['book_name']
  488. y['stage'] = stage
  489. y['user_id'] = x['openid']
  490. y['channel_id'] = channel_id
  491. dtime = datetime.datetime.strptime(x['create_time'][0:10], "%Y-%m-%d")
  492. y['date'] = ((int(
  493. time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
  494. y['platform'] = '悠书阁'
  495. y['reg_time'] = x['reg_time']
  496. y = sorted(y.items(), key=lambda item: item[0])
  497. y = dict(y)
  498. y = tuple(y.values())
  499. o = o + ((y),)
  500. page = page + 1
  501. cost_time = round((time.time() - time1) / 60, 1)
  502. print('get_ysg_order:', len(o), 'cost_minutes:', cost_time)
  503. return o
  504. def daily_reports_get(access_token, account_id, level, fields):
  505. interface = 'daily_reports/get'
  506. url = 'https://api.e.qq.com/v1.1/' + interface
  507. common_parameters = {
  508. 'access_token': access_token,
  509. 'timestamp': int(time.time()),
  510. 'nonce': str(time.time()) + str(random.randint(0, 999999)),
  511. }
  512. parameters = {
  513. "account_id": account_id,
  514. "level": level,
  515. "date_range":
  516. {
  517. "start_date": time.strftime("%Y-%m-%d", time.localtime()),
  518. "end_date": time.strftime("%Y-%m-%d", time.localtime())
  519. },
  520. "page": 1,
  521. "page_size": 1000,
  522. "fields": fields
  523. }
  524. parameters.update(common_parameters)
  525. for k in parameters:
  526. if type(parameters[k]) is not str:
  527. parameters[k] = json.dumps(parameters[k])
  528. r = requests.get(url, params=parameters)
  529. return r.json()
  530. def get_daily_vx():
  531. token_list_v = tl.token_list_vx
  532. t = ()
  533. time1 = time.time()
  534. for y in token_list_v:
  535. c = daily_reports_get(y[2], y[0], "REPORT_LEVEL_ADVERTISER_WECHAT", (
  536. 'date', 'cost', 'view_count', 'valid_click_count', 'ctr', 'official_account_follow_rate', 'order_amount',
  537. 'order_roi', 'order_count', 'order_rate', 'order_unit_price', 'web_order_cost', 'first_day_order_amount',
  538. 'first_day_order_count'))
  539. if 'data' in c.keys():
  540. for d in c['data']['list']:
  541. d['account_id'] = y[0]
  542. x = tuple(d.values())
  543. t = t + ((x),)
  544. cost_time = round((time.time() - time1) / 60, 1)
  545. print('get_daily_vx:', len(t), 'cost_minutes:', cost_time)
  546. return t
  547. def get_daily_qq():
  548. token_list_q = tl.token_list_qq
  549. t = ()
  550. time1 = time.time()
  551. for x in token_list_q:
  552. a = daily_reports_get(x[2], x[0], "REPORT_LEVEL_ADVERTISER", (
  553. 'date', 'view_count', 'valid_click_count', 'ctr', 'cpc', 'cost', 'web_order_count', 'web_order_rate',
  554. 'web_order_cost', 'follow_count', 'order_amount', 'order_roi', 'platform_page_view_count',
  555. 'web_commodity_page_view_count', 'from_follow_uv'))
  556. for b in a['data']['list']:
  557. b['account_id'] = x[0]
  558. y = tuple(b.values())
  559. t = t + ((y),)
  560. cost_time = round((time.time() - time1) / 60, 1)
  561. print('get_qq_order:', len(t), 'cost_minutes:', cost_time)
  562. return t
  563. def mysql_insert_order(data):
  564. db = pymysql.connect('rm-bp1c9cj79872tx3aaro.mysql.rds.aliyuncs.com', 'superc', 'Cc719199895', 'quchen_text')
  565. cursor = db.cursor()
  566. time1 = time.time()
  567. # sql = 'insert ignore into quchen_text.order (amount,channel,channel_id,date,from_novel,order_id,order_time,platform,reg_time,stage,user_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
  568. sql = 'replace into quchen_text.order(amount,channel,channel_id,date,from_novel,order_id,order_time,platform,reg_time,stage,user_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
  569. try:
  570. cursor.executemany(sql, data)
  571. db.commit()
  572. cost_time = round((time.time() - time1) / 60, 1)
  573. print('insert_order access', len(data), 'cost_minutes:', cost_time)
  574. except:
  575. db.rollback()
  576. print('insert_order defeat')
  577. def mysql_insert_daily_vx(data):
  578. db = pymysql.connect('rm-bp1c9cj79872tx3aaro.mysql.rds.aliyuncs.com', 'superc', 'Cc719199895', 'quchen_text')
  579. cursor = db.cursor()
  580. time1 = time.time()
  581. sql1 = 'delete from daily_vx where date = %s'
  582. sql2 = 'insert ignore into daily_vx (date,cost,view_count,valid_click_count,ctr,official_account_follow_rate,order_amount,order_roi,order_count,order_rate,order_unit_price,web_order_cost,first_day_order_amount,first_day_order_count,account_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
  583. try:
  584. cursor.execute(sql1, str(time.strftime("%Y-%m-%d", time.localtime())))
  585. db.commit()
  586. print('clear_daily_vx access')
  587. except:
  588. print('clear_daily_vx defeat')
  589. try:
  590. cursor.executemany(sql2, data)
  591. db.commit()
  592. cost_time = round((time.time() - time1) / 60, 1)
  593. print('insert_daily_vx access', len(data), 'cost_minutes:', cost_time)
  594. except:
  595. db.rollback()
  596. print('insert_daily_vx defeat')
  597. def mysql_insert_daily_qq(data):
  598. db = pymysql.connect('rm-bp1c9cj79872tx3aaro.mysql.rds.aliyuncs.com', 'superc', 'Cc719199895', 'quchen_text')
  599. cursor = db.cursor()
  600. time1 = time.time()
  601. sql1 = 'delete from daily_qq where date = %s'
  602. sql2 = 'insert ignore into daily_qq (date,view_count,valid_click_count,ctr,cpc,cost,web_order_count,web_order_rate,web_order_cost,follow_count,order_amount,order_roi,platform_page_view_count,web_commodity_page_view_count,from_follow_uv,account_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
  603. try:
  604. cursor.execute(sql1, str(time.strftime("%Y-%m-%d", time.localtime())))
  605. db.commit()
  606. print('clear_daily_qq access')
  607. except:
  608. print('clear_daily_qq defeat')
  609. try:
  610. cursor.executemany(sql2, data)
  611. db.commit()
  612. cost_time = round((time.time() - time1) / 60, 1)
  613. print('insert_daily_qq access', len(data), 'cost_minutes:', cost_time)
  614. except:
  615. db.rollback()
  616. print('insert_daily_qq defeat')
  617. def send_waring_email():
  618. msg_Sender = '2091961210@qq.cm' # 发送方邮箱
  619. msg_code = 'ylltzynbshmufdeh' # 发送方邮箱的授权码
  620. msg_Receiver = '2091961210@qq.cm' # 收件人邮箱
  621. subject = "数据库未按时更新" # 主题
  622. content = "数据库未按时更新" # 正文
  623. msg = MIMEText(content, _charset="utf-8")
  624. msg['Subject'] = subject
  625. msg['From'] = msg_Sender
  626. msg['To'] = msg_Receiver
  627. try:
  628. s = smtplib.SMTP_SSL("smtp.qq.com", 465) # 邮件服务器及端口号
  629. s.login(msg_Sender, msg_code)
  630. s.sendmail(msg_Sender, msg_Receiver, msg.as_string())
  631. print("发送成功")
  632. except Exception as e:
  633. print("发送失败", e)
  634. finally:
  635. s.quit()
  636. def update_check():
  637. st_unix = int((time.time() + 8 * 3600) // 3600 * 3600 - 8 * 3600 - 3600)
  638. et_unix = int((time.time() + 8 * 3600) // 3600 * 3600 - 8 * 3600)
  639. # et_unix = et_unix - 1
  640. st_dt = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(st_unix)) + '+08:00'
  641. et_dt = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(et_unix)) + '+08:00'
  642. db = pymysql.connect('rm-bp1c9cj79872tx3aaro.mysql.rds.aliyuncs.com', 'superc', 'Cc719199895', 'quchen_text')
  643. cursor = db.cursor()
  644. sql4 = "select platform,round(max(if(platform='掌读',order_time,unix_timestamp(order_time))),0) from quchen_text.order where date=%s and platform not in ('阳光','悠书阁') group by platform "
  645. data = (int((time.time() + 8 * 3600) // 86400 * 86400 - 8 * 3600))
  646. try:
  647. cursor.execute(sql4, data)
  648. db.commit()
  649. x = cursor.fetchall()
  650. print('access', x)
  651. except:
  652. db.rollback()
  653. print('defeat', x)
  654. zzy = x[0][1] // 3600 * 3600
  655. zhangdu = x[1][1] // 3600 * 3600
  656. huasheng = x[2][1] // 3600 * 3600
  657. yuewen = x[3][1] // 3600 * 3600
  658. if zzy < int((time.time() + 8 * 3600) // 3600 * 3600 - 8 * 3600 - 3600):
  659. mysql_insert_order(get_zzy_order(st_dt, et_dt))
  660. if zhangdu < int((time.time() + 8 * 3600) // 3600 * 3600 - 8 * 3600 - 3600):
  661. mysql_insert_order(get_zhangdu_order(st_unix, et_unix))
  662. if huasheng < int((time.time() + 8 * 3600) // 3600 * 3600 - 8 * 3600 - 3600):
  663. mysql_insert_order(get_huasheng_order(st_unix, et_unix))
  664. if yuewen < int((time.time() + 8 * 3600) // 3600 * 3600 - 8 * 3600 - 3600):
  665. mysql_insert_order(get_yuewen_order(st_unix, et_unix - 1))
  666. try:
  667. cursor.execute(sql4, data)
  668. db.commit()
  669. z = cursor.fetchall()
  670. print('access', z)
  671. except:
  672. db.rollback()
  673. print('defeat', z)
  674. a = []
  675. for k in x:
  676. a.append(k[1])
  677. y = min(a) // 3600 * 3600
  678. if y < int((time.time() + 8 * 3600) // 3600 * 3600 - 8 * 3600 - 3600):
  679. send_waring_email()
  680. def start_order_job():
  681. st_unix_time = time.time()
  682. st_unix = int((st_unix_time + 8 * 3600) // 3600 * 3600 - 8 * 3600 - 3600)
  683. et_unix = int((st_unix_time + 8 * 3600) // 3600 * 3600 - 8 * 3600)
  684. # et_unix = et_unix - 1
  685. st_dt = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(st_unix)) + '+08:00'
  686. et_dt = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(et_unix)) + '+08:00'
  687. print(st_dt, et_dt)
  688. mysql_insert_order(get_yuewen_order(st_unix, et_unix))
  689. mysql_insert_order(get_zhangdu_order(st_unix, et_unix))
  690. mysql_insert_order(get_ysg_order(st_unix, et_unix))
  691. mysql_insert_order(get_huasheng_order(st_unix, et_unix))
  692. mysql_insert_order(get_zzy_order(st_dt, et_dt))
  693. update_check()
  694. def start_cost_job():
  695. st_unix_time = time.time()
  696. st_unix = int((st_unix_time + 8 * 3600) // 3600 * 3600 - 8 * 3600 - 3600)
  697. et_unix = int((st_unix_time + 8 * 3600) // 3600 * 3600 - 8 * 3600)
  698. # et_unix = et_unix - 1
  699. st_dt = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(st_unix)) + '+08:00'
  700. et_dt = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(et_unix)) + '+08:00'
  701. print(st_dt, et_dt)
  702. mysql_insert_daily_vx(get_daily_vx())
  703. mysql_insert_daily_qq(get_daily_qq())
  704. start_order_time = '2020-10-09 16:10:00'
  705. start_cost_time = '2020-10-09 15:35:00'
  706. if __name__ == '__main__':
  707. scheduler = BlockingScheduler()
  708. scheduler.add_job(start_order_job, 'interval', max_instances=10, hours=1, start_date=start_order_time)
  709. scheduler.add_job(start_cost_job, 'interval', max_instances=10, hours=1, start_date=start_cost_time)
  710. scheduler.start()