get_data_hourly.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """
  4. Created on Thu Jun 4 15:06:05 2020
  5. @author: chencong
  6. """
  7. import requests
  8. import hashlib
  9. import time
  10. from urllib import parse
  11. import datetime
  12. import json
  13. import math
  14. import pymysql
  15. import random
  16. from apscheduler.schedulers.blocking import BlockingScheduler
  17. import account_list as al
  18. import token_list as tl
  19. import smtplib
  20. from email.mime.text import MIMEText
  21. import importlib
  22. importlib.reload(tl)
  23. def md5value(s):
  24. md5 = hashlib.md5()
  25. md5.update(s.encode("utf-8"))
  26. return md5.hexdigest()
  27. def get_yuewen_order(st, et):
  28. t = ()
  29. time1 = time.time()
  30. url = 'https://open.yuewen.com/cpapi/wxRecharge/querychargelog'
  31. version = 1
  32. secert_list = al.yuewen_account_list
  33. for secert in secert_list:
  34. start_time = st
  35. end_time = et
  36. email = secert[0]
  37. appsecert = secert[1]
  38. for i in range((et - st) // 86400 + 1):
  39. timestamp = int(time.time())
  40. s = ''
  41. page = 1
  42. order_status = 2
  43. data = {
  44. 'email': email,
  45. 'version': version,
  46. 'timestamp': timestamp,
  47. 'start_time': start_time,
  48. 'end_time': end_time,
  49. 'page': page,
  50. 'order_status': order_status
  51. }
  52. # 'last_min_id':last_min_id,
  53. # 'last_max_id':last_max_id,
  54. # 'total_count':total_count,
  55. # 'last_page':last_page}
  56. sorted_data = sorted(data.items())
  57. for k, v in sorted_data:
  58. s = s + str(k) + str(v)
  59. sign = md5value(appsecert + s).upper()
  60. data1 = {
  61. 'email': email,
  62. 'version': version,
  63. 'timestamp': timestamp,
  64. 'start_time': start_time,
  65. 'end_time': end_time,
  66. 'page': page,
  67. 'order_status': order_status,
  68. 'sign': sign
  69. }
  70. list1 = requests.get(url=url, params=data1)
  71. total_count = list1.json()['data']['total_count']
  72. last_min_id = list1.json()['data']['min_id']
  73. last_max_id = list1.json()['data']['max_id']
  74. last_page = list1.json()['data']['page']
  75. if total_count > 0:
  76. for x in list1.json()['data']['list']:
  77. y = {}
  78. dtime = datetime.datetime.strptime(x['order_time'], "%Y-%m-%d %H:%M:%S")
  79. y['date'] = ((int(time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
  80. y['platform'] = '阅文'
  81. y['channel'] = x['app_name']
  82. y['from_novel'] = x['book_name']
  83. y['user_id'] = x['openid']
  84. y['stage'] = ''
  85. y['channel_id'] = 0
  86. y['order_time'] = x['order_time']
  87. y['amount'] = x['amount']
  88. y['reg_time'] = x['reg_time']
  89. y['order_id'] = x['order_id']
  90. """
  91. del x['app_name']
  92. del x['order_status']
  93. del x['order_type']
  94. del x['openid']
  95. del x['user_name']
  96. del x['sex']
  97. del x['channel_name']
  98. del x['book_id']
  99. del x['book_name']
  100. del x['report_status']
  101. if y['order_time'] != time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(et)):
  102. """
  103. y = sorted(y.items(), key=lambda item: item[0])
  104. y = dict(y)
  105. y = tuple(y.values())
  106. t = t + ((y),)
  107. if total_count > 100:
  108. for page in range(2, math.ceil(total_count / 100) + 1):
  109. data = {
  110. 'email': email,
  111. 'version': version,
  112. 'timestamp': timestamp,
  113. 'start_time': start_time,
  114. 'end_time': end_time,
  115. 'page': page,
  116. 'last_min_id': last_min_id,
  117. 'last_max_id': last_max_id,
  118. 'total_count': total_count,
  119. 'last_page': last_page,
  120. 'order_status': order_status
  121. }
  122. sorted_data = sorted(data.items())
  123. s1 = ''
  124. for k, v in sorted_data:
  125. s1 = s1 + str(k) + str(v)
  126. sign = md5value(appsecert + s1).upper()
  127. data2 = {
  128. 'email': email,
  129. 'version': version,
  130. 'timestamp': timestamp,
  131. 'start_time': start_time,
  132. 'end_time': end_time,
  133. 'page': page,
  134. 'last_min_id': last_min_id,
  135. 'last_max_id': last_max_id,
  136. 'total_count': total_count,
  137. 'last_page': last_page,
  138. 'order_status': order_status,
  139. 'sign': sign
  140. }
  141. list2 = requests.get(url=url, params=data2)
  142. for x in list2.json()['data']['list']:
  143. y = {}
  144. dtime = datetime.datetime.strptime(x['order_time'], "%Y-%m-%d %H:%M:%S")
  145. y['date'] = ((int(time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
  146. y['platform'] = '阅文'
  147. y['channel'] = x['app_name']
  148. y['from_novel'] = x['book_name']
  149. y['user_id'] = x['openid']
  150. y['stage'] = ''
  151. y['channel_id'] = 0
  152. y['order_time'] = x['order_time']
  153. y['amount'] = x['amount']
  154. y['reg_time'] = x['reg_time']
  155. y['order_id'] = x['order_id']
  156. """
  157. del x['report_status']
  158. del x['app_name']
  159. del x['order_status']
  160. del x['order_type']
  161. del x['openid']
  162. del x['user_name']
  163. del x['sex']
  164. del x['channel_name']
  165. del x['book_id']
  166. del x['book_name']
  167. if y['order_time'] != time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(et)):
  168. """
  169. y = sorted(y.items(), key=lambda item: item[0])
  170. y = dict(y)
  171. y = tuple(y.values())
  172. t = t + ((y),)
  173. total_count = list2.json()['data']['total_count']
  174. last_min_id = list2.json()['data']['min_id']
  175. last_max_id = list2.json()['data']['max_id']
  176. last_page = list2.json()['data']['page']
  177. cost_time = round((time.time() - time1) / 60, 1)
  178. print('get_yuewen_order:', len(t), 'cost_minutes:', cost_time)
  179. return t
  180. def get_zhangdu_order(st, et):
  181. time1 = time.time()
  182. secert_list = al.zhangdu_account_list
  183. t = ()
  184. url = 'https://api.zhangdu520.com/channel/getorder'
  185. for item in secert_list: # 分渠道
  186. uid = item[0]
  187. appsecert = item[1]
  188. channel = item[2]
  189. timestamp = int(time.time())
  190. sign = md5value(str(uid) + '&' + appsecert + '&' + str(timestamp))
  191. page = 1
  192. starttime = st
  193. endtime = et
  194. for x in range((et - st) // 86400 + 1): # 分时段
  195. Params = {'uid': uid,
  196. 'timestamp': timestamp,
  197. 'sign': sign,
  198. 'starttime': starttime,
  199. 'endtime': endtime
  200. }
  201. list1 = requests.get(url=url, params=Params)
  202. pageCount = list1.json()['data']['pageCount']
  203. if pageCount > 0:
  204. for a in range(1, pageCount + 1): # 分页
  205. page = a
  206. Params = {
  207. 'uid': uid,
  208. 'timestamp': timestamp,
  209. 'sign': sign,
  210. 'starttime': starttime,
  211. 'endtime': endtime,
  212. 'page': page
  213. }
  214. list2 = requests.get(url=url, params=Params).json()['data']['list']
  215. for b in list2:
  216. c = {}
  217. c['channel_id'] = uid
  218. c['order_id'] = str(b['orderno'])
  219. c['order_time'] = b['ctime']
  220. c['user_id'] = b['openid']
  221. c['platform'] = '掌读'
  222. c['channel'] = channel
  223. c['stage'] = ''
  224. c['from_novel'] = ''
  225. c['reg_time'] = b['regtime']
  226. c['date'] = ((int(b['ctime']) + 8 * 3600) // 86400) * 86400 - 8 * 3600
  227. c['amount'] = b['amount']
  228. """
  229. del b['openid']
  230. del b['regtime']
  231. del b['ip']
  232. del b['ua']
  233. del b['id']
  234. del b['ctime']
  235. del b['userid']
  236. del b['orderno']
  237. del b['source']
  238. del b['sourceid']
  239. """
  240. if b['status'] == '1' and int(c['order_time']) < et:
  241. del b['status']
  242. x = sorted(c.items(), key=lambda item: item[0])
  243. x = dict(x)
  244. x = tuple(x.values())
  245. t = t + ((x),)
  246. cost_time = round((time.time() - time1) / 60, 1)
  247. print('get_zhangdu_order:', len(t), 'cost_minutes:', cost_time)
  248. return t
  249. def get_huasheng_order(st, et):
  250. time1 = time.time()
  251. apikey_list = al.huasheng_account_list
  252. url = 'https://vip.rlcps.cn/api/getMerchants'
  253. t = ()
  254. for key in apikey_list: # 获取每个vip账号下的channel_id
  255. apiKEY = key[0]
  256. apiSecurity = key[1]
  257. stage = key[2]
  258. timestamp = str(int(time.time()))
  259. sign = md5value(apiKEY + timestamp + apiSecurity).upper()
  260. data = {'apiKey': apiKEY,
  261. 'apiSecurity': apiSecurity,
  262. 'timestamp': timestamp,
  263. 'sign': sign
  264. }
  265. list0 = requests.post(url, data).json()
  266. for merchant in list0['data']:
  267. merchant_id = merchant['merchant_id']
  268. merchant_name = merchant['merchant_name']
  269. url1 = 'https://vip.rlcps.cn/api/orderList'
  270. start_time = st
  271. for i in range((et - st) // 86400 + 1):
  272. date = time.strftime("%Y-%m-%d", time.localtime(start_time))
  273. sign = md5value(apiKEY + date + str(merchant_id) + timestamp + apiSecurity).upper()
  274. page = 1
  275. data1 = {
  276. 'apiKey': apiKEY,
  277. 'apiSecurity': apiSecurity,
  278. 'timestamp': timestamp,
  279. 'date': date,
  280. 'merchant_id': merchant_id,
  281. 'sign': sign,
  282. 'page': page
  283. }
  284. list1 = requests.post(url1, data1).json()
  285. if 'data' in list1.keys() and len(list1['data']) > 0:
  286. for i in range(int(math.ceil(list1['count'] / 500))):
  287. data2 = {
  288. 'apiKey': apiKEY,
  289. 'apiSecurity': apiSecurity,
  290. 'timestamp': timestamp,
  291. 'date': date,
  292. 'merchant_id': merchant_id,
  293. 'sign': sign,
  294. 'page': page
  295. }
  296. list2 = requests.post(url1, data2).json()
  297. for x in list2['data']:
  298. if x['order_status'] == 1:
  299. t1 = time.strptime(x['pay_at'], "%Y-%m-%d %H:%M:%S")
  300. ts = int(time.mktime(t1))
  301. if ts >= st and ts < et:
  302. y = {}
  303. # dtime = datetime.datetime.strptime(x['pay_at'],"%Y-%m-%d")
  304. # y['date']= ((int(time.mktime(dtime.timetuple()))+8*3600)//86400)*86400-8*3600
  305. y['order_id'] = x['trans_id']
  306. y['order_time'] = x['pay_at']
  307. y['reg_time'] = x['join_at']
  308. y['date'] = (start_time + 8 * 3600) // 86400 * 86400 - 8 * 3600
  309. y['channel'] = merchant_name
  310. y['channel_id'] = merchant_id
  311. y['from_novel'] = x['book_name']
  312. y['platform'] = '花生'
  313. y['stage'] = stage
  314. y['user_id'] = x['openid']
  315. y['amount'] = x['amount']
  316. """
  317. del x['order_num']
  318. del x['book_name']
  319. del x['trans_id']
  320. del x['pay_at']
  321. del x['join_at']
  322. del x['subscribe_at']
  323. del x['openid']
  324. del x['charge_count']
  325. del x['book_id']
  326. del x['order_status']
  327. del x['user_name']
  328. del x['spread_name']
  329. del x['request_at']
  330. if y['order_time'] != time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(et)):
  331. """
  332. y = sorted(y.items(), key=lambda item: item[0])
  333. y = dict(y)
  334. y = tuple(y.values())
  335. t = t + ((y),)
  336. page = page + 1
  337. cost_time = round((time.time() - time1) / 60, 1)
  338. print('get_huasheng_order:', len(t), 'cost_minutes:', cost_time)
  339. return t
  340. def get_zzy_order(st, et):
  341. time1 = time.time()
  342. API_list = al.zzy_account_list
  343. r = ()
  344. for x in API_list:
  345. my_key = x[0]
  346. secert = x[1]
  347. stage = x[2]
  348. my_sign = md5value(secert + 'key=' + my_key)
  349. #url = 'https://openapi.818tu.com/partners/channel/channels/list?'
  350. url = 'https://inovel.818tu.com/partners/channel/channels/list?'
  351. parameter = 'key=' + my_key + '&sign=' + my_sign
  352. channel_list = requests.get(url + parameter) # 获取子渠道列表
  353. if 'data' in channel_list.json().keys():
  354. items = channel_list.json()['data']['items']
  355. elif len(x) > 3:
  356. my_key = x[3]
  357. secert = x[4]
  358. my_sign = md5value(secert + 'key=' + my_key)
  359. #url = 'https://openapi.818tu.com/partners/channel/channels/list?'
  360. url = 'https://inovel.818tu.com/partners/channel/channels/list?'
  361. parameter = 'key=' + my_key + '&sign=' + my_sign
  362. channel_list = requests.get(url + parameter)
  363. items = channel_list.json()['data']['items']
  364. else:
  365. print(channel_list.json())
  366. items = []
  367. if items != []:
  368. for item in items: # 获取channel_id 后逐个拉取历史orders
  369. channel_id = item['id']
  370. channel = item['nickname']
  371. status = str(1)
  372. per_page = str(1000)
  373. limit_time = et
  374. get_time = st
  375. lt = parse.urlencode({'created_at[lt]': limit_time})
  376. gt = parse.urlencode({'created_at[gte]': get_time})
  377. #url_1 = 'https://openapi.818tu.com/partners/channel/orders/list?'
  378. url_1 = 'https://inovel.818tu.com/partners/channel/orders/list?'
  379. my_sign_1 = md5value(secert + 'channel_id=' + str(
  380. channel_id) + '&created_at[gte]=' + get_time + '&created_at[lt]=' + limit_time + '&key=' + my_key + '&per_page=' + per_page + '&status=' + status)
  381. parameter_1 = 'channel_id=' + str(
  382. channel_id) + '&' + gt + '&' + lt + '&per_page=' + per_page + '&status=' + status + '&key=' + my_key + '&sign=' + my_sign_1
  383. orders = requests.get(url_1 + parameter_1)
  384. t = orders.json()['data']['count'] // int(per_page) + 1
  385. for page in range(1, t + 1):
  386. my_sign_2 = md5value(secert + 'channel_id=' + str(
  387. channel_id) + '&created_at[gte]=' + get_time + '&created_at[lt]=' + limit_time + '&key=' + my_key + '&page=' + str(
  388. page) + '&per_page=' + per_page + '&status=' + status)
  389. parameter_2 = 'channel_id=' + str(channel_id) + '&' + gt + '&' + lt + '&page=' + str(
  390. page) + '&per_page=' + per_page + '&status=' + status + '&key=' + my_key + '&sign=' + my_sign_2
  391. orders_1 = requests.get(url_1 + parameter_2)
  392. b = orders_1.json()['data']['items']
  393. for a in b:
  394. c = {}
  395. c['user_id'] = str(a['member']['openid'])
  396. c['channel'] = channel
  397. c['reg_time'] = a['member']['created_at']
  398. c['channel_id'] = channel_id
  399. c['amount'] = round(a['price'] / 100, 2)
  400. c['order_id'] = str(a['id'])
  401. c['order_time'] = a['created_at']
  402. c['platform'] = '掌中云'
  403. dtime = datetime.datetime.strptime(a['created_at'][0:10], "%Y-%m-%d")
  404. c['date'] = ((int(time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
  405. c['stage'] = stage
  406. if str(a['from_novel_id']) != 'None':
  407. c['from_novel'] = a['from_novel']['title']
  408. else:
  409. c['from_novel'] = 'None'
  410. """
  411. del[a['member']]
  412. del[a['referral_link_id']]
  413. del[a['id']]
  414. del[a['created_at']]
  415. del[a['paid_at']]
  416. del a['border_id']
  417. del a['from_novel_id']
  418. del a['status']
  419. del a['price']
  420. del a['agent_uid']
  421. """
  422. x = sorted(c.items(), key=lambda item: item[0])
  423. x = dict(x)
  424. x = tuple(x.values())
  425. r = r + ((x),)
  426. cost_time = round((time.time() - time1) / 60, 1)
  427. print('get_zzy_order:', len(r), 'cost_minutes:', cost_time)
  428. return r
  429. def get_ysg_order(st, et):
  430. time1 = time.time()
  431. key_list = al.ysg_account_list
  432. url = 'https://novel.youshuge.com/v2/open/orders'
  433. o = ()
  434. for key in key_list:
  435. host_name = key[0]
  436. channel_id = key[1]
  437. secert_key = key[2]
  438. channel = key[3]
  439. stage = key[4]
  440. timestamp = int(time.time())
  441. start_date = time.strftime("%Y-%m-%d", time.localtime(st))
  442. end_date = time.strftime("%Y-%m-%d", time.localtime(et + 86400))
  443. page = 1
  444. str1 = 'channel_id=' + str(channel_id) + '&end_date=' + end_date + '&host_name=' + host_name + '&page=' + str(
  445. page) + '&pay_status=1' + '&start_date=' + start_date + '&time=' + str(timestamp) + '&key=' + secert_key
  446. sign = md5value(str1).upper()
  447. data = {
  448. 'sign': sign,
  449. 'host_name': host_name,
  450. 'time': timestamp,
  451. 'channel_id': channel_id,
  452. 'page': page,
  453. 'pay_status': 1,
  454. 'start_date': start_date,
  455. 'end_date': end_date
  456. }
  457. r = requests.post(url, data).json()
  458. if 'data' in r.keys():
  459. if len(r['data']) > 0:
  460. for i in range((r['data'][0]['count'] - 1) // 100 + 1):
  461. timestamp = int(time.time())
  462. str1 = 'channel_id=' + str(
  463. channel_id) + '&end_date=' + end_date + '&host_name=' + host_name + '&page=' + str(
  464. page) + '&pay_status=1' + '&start_date=' + start_date + '&time=' + str(
  465. timestamp) + '&key=' + secert_key
  466. sign = md5value(str1).upper()
  467. data2 = {
  468. 'sign': sign,
  469. 'host_name': host_name,
  470. 'time': timestamp,
  471. 'channel_id': channel_id,
  472. 'page': page,
  473. 'pay_status': 1,
  474. 'start_date': start_date,
  475. 'end_date': end_date
  476. }
  477. r2 = requests.post(url, data2).json()
  478. if 'data' in r2.keys():
  479. if len(r2['data']) > 0:
  480. for x in r2['data']:
  481. t1 = time.strptime(x['create_time'], "%Y-%m-%d %H:%M:%S")
  482. ts = int(time.mktime(t1))
  483. if ts >= st and ts < et:
  484. y = {}
  485. y['order_id'] = x['order_num']
  486. y['amount'] = round(int(x['price']) / 100, 2)
  487. y['order_time'] = x['create_time']
  488. y['channel'] = channel
  489. y['from_novel'] = x['book_name']
  490. y['stage'] = stage
  491. y['user_id'] = x['openid']
  492. y['channel_id'] = channel_id
  493. dtime = datetime.datetime.strptime(x['create_time'][0:10], "%Y-%m-%d")
  494. y['date'] = ((int(
  495. time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
  496. y['platform'] = '悠书阁'
  497. y['reg_time'] = x['reg_time']
  498. y = sorted(y.items(), key=lambda item: item[0])
  499. y = dict(y)
  500. y = tuple(y.values())
  501. o = o + ((y),)
  502. page = page + 1
  503. cost_time = round((time.time() - time1) / 60, 1)
  504. print('get_ysg_order:', len(o), 'cost_minutes:', cost_time)
  505. return o
  506. def daily_reports_get(access_token, account_id, level, fields):
  507. interface = 'daily_reports/get'
  508. url = 'https://api.e.qq.com/v1.1/' + interface
  509. common_parameters = {
  510. 'access_token': access_token,
  511. 'timestamp': int(time.time()),
  512. 'nonce': str(time.time()) + str(random.randint(0, 999999)),
  513. }
  514. parameters = {
  515. "account_id": account_id,
  516. "level": level,
  517. "date_range":
  518. {
  519. "start_date": time.strftime("%Y-%m-%d", time.localtime()),
  520. "end_date": time.strftime("%Y-%m-%d", time.localtime())
  521. },
  522. "page": 1,
  523. "page_size": 1000,
  524. "fields": fields
  525. }
  526. parameters.update(common_parameters)
  527. for k in parameters:
  528. if type(parameters[k]) is not str:
  529. parameters[k] = json.dumps(parameters[k])
  530. r = requests.get(url, params=parameters)
  531. return r.json()
  532. def get_daily_vx():
  533. token_list_v = tl.token_list_vx
  534. t = ()
  535. time1 = time.time()
  536. for y in token_list_v:
  537. c = daily_reports_get(y[2], y[0], "REPORT_LEVEL_ADVERTISER_WECHAT", (
  538. 'date', 'cost', 'view_count', 'valid_click_count', 'ctr', 'official_account_follow_rate', 'order_amount',
  539. 'order_roi', 'order_count', 'order_rate', 'order_unit_price', 'web_order_cost', 'first_day_order_amount',
  540. 'first_day_order_count'))
  541. if 'data' in c.keys():
  542. for d in c['data']['list']:
  543. d['account_id'] = y[0]
  544. x = tuple(d.values())
  545. t = t + ((x),)
  546. cost_time = round((time.time() - time1) / 60, 1)
  547. print('get_daily_vx:', len(t), 'cost_minutes:', cost_time)
  548. return t
  549. def get_daily_qq():
  550. token_list_q = tl.token_list_qq
  551. t = ()
  552. time1 = time.time()
  553. for x in token_list_q:
  554. a = daily_reports_get(x[2], x[0], "REPORT_LEVEL_ADVERTISER", (
  555. 'date', 'view_count', 'valid_click_count', 'ctr', 'cpc', 'cost', 'web_order_count', 'web_order_rate',
  556. 'web_order_cost', 'follow_count', 'order_amount', 'order_roi', 'platform_page_view_count',
  557. 'web_commodity_page_view_count', 'from_follow_uv'))
  558. for b in a['data']['list']:
  559. b['account_id'] = x[0]
  560. y = tuple(b.values())
  561. t = t + ((y),)
  562. cost_time = round((time.time() - time1) / 60, 1)
  563. print('get_qq_order:', len(t), 'cost_minutes:', cost_time)
  564. return t
  565. def mysql_insert_order(data):
  566. db = pymysql.connect('rm-bp1c9cj79872tx3aaro.mysql.rds.aliyuncs.com', 'superc', 'Cc719199895', 'quchen_text')
  567. cursor = db.cursor()
  568. time1 = time.time()
  569. # sql = 'insert ignore into quchen_text.order (amount,channel,channel_id,date,from_novel,order_id,order_time,platform,reg_time,stage,user_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
  570. sql = 'replace into quchen_text.order(amount,channel,channel_id,date,from_novel,order_id,order_time,platform,reg_time,stage,user_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
  571. try:
  572. cursor.executemany(sql, data)
  573. db.commit()
  574. cost_time = round((time.time() - time1) / 60, 1)
  575. print('insert_order access', len(data), 'cost_minutes:', cost_time)
  576. except:
  577. db.rollback()
  578. print('insert_order defeat')
  579. def mysql_insert_daily_vx(data):
  580. db = pymysql.connect('rm-bp1c9cj79872tx3aaro.mysql.rds.aliyuncs.com', 'superc', 'Cc719199895', 'quchen_text')
  581. cursor = db.cursor()
  582. time1 = time.time()
  583. sql1 = 'delete from daily_vx where date = %s'
  584. sql2 = 'insert ignore into daily_vx (date,cost,view_count,valid_click_count,ctr,official_account_follow_rate,order_amount,order_roi,order_count,order_rate,order_unit_price,web_order_cost,first_day_order_amount,first_day_order_count,account_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
  585. try:
  586. cursor.execute(sql1, str(time.strftime("%Y-%m-%d", time.localtime())))
  587. db.commit()
  588. print('clear_daily_vx access')
  589. except:
  590. print('clear_daily_vx defeat')
  591. try:
  592. cursor.executemany(sql2, data)
  593. db.commit()
  594. cost_time = round((time.time() - time1) / 60, 1)
  595. print('insert_daily_vx access', len(data), 'cost_minutes:', cost_time)
  596. except:
  597. db.rollback()
  598. print('insert_daily_vx defeat')
  599. def mysql_insert_daily_qq(data):
  600. db = pymysql.connect('rm-bp1c9cj79872tx3aaro.mysql.rds.aliyuncs.com', 'superc', 'Cc719199895', 'quchen_text')
  601. cursor = db.cursor()
  602. time1 = time.time()
  603. sql1 = 'delete from daily_qq where date = %s'
  604. sql2 = 'insert ignore into daily_qq (date,view_count,valid_click_count,ctr,cpc,cost,web_order_count,web_order_rate,web_order_cost,follow_count,order_amount,order_roi,platform_page_view_count,web_commodity_page_view_count,from_follow_uv,account_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
  605. try:
  606. cursor.execute(sql1, str(time.strftime("%Y-%m-%d", time.localtime())))
  607. db.commit()
  608. print('clear_daily_qq access')
  609. except:
  610. print('clear_daily_qq defeat')
  611. try:
  612. cursor.executemany(sql2, data)
  613. db.commit()
  614. cost_time = round((time.time() - time1) / 60, 1)
  615. print('insert_daily_qq access', len(data), 'cost_minutes:', cost_time)
  616. except:
  617. db.rollback()
  618. print('insert_daily_qq defeat')
  619. def send_waring_email():
  620. msg_Sender = '2091961210@qq.cm' # 发送方邮箱
  621. msg_code = 'ylltzynbshmufdeh' # 发送方邮箱的授权码
  622. msg_Receiver = '2091961210@qq.cm' # 收件人邮箱
  623. subject = "数据库未按时更新" # 主题
  624. content = "数据库未按时更新" # 正文
  625. msg = MIMEText(content, _charset="utf-8")
  626. msg['Subject'] = subject
  627. msg['From'] = msg_Sender
  628. msg['To'] = msg_Receiver
  629. try:
  630. s = smtplib.SMTP_SSL("smtp.qq.com", 465) # 邮件服务器及端口号
  631. s.login(msg_Sender, msg_code)
  632. s.sendmail(msg_Sender, msg_Receiver, msg.as_string())
  633. print("发送成功")
  634. except Exception as e:
  635. print("发送失败", e)
  636. finally:
  637. s.quit()
  638. def update_check():
  639. st_unix = int((time.time() + 8 * 3600) // 3600 * 3600 - 8 * 3600 - 3600)
  640. et_unix = int((time.time() + 8 * 3600) // 3600 * 3600 - 8 * 3600)
  641. # et_unix = et_unix - 1
  642. st_dt = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(st_unix)) + '+08:00'
  643. et_dt = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(et_unix)) + '+08:00'
  644. db = pymysql.connect('rm-bp1c9cj79872tx3aaro.mysql.rds.aliyuncs.com', 'superc', 'Cc719199895', 'quchen_text')
  645. cursor = db.cursor()
  646. sql4 = "select platform,round(max(if(platform='掌读',order_time,unix_timestamp(order_time))),0) from quchen_text.order where date=%s and platform not in ('阳光','悠书阁') group by platform "
  647. data = (int((time.time() + 8 * 3600) // 86400 * 86400 - 8 * 3600))
  648. try:
  649. cursor.execute(sql4, data)
  650. db.commit()
  651. x = cursor.fetchall()
  652. print('access', x)
  653. except:
  654. db.rollback()
  655. print('defeat', x)
  656. zzy = x[0][1] // 3600 * 3600
  657. zhangdu = x[1][1] // 3600 * 3600
  658. huasheng = x[2][1] // 3600 * 3600
  659. yuewen = x[3][1] // 3600 * 3600
  660. if zzy < int((time.time() + 8 * 3600) // 3600 * 3600 - 8 * 3600 - 3600):
  661. mysql_insert_order(get_zzy_order(st_dt, et_dt))
  662. if zhangdu < int((time.time() + 8 * 3600) // 3600 * 3600 - 8 * 3600 - 3600):
  663. mysql_insert_order(get_zhangdu_order(st_unix, et_unix))
  664. if huasheng < int((time.time() + 8 * 3600) // 3600 * 3600 - 8 * 3600 - 3600):
  665. mysql_insert_order(get_huasheng_order(st_unix, et_unix))
  666. if yuewen < int((time.time() + 8 * 3600) // 3600 * 3600 - 8 * 3600 - 3600):
  667. mysql_insert_order(get_yuewen_order(st_unix, et_unix - 1))
  668. try:
  669. cursor.execute(sql4, data)
  670. db.commit()
  671. z = cursor.fetchall()
  672. print('access', z)
  673. except:
  674. db.rollback()
  675. print('defeat', z)
  676. a = []
  677. for k in x:
  678. a.append(k[1])
  679. y = min(a) // 3600 * 3600
  680. if y < int((time.time() + 8 * 3600) // 3600 * 3600 - 8 * 3600 - 3600):
  681. send_waring_email()
  682. def start_order_job():
  683. st_unix_time = time.time()
  684. st_unix = int((st_unix_time + 8 * 3600) // 3600 * 3600 - 8 * 3600 - 3600)
  685. et_unix = int((st_unix_time + 8 * 3600) // 3600 * 3600 - 8 * 3600)
  686. # et_unix = et_unix - 1
  687. st_dt = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(st_unix)) + '+08:00'
  688. et_dt = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(et_unix)) + '+08:00'
  689. print(st_dt, et_dt)
  690. mysql_insert_order(get_yuewen_order(st_unix, et_unix))
  691. mysql_insert_order(get_zhangdu_order(st_unix, et_unix))
  692. mysql_insert_order(get_ysg_order(st_unix, et_unix))
  693. mysql_insert_order(get_huasheng_order(st_unix, et_unix))
  694. mysql_insert_order(get_zzy_order(st_dt, et_dt))
  695. update_check()
  696. def start_cost_job():
  697. st_unix_time = time.time()
  698. st_unix = int((st_unix_time + 8 * 3600) // 3600 * 3600 - 8 * 3600 - 3600)
  699. et_unix = int((st_unix_time + 8 * 3600) // 3600 * 3600 - 8 * 3600)
  700. # et_unix = et_unix - 1
  701. st_dt = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(st_unix)) + '+08:00'
  702. et_dt = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(et_unix)) + '+08:00'
  703. print(st_dt, et_dt)
  704. mysql_insert_daily_vx(get_daily_vx())
  705. mysql_insert_daily_qq(get_daily_qq())
  706. #start_order_time = '2020-10-13 17:10:00'
  707. start_cost_time = '2020-11-17 18:15:00'
  708. if __name__ == '__main__':
  709. scheduler = BlockingScheduler()
  710. # scheduler.add_job(start_order_job, 'interval', max_instances=10, hours=1, start_date=start_order_time)
  711. scheduler.add_job(start_cost_job, 'interval', max_instances=10, hours=1, start_date=start_cost_time)
  712. scheduler.start()