同步服务器上的最新文件

dgp/get_order_daily.py

@@ -2,17 +2,19 @@
 # -*- coding: utf-8 -*-
 """
 Created on Mon May 18 13:13:00 2020
-
 @author: chencong
+## amend1  20200925 11:39:00
+## amend2  
+
 """
 
 import requests
 import hashlib
-import pandas as pd
+
 import time
-from urllib import parse
 import datetime
-import csv
+from urllib import parse
+
 import math
 import pymysql
 from apscheduler.schedulers.blocking import BlockingScheduler
@@ -26,7 +28,7 @@ def md5value(s):
     return md5.hexdigest()
 
 
-
+##《1》阅文
 def get_yuewen_order(st,et):
     url = 'https://open.yuewen.com/cpapi/wxRecharge/querychargelog'
     version = 1
@@ -36,7 +38,9 @@ def get_yuewen_order(st,et):
         start_time = st
         email= secert[0]
         appsecert = secert[1]
+
         for i in range(int((et-st)/86400)):
+            #time.sleep(61) 
             t =()
             end_time = min(start_time+86400,et)
             timestamp = int(time.time())
@@ -44,18 +48,18 @@ def get_yuewen_order(st,et):
             page = 1
             order_status = 2
             data = {
-            	'email':email,
-		        'version':version,
-		        'timestamp':timestamp,
-		        'start_time':start_time,
-		        'end_time':end_time,
-		        'page':page,
-		        'order_status':order_status
-		        #'last_min_id':last_min_id,
-		        #'last_max_id':last_max_id,
-		        #'total_count':total_count,
-		        #'last_page':last_page
-        	}
+                'email':email,
+                'version':version,
+                'timestamp':timestamp,
+                'start_time':start_time,
+                'end_time':end_time,
+                'page':page,
+                'order_status':order_status
+                #'last_min_id':last_min_id,
+                #'last_max_id':last_max_id,
+                #'total_count':total_count,
+                #'last_page':last_page
+            }
             sorted_data = sorted(data.items())
             for k,v in sorted_data:
                 s = s+str(k)+str(v)
@@ -63,21 +67,22 @@ def get_yuewen_order(st,et):
             sign = md5value(appsecert+s).upper()
             
             data1 = {
-            	'email':email,
-		        'version':version,
-		        'timestamp':timestamp,
-		        'start_time':start_time,
-		        'end_time':end_time,
-		        'page':page,
-		        'order_status':order_status,
-		        'sign':sign
-        	}
+                'email':email,
+                'version':version,
+                'timestamp':timestamp,
+                'start_time':start_time,
+                'end_time':end_time,
+                'page':page,
+                'order_status':order_status,
+                'sign':sign
+            }
             list1 = requests.get(url=url,params=data1)
-            
+
             total_count = list1.json()['data']['total_count']
             last_min_id = list1.json()['data']['min_id']
             last_max_id = list1.json()['data']['max_id']
             last_page = list1.json()['data']['page']
+
             if total_count>0:
                 for x in list1.json()['data']['list']:
                     y={}
@@ -105,7 +110,7 @@ def get_yuewen_order(st,et):
                     del x['book_name']
                     del x['report_status']
                     """
-                    y= sorted(y.items(), key=lambda item:item[0])
+                    y = sorted(y.items(), key=lambda item:item[0])
                     y = dict(y)
                     y = tuple(y.values())
                     t = t+((y),)
@@ -113,37 +118,37 @@ def get_yuewen_order(st,et):
             if total_count>100:
                 for page in range(2,math.ceil(total_count/100)+1):
                     data = {
-                    	'email':email,
-				        'version':version,
-				        'timestamp':timestamp,
-				        'start_time':start_time,
-				        'end_time':end_time,
-				        'page':page,
-				        'last_min_id':last_min_id,
-				        'last_max_id':last_max_id,
-				        'total_count':total_count,
-				        'last_page':last_page,
-				        'order_status':order_status
-        			}
+                        'email':email,
+                        'version':version,
+                        'timestamp':timestamp,
+                        'start_time':start_time,
+                        'end_time':end_time,
+                        'page':page,
+                        'last_min_id':last_min_id,
+                        'last_max_id':last_max_id,
+                        'total_count':total_count,
+                        'last_page':last_page,
+                        'order_status':order_status
+                    }
                     sorted_data = sorted(data.items())
                     s1 = ''
                     for k,v in sorted_data:
                         s1 = s1 + str(k)+str(v)
                         sign = md5value(appsecert+s1).upper()
                         data2 = {
-                        	'email':email,
-					        'version':version,
-					        'timestamp':timestamp,
-					        'start_time':start_time,
-					        'end_time':end_time,
-					        'page':page,
-					        'last_min_id':last_min_id,
-					        'last_max_id':last_max_id,
-					        'total_count':total_count,
-					        'last_page':last_page,
-					        'order_status':order_status,
-					        'sign':sign
-						}
+                            'email':email,
+                            'version':version,
+                            'timestamp':timestamp,
+                            'start_time':start_time,
+                            'end_time':end_time,
+                            'page':page,
+                            'last_min_id':last_min_id,
+                            'last_max_id':last_max_id,
+                            'total_count':total_count,
+                            'last_page':last_page,
+                            'order_status':order_status,
+                            'sign':sign
+                        }
                     list2 = requests.get(url=url,params=data2)
                     for x in list2.json()['data']['list']:
                         y={}
@@ -187,92 +192,14 @@ def get_yuewen_order(st,et):
                 mysql_insert_order(t)
 
 
-
-def get_zzy_order(st,et):
-    API_list = al.zzy_account_list
-    
-    for x in API_list:
-        
-        my_key = x[0]
-        secert = x[1]
-        stage  = x[2]
-        my_sign=md5value(secert+'key='+my_key)
-        url = 'https://openapi.818tu.com/partners/channel/channels/list?'
-        parameter = 'key='+my_key+'&sign='+my_sign
-        channel_list = requests.get(url+parameter) #获取子渠道列表
-        
-        items = channel_list.json()['data']['items']
-        
-        for item in items :  #获取channel_id 后逐个拉取历史orders
-            r=()
-            channel_id = item['id']
-            channel = item['nickname']
-            status = str(1)
-            per_page = str(1000)
-            limit_time = et
-            get_time = st
-            lt = parse.urlencode({'created_at[lt]':limit_time})
-            gt = parse.urlencode({'created_at[gt]':get_time})
-            url_1 = 'https://openapi.818tu.com/partners/channel/orders/list?'
-            my_sign_1 = md5value(secert+'channel_id='+str(channel_id)+'&created_at[gt]='+get_time+'&created_at[lt]='+limit_time+'&key='+my_key+'&per_page='+per_page+'&status='+status)
-            parameter_1 = 'channel_id='+str(channel_id)+'&'+gt+'&'+lt+'&per_page='+per_page+'&status='+status+'&key='+my_key+'&sign='+my_sign_1
-            orders = requests.get(url_1+parameter_1)
-            t = orders.json()['data']['count']//int(per_page)+1
-            for page in range(1,t+1):
-                my_sign_2 = md5value(secert+'channel_id='+str(channel_id)+'&created_at[gt]='+get_time+'&created_at[lt]='+limit_time+'&key='+my_key+'&page='+str(page)+'&per_page='+per_page+'&status='+status)
-                parameter_2 ='channel_id='+str(channel_id)+'&'+gt+'&'+lt+'&page='+str(page)+'&per_page='+per_page+'&status='+status+'&key='+my_key+'&sign='+my_sign_2
-                orders_1 = requests.get(url_1+parameter_2)
-                b = orders_1.json()['data']['items']
-                
-                for a in b:
-                    c={}
-                    c['user_id']=str(a['member']['openid'])
-                    c['channel'] = channel
-                    c['reg_time']=a['member']['created_at']               
-                    c['channel_id']=channel_id
-                    c['amount']= round(a['price']/100,2)
-                    c['order_id']=str(a['id'])
-                    c['order_time']=a['created_at']
-                    c['platform']='掌中云'
-                    c['stage']=stage
-                    #c['amount']=a['amount']
-                    dtime = datetime.datetime.strptime(a['created_at'][0:10],"%Y-%m-%d")
-                    c['date']= ((int(time.mktime(dtime.timetuple()))+8*3600)//86400)*86400-8*3600
-                    
-                    if str(a['from_novel_id'])!='None':
-                        c['from_novel']=a['from_novel']['title']
-                    else :
-                        c['from_novel']='None'
-                    """
-                    del a['member']
-                    del a['referral_link_id']
-                    del a['id']
-                    del a['created_at']
-                    del a['paid_at']
-                    del a['border_id']
-                    del a['from_novel_id']
-                    del a['status']
-                    del a['price']
-                    del a['agent_uid']
-                    """
-                    x= sorted(c.items(), key=lambda item:item[0])
-                    x = dict(x)
-                    x = tuple(x.values())   
-                    r = r+((x),)                   
-
-            if len(r)>0:
-                mysql_insert_order(r)
-                
-            print('zzy',channel,len(r))
-
- 
-
+##《2》掌读
 def get_zhangdu_order(st,et):
     secert_list = al.zhangdu_account_list
     url = 'https://api.zhangdu520.com/channel/getorder'
     
     for item in secert_list:  #分渠道
         t=()
+
         uid = item[0]
         appsecert = item[1]
         channel = item[2]
@@ -282,13 +209,14 @@ def get_zhangdu_order(st,et):
         starttime = st
         timespace = 90*3600*24
         endtime = min(et,st+timespace)
+        
         for x in range((et-st)//timespace+1): #分时段
             Params = {
-            	'uid':uid,
-				'timestamp':timestamp,
-				'sign':sign,
-				'starttime':starttime,
-				'endtime':endtime
+                'uid':uid,
+                'timestamp':timestamp,
+                'sign':sign,
+                'starttime':starttime,
+                'endtime':endtime
             }
             list1=requests.get(url=url,params=Params)
             pageCount= list1.json()['data']['pageCount']
@@ -296,16 +224,16 @@ def get_zhangdu_order(st,et):
                 for a in range(1,pageCount+1):  #分页
                     page = a    
                     Params = {
-                    	'uid':uid,
-                 		'timestamp':timestamp,
-                 		'sign':sign,
-                 		'starttime':starttime,
-                 		'endtime':endtime,
-                 		'page':page
-                 	}  
+                        'uid':uid,
+                        'timestamp':timestamp,
+                        'sign':sign,
+                        'starttime':starttime,
+                        'endtime':endtime,
+                        'page':page
+                    }  
                     list2=requests.get(url=url,params=Params).json()
                     if 'data' in list2.keys():
-                        for  b in list2['data']['list']:
+                        for b in list2['data']['list']:
                             c={}
                             c['amount']=b['amount']
                             c['channel_id']=uid
@@ -331,7 +259,8 @@ def get_zhangdu_order(st,et):
                             del b['sourceid']
                             """
                             if b['status']=='1':
-                                del b['status']
+                                #del b['status']
+                                del b
                                 x= sorted(c.items(), key=lambda item:item[0])
                                 x = dict(x)
                                 x = tuple(x.values())
@@ -345,12 +274,11 @@ def get_zhangdu_order(st,et):
         print('掌读',channel,len(t))
 
 
-
+##《3》花生
 def get_huasheng_order(st,et):
     apikey_list = al.huasheng_account_list
     url = 'https://vip.rlcps.cn/api/getMerchants'
 
-
     for key in apikey_list: #获取每个vip账号下的channel_id
         apiKEY=key[0]
         apiSecurity=key[1]
@@ -358,7 +286,7 @@ def get_huasheng_order(st,et):
         timestamp = str(int(time.time()))
         sign = md5value(apiKEY+timestamp+apiSecurity).upper()
         data = {
-        	'apiKey':apiKEY,
+            'apiKey':apiKEY,
             'apiSecurity':apiSecurity,
             'timestamp':timestamp,
             'sign':sign
@@ -371,26 +299,27 @@ def get_huasheng_order(st,et):
             merchant_name = merchant['merchant_name']
             url1 = 'https://vip.rlcps.cn/api/orderList'
             start_time = st
+
             for i in range((et-st)//86400):
                 page = 1
                 date = time.strftime("%Y-%m-%d",time.localtime(start_time))
                 sign = md5value(apiKEY+date+str(merchant_id)+timestamp+apiSecurity).upper()
                 data1 = {
-	                'apiKey':apiKEY,
-	                'apiSecurity':apiSecurity,
-	                'timestamp':timestamp,
-	                'date':date,
-	                'merchant_id':merchant_id,
-	                'sign':sign,
-	                'page':page
-	            }
+                    'apiKey':apiKEY,
+                    'apiSecurity':apiSecurity,
+                    'timestamp':timestamp,
+                    'date':date,
+                    'merchant_id':merchant_id,
+                    'sign':sign,
+                    'page':page
+                }
                 list1 = requests.post(url1,data1).json()
                 
                 if 'data' in list1.keys() and len(list1['data'])>0 :    
                             
                     for i in range(int(math.ceil(list1['count']/500))):
                         data2= {
-                        	'apiKey':apiKEY,
+                            'apiKey':apiKEY,
                             'apiSecurity':apiSecurity,
                             'timestamp':timestamp,
                             'date':date,
@@ -409,7 +338,6 @@ def get_huasheng_order(st,et):
                                 y['order_id']=x['trans_id']
                                 y['order_time'] = x['pay_at']
                                 y['reg_time'] = x['join_at']
-                                #y['date'] = start_time
                                 y['date'] = (start_time+8*3600)//86400*86400-8*3600
                                 y['channel'] = merchant_name
                                 y['channel_id'] = merchant_id                    
@@ -446,11 +374,103 @@ def get_huasheng_order(st,et):
     print(stage,merchant_name,len(t))
 
 
-            
+##《4》掌中云
+def get_zzy_order(st,et):
+    API_list = al.zzy_account_list
+    url = 'https://openapi.818tu.com/partners/channel/channels/list?'
+
+    for x in API_list:
+        my_key = x[0]
+        secert = x[1]
+        stage  = x[2]
+        my_sign=md5value(secert+'key='+my_key)
+        parameter = 'key='+my_key+'&sign='+my_sign
+        channel_list = requests.get(url+parameter) #获取子渠道列表
+
+        if 'data' in channel_list.json().keys():
+            items = channel_list.json()['data']['items']
+        elif len(x)>3:
+            #print(channel_list.json())
+            my_key=x[3]
+            secert=x[4]
+            my_sign=md5value(secert+'key='+my_key)
+            parameter = 'key='+my_key+'&sign='+my_sign
+            #url = 'https://openapi.818tu.com/partners/channel/channels/list?'
+            channel_list = requests.get(url+parameter)
+            items = channel_list.json()['data']['items']
+        else:
+            print(channel_list.json())
+            items = []       
+        
+        for item in items :  #获取channel_id 后逐个拉取历史orders
+            r=()
+            channel_id = item['id']
+            channel = item['nickname']
+            status = str(1)
+            per_page = str(1000)
+            limit_time = et
+            get_time = st
+            lt = parse.urlencode({'created_at[lt]':limit_time})
+            gt = parse.urlencode({'created_at[gt]':get_time})
+            url_1 = 'https://openapi.818tu.com/partners/channel/orders/list?'
+            my_sign_1 = md5value(secert+'channel_id='+str(channel_id)+'&created_at[gt]='+get_time+'&created_at[lt]='+limit_time+'&key='+my_key+'&per_page='+per_page+'&status='+status)
+            parameter_1 = 'channel_id='+str(channel_id)+'&'+gt+'&'+lt+'&per_page='+per_page+'&status='+status+'&key='+my_key+'&sign='+my_sign_1
+            orders = requests.get(url_1+parameter_1)
+            t = orders.json()['data']['count']//int(per_page)+1
+            for page in range(1,t+1):
+                my_sign_2 = md5value(secert+'channel_id='+str(channel_id)+'&created_at[gt]='+get_time+'&created_at[lt]='+limit_time+'&key='+my_key+'&page='+str(page)+'&per_page='+per_page+'&status='+status)
+                parameter_2 ='channel_id='+str(channel_id)+'&'+gt+'&'+lt+'&page='+str(page)+'&per_page='+per_page+'&status='+status+'&key='+my_key+'&sign='+my_sign_2
+                orders_1 = requests.get(url_1+parameter_2)
+                b = orders_1.json()['data']['items']
+                
+                for a in b:
+                    c={}
+                    c['user_id']=str(a['member']['openid'])
+                    c['channel'] = channel
+                    c['reg_time']=a['member']['created_at']               
+                    c['channel_id']=channel_id
+                    c['amount']= round(a['price']/100,2)
+                    c['order_id']=str(a['id'])
+                    c['order_time']=a['created_at']
+                    c['platform']='掌中云'
+                    c['stage']=stage
+                    #c['amount']=a['amount']
+                    dtime = datetime.datetime.strptime(a['created_at'][0:10],"%Y-%m-%d")
+                    c['date']= ((int(time.mktime(dtime.timetuple()))+8*3600)//86400)*86400-8*3600
+                    
+                    if str(a['from_novel_id'])!='None':
+                        c['from_novel']=a['from_novel']['title']
+                    else :
+                        c['from_novel']='None'
+                    """
+                    del a['member']
+                    del a['referral_link_id']
+                    del a['id']
+                    del a['created_at']
+                    del a['paid_at']
+                    del a['border_id']
+                    del a['from_novel_id']
+                    del a['status']
+                    del a['price']
+                    del a['agent_uid']
+                    """
+                    x= sorted(c.items(), key=lambda item:item[0])
+                    x = dict(x)
+                    x = tuple(x.values())   
+                    r = r+((x),)                   
+
+            if len(r)>0:
+                mysql_insert_order(r)
+                
+            print('zzy',channel,len(r))
+
+
+##《5》 悠书阁          
 def get_ysg_order(st,et):
     key_list =al.ysg_account_list
     url='https://novel.youshuge.com/v2/open/orders'
     o = ()
+
     for key in key_list:
         host_name=key[0]
         channel_id = key[1]
@@ -458,21 +478,20 @@ def get_ysg_order(st,et):
         channel = key[3]
         stage = key[4]
         timestamp = int(time.time())
-        start_date=time.strftime("%Y-%m-%d",time.localtime(st))
-        end_date=time.strftime("%Y-%m-%d",time.localtime(et))
-        page=1
+        start_date = time.strftime("%Y-%m-%d",time.localtime(st))
+        end_date = time.strftime("%Y-%m-%d",time.localtime(et))
+        page = 1
         str1 = 'channel_id='+str(channel_id)+'&end_date='+end_date+'&host_name='+host_name+'&page='+str(page)+'&pay_status=1'+'&start_date='+start_date+'&time='+str(timestamp)+'&key='+secert_key
         sign = md5value(str1).upper() 
-
         data = {
-	        'sign':sign,
-	        'host_name':host_name,
-	        'time':timestamp,
-	        'channel_id':channel_id,
-	        'page':page,
-	        'pay_status':1,
-	        'start_date':start_date,
-	        'end_date':end_date
+            'sign':sign,
+            'host_name':host_name,
+            'time':timestamp,
+            'channel_id':channel_id,
+            'page':page,
+            'pay_status':1,
+            'start_date':start_date,
+            'end_date':end_date
         }
         r =requests.post(url,data).json()
         
@@ -483,7 +502,7 @@ def get_ysg_order(st,et):
                     str1 = 'channel_id='+str(channel_id)+'&end_date='+end_date+'&host_name='+host_name+'&page='+str(page)+'&pay_status=1'+'&start_date='+start_date+'&time='+str(timestamp)+'&key='+secert_key
                     sign = md5value(str1).upper() 
                     data2= {
-                    	'sign':sign,
+                        'sign':sign,
                         'host_name':host_name,
                         'time':timestamp,
                         'channel_id':channel_id,
@@ -520,12 +539,14 @@ def get_ysg_order(st,et):
         mysql_insert_order(o)
 
 
-
+## 数据导入表采用replace替换主键orderid的方法
 def mysql_insert_order(data):
     db = pymysql.connect('rm-bp1c9cj79872tx3aaro.mysql.rds.aliyuncs.com','superc','Cc719199895','quchen_text')
     cursor = db.cursor()   
-    sql = 'insert ignore into quchen_text.order_daily (amount,channel,channel_id,date,from_novel,order_id,order_time,platform,reg_time,stage,user_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
+    #sql = 'insert ignore into quchen_text.order_daily (amount,channel,channel_id,date,from_novel,order_id,order_time,platform,reg_time,stage,user_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
     #sql = "update quchen_text.order set amount =%s where platform='掌中云' and order_id =%s"
+    sql = 'replace into quchen_text.order_daily (amount,channel,channel_id,date,from_novel,order_id,order_time,platform,reg_time,stage,user_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
+
     try:
         cursor.executemany(sql,data)
         db.commit()
@@ -533,16 +554,16 @@ def mysql_insert_order(data):
     except:
         db.rollback()
         print('defeat')
-    
+
 
 
 def start_all_job():
-    st_unix = int((time.time()+8*3600)//86400*86400-8*3600-86400) 
-    et_unix = int((time.time()+8*3600)//86400*86400-8*3600)
-    #et_unix = et_unix - 1
+    request_time_stamp = time.time()
+    st_unix = int((request_time_stamp+8*3600)//86400*86400-8*3600-86400) 
+    et_unix = int((request_time_stamp+8*3600)//86400*86400-8*3600)
     st_dt = time.strftime("%Y-%m-%dT%H:%M:%S",time.localtime(st_unix))+'+08:00'
     et_dt = time.strftime("%Y-%m-%dT%H:%M:%S",time.localtime(et_unix))+'+08:00'
-    get_yuewen_order(st_unix,et_unix-1)
+    get_yuewen_order(st_unix,et_unix)
     get_ysg_order(st_unix,et_unix)
     get_zhangdu_order(st_unix,et_unix)  
     get_zzy_order(st_dt,et_dt)
@@ -550,7 +571,7 @@ def start_all_job():
 
 
 
-start_job_time = '2020-09-23 01:05:00'
+start_job_time = '2020-09-26 03:05:00'
 
 
 if __name__ == '__main__':