测试代码修改：

所有平台抓取订单的接口代码已优化

README.md

@@ -25,7 +25,7 @@ Data grabbing platform(DGP)数据采集平台
 - [x] 账号，token配置也需要独立成配置文件，方便实时获取最新的数据，而不需要重启脚本
 - [x] 多线程，异步获取订单数据，目前获取数据都是单线程，太慢了，掌中云2020-09-25的订单数据共12106条，执行时间为529秒，需要特别优化。
 - [ ] 业务报警，钉钉提醒，邮件？
-- [ ] 异常处理
+- [x] 异常处理
 - [ ] 日志输出
 - [ ] 全局常量配置
 

dgp/tests/check_order_new.py

@@ -41,14 +41,16 @@ def start_all_job():
     print('查询开始时间：', st_unix, date_util.getSecondsToDatetime(st_unix))
     print('查询结束时间：', et_unix, date_util.getSecondsToDatetime(et_unix))
 
-    # order_list = platform_order_api_util.get_yuewen_order(st_unix, et_unix)
+    # order_list = platform_order_api_util.get_youshuge_order(st_unix, et_unix)
     # db_order_util.batch_save_order(order_list)
 
     # print(platform_order_api_util.nor())
     # platform_order_api_util.throw_exception()
     # print(platform_order_api_util.nor())
-    #
+
+    ########################################## 测试开关 ##########################################
     # exit_flag = True
+    ########################################## 测试开关 ##########################################
     # if exit_flag:
     #     exit()  #这里是为了测试，不让代码继续执行
 

dgp/tests/util/platform_order_api_util.py

@@ -158,7 +158,7 @@ def get_yuewen_order_task(st, et, account):
                 order_time_unix = int(time.mktime(dtime.timetuple()))
                 order_id = order_item['order_id']
                 if date_util.checkInterval(start_time, end_time, order_time_unix) == False:
-                    print('阅文账号【{key}】, 查询时间【{start_time} - {end_time}】，有不符合该时间范围的订单，订单Id【{order_id}】的时间为【{order_time}】'
+                    print('阅文账号【{key}】，查询时间【{start_time} - {end_time}】，有不符合该时间范围的订单，订单Id【{order_id}】的时间为【{order_time}】'
                           .format(key=email, start_time=date_util.getSecondsToDatetime(start_time),
                                   end_time=date_util.getSecondsToDatetime(end_time), order_id=order_id,
                                   order_time=order_time))
@@ -241,19 +241,8 @@ def get_zhangdu_order_task(st, et, account):
         if x > 0:
             print('掌读跨天数查询:', x)
 
-        params = {
-            'uid': uid,
-            'timestamp': timestamp,
-            'sign': sign,
-            'starttime': starttime,
-            'endtime': endtime
-        }
-        response_result_json = requests.get(url=url, params=params).json()
-        pageCount = response_result_json['data']['pageCount']
-        if pageCount == 0:
-            continue
-
-        for page in range(1, pageCount + 1):  # 分页
+        page = 1
+        while True:
             params = {
                 'uid': uid,
                 'timestamp': timestamp,
@@ -262,27 +251,46 @@ def get_zhangdu_order_task(st, et, account):
                 'endtime': endtime,
                 'page': page
             }
-            list2 = requests.get(url=url, params=params).json()
-            if 'data' in list2.keys():
-                for b in list2['data']['list']:
-                    if b['status'] == '1':
-                        c = {}
-                        c['amount'] = b['amount']
-                        c['channel_id'] = uid
-                        c['order_id'] = str(b['orderno'])
-                        c['order_time'] = b['ctime']
-                        c['user_id'] = b['openid']
-                        c['platform'] = '掌读'
-                        c['channel'] = channel
-                        c['reg_time'] = b['regtime']
-                        c['from_novel'] = ''
-                        c['stage'] = ''
-                        c['date'] = ((int(b['ctime']) + 8 * 3600) // 86400) * 86400 - 8 * 3600
-
-                        x = sorted(c.items(), key=lambda item: item[0])
-                        x = dict(x)
-                        x = tuple(x.values())
-                        order_list = order_list + ((x),)
+            response_result_json = requests.get(url=url, params=params).json()
+            if 'data' not in response_result_json.keys():
+                print('掌读账号【{key}】, 查询时间【{start_time} - {end_time}】，本次请求数据异常，响应报文【{result}】'
+                      .format(key=uid, start_time=date_util.getSecondsToDatetime(starttime),
+                                  end_time=date_util.getSecondsToDatetime(endtime), result=response_result_json))
+                break
+
+            result_data = response_result_json['data']
+            page_count = result_data['pageCount']
+            if page_count == 0:
+                break
+
+            order_item_list = result_data['list']
+            for order_item in order_item_list:
+                if order_item['status'] != '1':#1为已支付
+                    continue
+
+                order = {}
+                order['amount'] = order_item['amount']
+                order['channel_id'] = uid
+                order['order_id'] = str(order_item['orderno'])
+                order['order_time'] = order_item['ctime']
+                order['user_id'] = order_item['openid']
+                order['platform'] = '掌读'
+                order['channel'] = channel
+                order['reg_time'] = order_item['regtime']
+                order['from_novel'] = ''
+                order['stage'] = ''
+                order['date'] = ((int(order_item['ctime']) + 8 * 3600) // 86400) * 86400 - 8 * 3600
+
+                x = sorted(order.items(), key=lambda item: item[0])
+                x = dict(x)
+                x = tuple(x.values())
+                order_list = order_list + ((x),)
+
+            if page == page_count: #是最后一页
+                break
+
+            page = page + 1
+
 
         starttime = starttime + timespace
         endtime = min(et, starttime + timespace)
@@ -315,7 +323,7 @@ def get_huasheng_order(st, et):
         response_result_json = requests.post(url, params).json()
 
         if 'data' not in response_result_json.keys():
-            # print('花生账号【{apiKey}】本次请求数据为空，响应报文【{result}】'.format(apiKey=apiKey, result=response_result_json))
+            print('花生账号【{apiKey}】本次请求数据异常，响应报文【{result}】'.format(apiKey=apiKey, result=response_result_json))
             continue
 
         for merchant in response_result_json['data']:
@@ -365,10 +373,13 @@ def get_huasheng_order_task(st, et, account, merchant):
             }
             response_result_json = requests.post(order_url, order_params).json()
 
-            if 'data' not in response_result_json.keys() or len(response_result_json['data']) == 0:
-                # print('花生账号【{key}】, 渠道【{merchant_id}:{merchant_name}】本次请求数据为空，响应报文【{result}】'
-                #       .format(key=apiKey, merchant_id=merchant_id, merchant_name=merchant_name,
-                #               result=response_result_json))
+            if 'data' not in response_result_json.keys():
+                print('花生账号【{key}】, 查询时间【{date}】, 渠道【{merchant_id}:{merchant_name}】本次请求数据异常，响应报文【{result}】'
+                      .format(key=apiKey, date=date, merchant_id=merchant_id, merchant_name=merchant_name,
+                              result=response_result_json))
+                break
+
+            if len(response_result_json['data']) == 0:
                 break
 
             total_count = response_result_json['count']
@@ -432,7 +443,7 @@ def get_zhangzhongyun_order(st, et):
         response_result_json = requests.get(url + params).json()  # 获取子渠道列表
 
         if 'data' not in response_result_json.keys():
-            # print('掌中云账号【{key}】本次请求数据为空，响应报文【{result}】'.format(key=key, result=response_result_json))
+            print('掌中云账号【{key}】本次请求数据异常，响应报文【{result}】'.format(key=key, result=response_result_json))
             continue
 
         items = response_result_json['data']['items']
@@ -484,8 +495,9 @@ def get_zhangzhongyun_order_task(st, et, account, channel):
         response_result_json = requests.get(order_url + params).json()
 
         if 'data' not in response_result_json.keys():
-            # print('掌中云账号【{key}】, 渠道【{channel_id}:{channel_name}】本次请求数据为空，响应报文【{result}】'
-            #       .format(key=key, channel_id=channel_id, channel_name=channel_name, result=response_result_json))
+            print('掌中云账号【{key}】，查询时间【{start_time} - {end_time}】，渠道【{channel_id}:{channel_name}】本次请求数据异常，响应报文【{result}】'
+                  .format(key=key, start_time=date_util.getSecondsToDatetime(get_time),
+                                  end_time=date_util.getSecondsToDatetime(limit_time),channel_id=channel_id, channel_name=channel_name, result=response_result_json))
             break
 
         total_count = response_result_json['data']['count']  # 总数量
@@ -539,6 +551,7 @@ def get_youshuge_order(st, et):
     for account in account_list:
         future = executor.submit(get_youshuge_order_task, st, et, account)
         futures.append(future)
+
     executor.shutdown(True)
 
     for future in futures:
@@ -552,9 +565,6 @@ def get_youshuge_order(st, et):
 
 def get_youshuge_order_task(st, et, account):
     order_list = ()
-    url = 'https://novel.youshuge.com/v2/open/orders'
-    # 超过100条就需要分页，别问我为什么知道，看代码看出来的
-    max_page_size = 100
 
     host_name = account[0]
     channel_id = int(account[1])
@@ -562,55 +572,67 @@ def get_youshuge_order_task(st, et, account):
     channel = account[3]
     stage = account[4]
 
+    url = 'https://novel.youshuge.com/v2/open/orders'
+    page = 1
     timestamp = int(time.time())
     start_date = time.strftime("%Y-%m-%d", time.localtime(st))
     end_date = time.strftime("%Y-%m-%d", time.localtime(et))
-    page = 1
-    str1 = 'channel_id=' + str(channel_id) + '&end_date=' + end_date + '&host_name=' + host_name + '&page=' + str(
-        page) + '&pay_status=1' + '&start_date=' + start_date + '&time=' + str(timestamp) + '&key=' + secert_key
-    sign = md5(str1).upper()
-    data = {
-        'sign': sign,
-        'host_name': host_name,
-        'time': timestamp,
-        'channel_id': channel_id,
-        'page': page,
-        'pay_status': 1,
-        'start_date': start_date,
-        'end_date': end_date
-    }
-    respone = requests.post(url, data)
-    if respone.status_code == 400:
-        print('respone', respone)
-
-    result_json = respone.json()
-    first_page_order = build_ysg_order_data(channel, channel_id, result_json, stage)
-    order_list = order_list + first_page_order
-    if len(first_page_order) == 0:
-        return order_list
-
-    total_count = result_json['data'][0]['count']
-    if total_count > max_page_size:
-        for i in range((total_count - 1) // max_page_size + 1):
-            timestamp = int(time.time())
-            str1 = 'channel_id=' + str(
-                channel_id) + '&end_date=' + end_date + '&host_name=' + host_name + '&page=' + str(
-                page) + '&pay_status=1' + '&start_date=' + start_date + '&time=' + str(timestamp) + '&key=' + secert_key
-            sign = md5(str1).upper()
-            data2 = {
-                'sign': sign,
-                'host_name': host_name,
-                'time': timestamp,
-                'channel_id': channel_id,
-                'page': page,
-                'pay_status': 1,
-                'start_date': start_date,
-                'end_date': end_date
-            }
-            r2 = requests.post(url, data2).json()
 
-            order_list = order_list + build_ysg_order_data(channel, channel_id, r2, stage)
-            page = page + 1
+    while True:
+        sign = md5('channel_id=' + str(channel_id) + '&end_date=' + end_date + '&host_name=' + host_name + '&page='
+                   + str(page) + '&pay_status=1' + '&start_date=' + start_date + '&time=' + str(timestamp) + '&key=' + secert_key).upper()
+        params = {
+            'sign': sign,
+            'host_name': host_name,
+            'time': timestamp,
+            'channel_id': channel_id,
+            'page': page,
+            'pay_status': 1,
+            'start_date': start_date,
+            'end_date': end_date
+        }
+        respone = requests.post(url, params)
+        if respone.status_code == 400:
+            print('respone', respone)
+
+        response_result_json = respone.json()
+        if 'data' not in response_result_json.keys():
+            print('悠书阁账号【{key}】，查询时间【{start_time} - {end_time}】，渠道【{channel_id}:{channel_name}】本次请求数据异常，响应报文【{result}】'
+                  .format(key=host_name, start_time=start_date, end_time=end_date, channel_id=channel_id, result=response_result_json))
+            break
+
+        order_item_list = response_result_json['data']
+        if len(order_item_list) == 0:
+            break
+
+        for order_item in order_item_list:
+            order = {}
+            dtime = datetime.datetime.strptime(order_item['create_time'][0:10], "%Y-%m-%d")
+            order['date'] = ((int(
+                time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
+            order['order_id'] = order_item['order_num']
+            order['amount'] = round(int(order_item['price']) / 100, 2)
+            order['order_time'] = order_item['create_time']
+            order['channel'] = channel
+            order['from_novel'] = order_item['book_name']
+            order['stage'] = stage
+            order['user_id'] = order_item['openid']
+            order['channel_id'] = channel_id
+            order['platform'] = '悠书阁'
+            order['reg_time'] = order_item['reg_time']
+
+            order = sorted(order.items(), key=lambda item: item[0])
+            order = dict(order)
+            order = tuple(order.values())
+            order_list = order_list + ((order),)
+
+        total_count = order_item_list[0]['count']
+
+        if page == total_count:
+            break
+
+        page = page + 1
+
 
     return order_list