代码优化

dgp/get_data_hourly.py

@@ -779,7 +779,7 @@ start_cost_time = '2020-10-10 16:35:00'
 
 if __name__ == '__main__':
 	scheduler = BlockingScheduler()
-	scheduler.add_job(start_order_job, 'interval', max_instances=10, hours=1, start_date=start_order_time)
+	# scheduler.add_job(start_order_job, 'interval', max_instances=10, hours=1, start_date=start_order_time)
 	scheduler.add_job(start_cost_job, 'interval', max_instances=10, hours=1, start_date=start_cost_time)
 	scheduler.start()
 

dgp/tests/check_order.py

@@ -30,7 +30,7 @@ from util import db_order_util
 from util import platform_order_api_util
 
 
-def start_all_job():
+def start_order_job():
     start_exec_seconds = date_util.getCurrentSecondTime()
 
     st_unix = date_util.getYesterdayStartTime()
@@ -122,9 +122,9 @@ def start_all_job():
 
 
 if __name__ == '__main__':
-    # start_all_job()
+    # start_order_job()
 
     scheduler = BlockingScheduler()
     #每天凌晨3点到4点的30分钟都执行一次
-    scheduler.add_job(start_all_job, 'cron', hour='3-4', minute='35')
+    scheduler.add_job(start_order_job, 'cron', hour='3-4', minute='35')
     scheduler.start()

dgp/tests/demo/a.py

@@ -1,98 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-get_data_hourly.py
-"""
-
-
-def get_zzy_order(st, et):
-    time1 = time.time()
-    API_list = al.zzy_account_list
-    r = ()
-
-    for x in API_list:
-        my_key = x[0]
-        secert = x[1]
-        stage = x[2]
-        my_sign = md5value(secert + 'key=' + my_key)
-        url = 'https://openapi.818tu.com/partners/channel/channels/list?'
-        parameter = 'key=' + my_key + '&sign=' + my_sign
-        channel_list = requests.get(url + parameter)  # 获取子渠道列表
-
-        if 'data' in channel_list.json().keys():
-            items = channel_list.json()['data']['items']
-        elif len(x) > 3:
-            my_key = x[3]
-            secert = x[4]
-            my_sign = md5value(secert + 'key=' + my_key)
-            url = 'https://openapi.818tu.com/partners/channel/channels/list?'
-            parameter = 'key=' + my_key + '&sign=' + my_sign
-            channel_list = requests.get(url + parameter)
-            items = channel_list.json()['data']['items']
-        else:
-            print(channel_list.json())
-            items = []
-
-        if items != []:
-            for item in items:  # 获取channel_id 后逐个拉取历史orders
-                channel_id = item['id']
-                channel = item['nickname']
-                status = str(1)
-                per_page = str(1000)
-                limit_time = et
-                get_time = st
-                lt = parse.urlencode({'created_at[lt]': limit_time})
-                gt = parse.urlencode({'created_at[gte]': get_time})
-                url_1 = 'https://openapi.818tu.com/partners/channel/orders/list?'
-                my_sign_1 = md5value(secert + 'channel_id=' + str(
-                    channel_id) + '&created_at[gte]=' + get_time + '&created_at[lt]=' + limit_time + '&key=' + my_key + '&per_page=' + per_page + '&status=' + status)
-                parameter_1 = 'channel_id=' + str(
-                    channel_id) + '&' + gt + '&' + lt + '&per_page=' + per_page + '&status=' + status + '&key=' + my_key + '&sign=' + my_sign_1
-                orders = requests.get(url_1 + parameter_1)
-                t = orders.json()['data']['count'] // int(per_page) + 1
-                for page in range(1, t + 1):
-                    my_sign_2 = md5value(secert + 'channel_id=' + str(
-                        channel_id) + '&created_at[gte]=' + get_time + '&created_at[lt]=' + limit_time + '&key=' + my_key + '&page=' + str(
-                        page) + '&per_page=' + per_page + '&status=' + status)
-                    parameter_2 = 'channel_id=' + str(channel_id) + '&' + gt + '&' + lt + '&page=' + str(
-                        page) + '&per_page=' + per_page + '&status=' + status + '&key=' + my_key + '&sign=' + my_sign_2
-                    orders_1 = requests.get(url_1 + parameter_2)
-                    b = orders_1.json()['data']['items']
-
-                    for a in b:
-                        c = {}
-                        c['user_id'] = str(a['member']['openid'])
-                        c['channel'] = channel
-                        c['reg_time'] = a['member']['created_at']
-                        c['channel_id'] = channel_id
-                        c['amount'] = round(a['price'] / 100, 2)
-                        c['order_id'] = str(a['id'])
-                        c['order_time'] = a['created_at']
-                        c['platform'] = '掌中云'
-                        dtime = datetime.datetime.strptime(a['created_at'][0:10], "%Y-%m-%d")
-                        c['date'] = ((int(time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
-                        c['stage'] = stage
-
-                        if str(a['from_novel_id']) != 'None':
-                            c['from_novel'] = a['from_novel']['title']
-                        else:
-                            c['from_novel'] = 'None'
-                        """
-                        del[a['member']]
-                        del[a['referral_link_id']]
-                        del[a['id']]
-                        del[a['created_at']]
-                        del[a['paid_at']]
-                        del a['border_id']
-                        del a['from_novel_id']
-                        del a['status']
-                        del a['price']
-                        del a['agent_uid']
-                        """
-                        x = sorted(c.items(), key=lambda item: item[0])
-                        x = dict(x)
-                        x = tuple(x.values())
-                        r = r + ((x),)
-    cost_time = round((time.time() - time1) / 60, 1)
-    print('get_zzy_order:', len(r), 'cost_minutes:', cost_time)
-    return r

dgp/tests/demo/b.py

@@ -1,100 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-get_order_daily.py
-"""
-
-
-def get_zzy_order(st, et):
-    API_list = al.zzy_account_list
-    url = 'https://openapi.818tu.com/partners/channel/channels/list?'
-
-    for x in API_list:
-        my_key = x[0]
-        secert = x[1]
-        stage = x[2]
-        my_sign = md5value(secert + 'key=' + my_key)
-        parameter = 'key=' + my_key + '&sign=' + my_sign
-        channel_list = requests.get(url + parameter)  # 获取子渠道列表
-
-        if 'data' in channel_list.json().keys():
-            items = channel_list.json()['data']['items']
-        elif len(x) > 3:
-            # print(channel_list.json())
-            my_key = x[3]
-            secert = x[4]
-            my_sign = md5value(secert + 'key=' + my_key)
-            parameter = 'key=' + my_key + '&sign=' + my_sign
-            # url = 'https://openapi.818tu.com/partners/channel/channels/list?'
-            channel_list = requests.get(url + parameter)
-            items = channel_list.json()['data']['items']
-        else:
-            print(channel_list.json())
-            items = []
-
-        for item in items:  # 获取channel_id 后逐个拉取历史orders
-            r = ()
-            channel_id = item['id']
-            channel = item['nickname']
-            status = str(1)
-            per_page = str(1000)
-            limit_time = et
-            get_time = st
-            lt = parse.urlencode({'created_at[lt]': limit_time})
-            gt = parse.urlencode({'created_at[gt]': get_time})
-            url_1 = 'https://openapi.818tu.com/partners/channel/orders/list?'
-            my_sign_1 = md5value(secert + 'channel_id=' + str(
-                channel_id) + '&created_at[gt]=' + get_time + '&created_at[lt]=' + limit_time + '&key=' + my_key + '&per_page=' + per_page + '&status=' + status)
-            parameter_1 = 'channel_id=' + str(
-                channel_id) + '&' + gt + '&' + lt + '&per_page=' + per_page + '&status=' + status + '&key=' + my_key + '&sign=' + my_sign_1
-            orders = requests.get(url_1 + parameter_1)
-            t = orders.json()['data']['count'] // int(per_page) + 1
-            for page in range(1, t + 1):
-                my_sign_2 = md5value(secert + 'channel_id=' + str(
-                    channel_id) + '&created_at[gt]=' + get_time + '&created_at[lt]=' + limit_time + '&key=' + my_key + '&page=' + str(
-                    page) + '&per_page=' + per_page + '&status=' + status)
-                parameter_2 = 'channel_id=' + str(channel_id) + '&' + gt + '&' + lt + '&page=' + str(
-                    page) + '&per_page=' + per_page + '&status=' + status + '&key=' + my_key + '&sign=' + my_sign_2
-                orders_1 = requests.get(url_1 + parameter_2)
-                b = orders_1.json()['data']['items']
-
-                for a in b:
-                    c = {}
-                    c['user_id'] = str(a['member']['openid'])
-                    c['channel'] = channel
-                    c['reg_time'] = a['member']['created_at']
-                    c['channel_id'] = channel_id
-                    c['amount'] = round(a['price'] / 100, 2)
-                    c['order_id'] = str(a['id'])
-                    c['order_time'] = a['created_at']
-                    c['platform'] = '掌中云'
-                    c['stage'] = stage
-                    # c['amount']=a['amount']
-                    dtime = datetime.datetime.strptime(a['created_at'][0:10], "%Y-%m-%d")
-                    c['date'] = ((int(time.mktime(dtime.timetuple())) + 8 * 3600) // 86400) * 86400 - 8 * 3600
-
-                    if str(a['from_novel_id']) != 'None':
-                        c['from_novel'] = a['from_novel']['title']
-                    else:
-                        c['from_novel'] = 'None'
-                    """
-                    del a['member']
-                    del a['referral_link_id']
-                    del a['id']
-                    del a['created_at']
-                    del a['paid_at']
-                    del a['border_id']
-                    del a['from_novel_id']
-                    del a['status']
-                    del a['price']
-                    del a['agent_uid']
-                    """
-                    x = sorted(c.items(), key=lambda item: item[0])
-                    x = dict(x)
-                    x = tuple(x.values())
-                    r = r + ((x),)
-
-            if len(r) > 0:
-                mysql_insert_order(r)
-
-            print('zzy', channel, len(r))

dgp/tests/get_order_hourly.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+__title__ = '每小时获取各平台的的订单数据'
+
+@Time    : 2020/10/11 15:00
+@Author  : Kenny-PC
+@Software: PyCharm
+
+# code is far away from bugs with the god animal protecting
+    I love animals. They taste delicious.
+              ┏┓      ┏┓
+            ┏┛┻━━━┛┻┓
+            ┃      ☃      ┃
+            ┃  ┳┛  ┗┳  ┃
+            ┃      ┻      ┃
+            ┗━┓      ┏━┛
+                ┃      ┗━━━┓
+                ┃  神兽保佑    ┣┓
+                ┃　永无BUG！   ┏┛
+                ┗┓┓┏━┳┓┏┛
+                  ┃┫┫  ┃┫┫
+                  ┗┻┛  ┗┻┛
+"""
+from apscheduler.schedulers.blocking import BlockingScheduler
+
+from util import date_util
+from util import db_order_util
+from util import platform_order_api_util
+
+
+def start_order_job():
+    start_exec_seconds = date_util.getCurrentSecondTime()
+
+    st_unix, et_unix = date_util.getPreviousHourAndCurrentHourSecondTime(start_exec_seconds)
+
+    # st_unix = 1602313200  # 2020/10/10 15:0:0
+    # et_unix = 1602316800  # 2020/10/10 16:0:0
+
+    print('查询开始时间：', st_unix, date_util.getSecondsToDatetime(st_unix))
+    print('查询结束时间：', et_unix, date_util.getSecondsToDatetime(et_unix))
+
+    ########################################## 测试开关 ##########################################
+    # order_list = platform_order_api_util.get_zhangzhongyun_order(st_unix, et_unix)
+    # # print(order_list)
+    # db_order_util.batch_save_order(order_list)
+    #
+    # exit_flag = True
+    # if exit_flag:
+    #     exit()  #这里是为了测试，不让代码继续执行
+    ########################################## 测试开关 ##########################################
+
+    db_order_util.batch_save_order(platform_order_api_util.get_zhangzhongyun_order(st_unix, et_unix))
+    db_order_util.batch_save_order(platform_order_api_util.get_yuewen_order(st_unix, et_unix))
+    db_order_util.batch_save_order(platform_order_api_util.get_huasheng_order(st_unix, et_unix))
+    db_order_util.batch_save_order(platform_order_api_util.get_youshuge_order(st_unix, et_unix))
+    db_order_util.batch_save_order(platform_order_api_util.get_zhangdu_order(st_unix, et_unix))
+
+    print('订单同步执行时间（秒）：', date_util.getCurrentSecondTime() - start_exec_seconds)
+
+
+if __name__ == '__main__':
+    # start_order_job()
+
+    scheduler = BlockingScheduler()
+    #每小时25分钟就执行一次
+    scheduler.add_job(start_order_job, 'cron', minute='25')
+    scheduler.start()

dgp/tests/util/date_util.py

@@ -252,6 +252,17 @@ def getTomorrowEndTime():
     yesterday = datetime.date.today() + datetime.timedelta(days=2)
     return int(time.mktime(time.strptime(str(yesterday), '%Y-%m-%d'))) - 1
 
+def getPreviousHourAndCurrentHourSecondTime(seconds=getCurrentSecondTime()):
+    """
+    description:  接受时间戳并返回上一小时整点时间和当前小时整点时间
+    seconds:      默认当前时间 可传second
+    return:       1602313200,1602316800 -> （previous_hour_second_time,current_hour_second_time）
+    tips:         seconds 不传参则取当前时间
+    """
+    previous_hour_second_time = int(seconds // 3600 * 3600 - 3600)
+    current_hour_second_time = int(seconds // 3600 * 3600)
+    return previous_hour_second_time, current_hour_second_time
+
 
 def getCurrentWeekTime():
     """

dgp/tests/util/db_order_util.py

@@ -38,7 +38,7 @@ def batch_save_order(data):
             num = connect.batch(sql, data)
             # 提交
             connect.commit()
-            print(num, '条订单数据入库成功')
+            print('订单数据最终入库【{num}】条'.format(num=num))
         except Exception as e:
             print('订单数据入库失败：', e)
         finally:

dgp/tests/util/platform_config_util.py

@@ -93,9 +93,24 @@ def get_account_list(platform, account_file_name):
     return account_list
 
 
-def get_zhangzhongyun_format_time(st_unix):
+def get_zhangzhongyun_query_time(st_unix, et_unix):
     """
     description:  掌中云的时间格式比较特殊，需要转换下
-    return:       2020-09-25T00:00:00+08:00 -> str
+    return:       2020-09-25T00:00:00+08:00,2020-09-26T00:00:00+08:00 -> (start_time,end_time)
     """
-    return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(st_unix)) + '+08:00'
+    start_time = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(st_unix)) + '+08:00'
+    end_time = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(et_unix)) + '+08:00'
+    return start_time, end_time
+
+
+def get_youshuge_order_query_time(st_unix, et_unix):
+    """
+    description:  悠书阁的订单查询是按天查询的，没有按小时查询，且每小时查询的时候要判断结束时间要比开始时间大一天，否则查询不到数据
+    return:       2020-10-10,2020-10-11 -> (start_time,end_time)
+    """
+    start_time = time.strftime("%Y-%m-%d", time.localtime(st_unix))
+    end_time = time.strftime("%Y-%m-%d", time.localtime(et_unix))
+    if start_time == end_time:
+        # 如果是同一天，则结束时间加一天
+        end_time = time.strftime("%Y-%m-%d", time.localtime(et_unix + 86400))
+    return start_time, end_time

dgp/tests/util/platform_order_api_util.py

@@ -462,8 +462,7 @@ def get_zhangzhongyun_order(st, et):
 
 def get_zhangzhongyun_order_task(st, et, account, channel):
     # 掌中云的时间格式比较特殊,转换下
-    st = platform_config_util.get_zhangzhongyun_format_time(st)
-    et = platform_config_util.get_zhangzhongyun_format_time(et)
+    get_time, limit_time = platform_config_util.get_zhangzhongyun_query_time(st, et)
 
     order_list = ()
 
@@ -477,8 +476,6 @@ def get_zhangzhongyun_order_task(st, et, account, channel):
     status = str(1)
     page = str(1)
     per_page = str(1000)
-    get_time = st
-    limit_time = et
     gte = parse.urlencode({'created_at[gte]': get_time})  # gte就是ge 大于等于开始时间
     lt = parse.urlencode({'created_at[lt]': limit_time})  # 小于 结束时间
 
@@ -561,6 +558,9 @@ def get_youshuge_order(st, et):
 
 
 def get_youshuge_order_task(st, et, account):
+    # 悠书阁的查询end_date时间必须要大于start_date
+    start_date, end_date = platform_config_util.get_youshuge_order_query_time(st, et)
+
     order_list = ()
 
     host_name = account[0]
@@ -572,8 +572,6 @@ def get_youshuge_order_task(st, et, account):
     url = 'https://novel.youshuge.com/v2/open/orders'
     page = 1
     timestamp = int(time.time())
-    start_date = time.strftime("%Y-%m-%d", time.localtime(st))
-    end_date = time.strftime("%Y-%m-%d", time.localtime(et))
 
     while True:
         sign = md5('channel_id=' + str(channel_id) + '&end_date=' + end_date + '&host_name=' + host_name + '&page='
@@ -590,7 +588,7 @@ def get_youshuge_order_task(st, et, account):
         }
         respone = requests.post(url, params)
         if respone.status_code == 400:
-            print('respone', respone)
+            print('悠书阁订单查询接口respone', respone)
 
         response_result_json = respone.json()
         if 'data' not in response_result_json.keys():