qc-dgp/dgp/tests/get_order_dairly_qiyue.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# 20201120
## 20201120新添加 七悦 平台接口
## 七悦api接口地址 https://vip.zhangwenwenhua.com/uploads/attachments/open-api/open-api.pdf
########################## 七悦订单接口，返回数据格式
'''
{'msg': 'ok', 'code': 0, 
    'data': {
        'current_page': 1, 
        'data': [
            {'id': 167100631495102464, 'money': 30, 'type': 1, 'state': 2, 'user_id': 101280121, 'channel_id': 14880, 'create_time': 1605062783, 'finish_time': 1605062790
            , 'trade_no': '20201111104622_101280121_Ykhp', 'transaction_no': '4200000845202011115590974617', 'user_open_id': 'oxw0X6zGsL6dSX-Y8lsXMhPG1_cY'
            , 'user_is_subscribe': 1, 'user_subscribe_time': 1605045625, 'user_create_time': 1605045625
            , 'referral_url': 'https://wxaef9ff2c63776f6f.wenhuazw.com/index/book/chapter?book_id=10076083&sid=1007608300004&referral_id=727741'
            , 'book_name': '我的超级老婆', 'book_keywords': '', 'wechat_app_id': 'wxaef9ff2c63776f6f', 'wechat_app_name': '炎兵文楼', 'channel_name': '炎兵文楼（趣程）'
            , 'state_desc': '完成', 'type_desc': '书币充值'}
            ,{...}]
        ,'first_page_url': 'https://api.zhangwenwenhua.com/v1/orders?page=1'
        , 'from': 1
        , 'last_page': 1
        , 'last_page_url': 'https://api.zhangwenwenhua.com/v1/orders?page=1'
        , 'next_page_url': None
        , 'path': 'https://api.zhangwenwenhua.com/v1/orders'
        , 'per_page': '10'
        , 'prev_page_url': None
        , 'to': 5
        , 'total': 5
    }
}
'''
#########################
import time 
import requests

from concurrent.futures import ProcessPoolExecutor
import requests
from util import date_util
from util import platform_config_util      ## 账号配置
from util import robust_util
from apscheduler.schedulers.blocking import BlockingScheduler
from util.MySQLConnection import MySQLConnection


def get_qiyue_account_list():
    """
    des cription:  七悦账号列表
    return:       [['stage','token']] ->list
    """
    return platform_config_util.get_account_list('七悦', 'qiyue_account_config.csv')


def get_qiyue_order_task(st,et,account):
    order_list = ()
    url_frequency = 1 
    # 参数
    order_url = "https://api.zhangwenwenhua.com" + "/v1/orders"
    stage = account[0]
    token = account[1]
    page = 1
    size = 50
    
    order_st_date = date_util.getSelfDateStr(int(st-86400),date_format='%Y-%m-%d')
    for i in range((et-st)//86400 + 1):
        statis_unix_time = et - (i+1)*86400
        date = date_util.getSelfDateStr(statis_unix_time,date_format='%Y-%m-%d')

        while True:
            if st == et:
                break

            if url_frequency % 6 == 0 :
                time.sleep(61)
            url_frequency += 1   

            timestamp = int(time.time())
            url = order_url + "?" + "token=" + str(token) +"&timestamp=" + str(timestamp) +"&page=" + str(page) +"&size=" + str(size) +"&date=" + date
            response_result_json = requests.get(url=url).json()

            code = response_result_json['code']
            if code != 0:
                print(stage,'七悦充值接口异常：',response_result_json)
                break

            result_data = response_result_json['data']

            total = result_data['total']
            if total <= 0:
                break


            #total = result_data['total']
            #last_page2 = int(total//size)+1
            last_page = result_data['last_page']
            order_item_list = result_data['data']
            for x in order_item_list: 
                if int(x['state']) != 2:
                    continue  
                y={}
                y['date'] = (int(x['finish_time'])+ 8 * 3600) // 86400 * 86400 - 8 * 3600 
                y['platform'] = '七悦'
                y['channel'] = x['wechat_app_name']      ## 公众号名称
                y['channel_id'] = x['channel_id']     ## 公众号id
                y['from_novel'] = x['book_name']         ## 小说名称
                y['user_id'] = x['user_id']              ## 付费用户uid 
                y['stage'] = stage                       ## 期数
                createTime = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(x['create_time'])) 
                y['order_time']= createTime              ## 订单生成时间
                y['amount']=x['money']               ## 原数据单位：元 
                uid_reg_time = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(x['user_create_time'])) 
                y['reg_time']= uid_reg_time              ## 用户注册时间
                y['order_id']= x['id']                   ## 订单id

                y = sorted(y.items(), key=lambda item:item[0])
                y = dict(y)
                y = tuple(y.values())
                order_list = order_list+((y),)
            '''
            if page == last_page:
                break
            page += 1
            '''
            next_page_url = result_data['next_page_url']
            if next_page_url is None:
                break
            page += 1

    print(f"数据日期-{order_st_date}到{date}-期数-{stage}-获取数据-{len(order_list)}条，例如: {order_list[0:1]}")
    return order_list


@robust_util.catch_exception
def get_qiyue_order(st,et,account_list):
    total_order_list = ()
    start_exec_seconds = date_util.getCurrentSecondTime()
    #account_list = get_qiyue_account_list()

    executor = ProcessPoolExecutor(max_workers=5)

    futures = []
    for account in account_list:
        future = executor.submit(get_qiyue_order_task, st, et, account)
        futures.append(future)

    executor.shutdown(True)

    for future in futures:
        order_list = future.result()
        if len(order_list) > 0:
            total_order_list = order_list + total_order_list

    print('七悦订单数量：', len(total_order_list), '执行时长（秒）：', date_util.getCurrentSecondTime() - start_exec_seconds)
    return total_order_list


def batch_save_order(data):
    if data is None or len(data) == 0:
        print('数据为空，不执行数据库操作！')
    else:
        sql = 'INSERT IGNORE INTO quchen_text.order(amount,channel,channel_id,date,from_novel,order_id,order_time,platform,reg_time,stage,user_id) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);'
        connect = MySQLConnection()
        try:
            num = connect.batch(sql, data)
            # 提交
            connect.commit()
            print('订单数据最终入库【{num}】条'.format(num=num))
        except Exception as e:
            print('订单数据入库失败：', e)
        finally:
            connect.close()


def start_order_job():
    start_exec_seconds = date_util.getCurrentSecondTime()
    #st_unix, et_unix = date_util.getPreviousHourAndCurrentHourSecondTime(start_exec_seconds)
    # st_unix = 1602313200  # 2020/10/10 15:0:0
    # et_unix = 1602316800  # 2020/10/10 16:0:0
    #print('查询开始时间：', st_unix, date_util.getSecondsToDatetime(st_unix))
    #print('查询结束时间：', et_unix, date_util.getSecondsToDatetime(et_unix))
    ## 20201105添加文鼎平台
    st_unix = date_util.getYesterdayStartTime()
    et_unix = date_util.getTodayStartTime()
    st_unix = et_unix - 60
    account_list = get_qiyue_account_list()
    '''
    ## 2个账户测试看看
    account_list = [
        ['趣程26期','eyJpdiI6IlBjV0Z4aVlWNWFUemczdWt5Y0R1QXc9PSIsInZhbHVlIjoiOHZtZUw1Q2RuVnRYd0p1M01WSWoxZz09IiwibWFjIjoiMDE2Mzk5NDFkMDkwZTBmNzU2ZWNmMWMxMzk5YWYwMWIwZmRmZjYzMTgyZDRjYTVkZGY1ZGI5ZGQxNjYyMGRiMSJ9']
        ,['趣程27期','eyJpdiI6IithdDArQ2hhazlxVXJKUXIzK01PQXc9PSIsInZhbHVlIjoiMjAyTEtNOUI2TlkwdklJXC9tNHgwc1E9PSIsIm1hYyI6IjA0YjE0Mjc1MWJjN2VlYmViNWM3YTRiNmI0ZDE5M2QyMDJhMGU3NjRlNTQ4MTliYTgwMzg0ZDNjYmUwY2ZmNWEifQ==']
    ]
    '''
    batch_save_order(get_qiyue_order(st_unix,et_unix,account_list))
    print('订单同步执行时间（秒）：', date_util.getCurrentSecondTime() - start_exec_seconds)



#start_job_time = '2020-11-20 02:00:00'
if __name__ == '__main__':
    scheduler = BlockingScheduler()
    #scheduler.add_job(start_order_job, 'interval', days =1 ,start_date=start_job_time)
    scheduler.add_job(start_order_job,'cron',hour ="02",minute="25")
    scheduler.start()